From 4c078da95a3666c94148d00e82f9dfc8a4a44a95 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 16 Nov 2023 10:51:07 -0500 Subject: [PATCH 01/30] adds plugins.md as starting point for plugin how-to documentation links README.md to plugins.md --- README.md | 2 +- docs/plugins.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 docs/plugins.md diff --git a/README.md b/README.md index df2c668..0b8722b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ It's designed to support various artifacts and expose them through dedicated plu ### Features -* Modular/Extensible via plugins +* Modular/Extensible via [plugins](docs/plugins.md) * Support for YUM repositories (beskar-yum) * Support for static file repositories (beskar-static) diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 0000000..18c9952 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,17 @@ +# Beskar Plugins + +TODOS: +- [ ] Generate with Kun - See build/mage/build.go:86 for autogeneration +- [ ] Map artifact paths with a separator like `/artifacts/ostree/{repo_name}/separtor/path/to/{artifact_name}`. This will be translated to `/v2/%s/blobs/sha256:%s` +- [ ] Create router.rego & data.json so that Beskar knows how to route requests to plugin server(s) +- [ ] mediatypes may be needed for each file type + - [ ] `application/vnd.ciq.ostree.file.v1.file` + - [ ] `application/vnd.ciq.ostree.summary.v1.summary` + + + +See internal/plugins/yum/embedded/router.rego for example +/artifacts/ostree/{repo_name}/separtor/path/to/{artifact_name} + +/2/artifacts/ostree/{repo_name}/files:summary +/2/artifacts/ostree/{repo_name}/files:{sha256("/path/to/{artifact_name}")} \ No newline at end of file From 1a683b2a38a7a7c6b11465e44263cf589037acb1 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 22:58:16 -0500 Subject: [PATCH 02/30] adds IsTLSMiddleware for convenience refactors repository.Manager to non-generic form to allow nil without needing to satisfy generic constraints refactors code depending on generic repository.Manager to new non-generic form various comments --- internal/pkg/beskar/plugin.go | 12 +++ internal/pkg/pluginsrv/webhandler.go | 23 ++++- internal/pkg/repository/handler.go | 13 +++ internal/pkg/repository/manager.go | 29 +++--- internal/plugins/static/api.go | 52 ++++++++-- .../static/pkg/staticrepository/handler.go | 2 +- internal/plugins/static/plugin.go | 19 +--- internal/plugins/yum/api.go | 94 ++++++++++++++++--- .../plugins/yum/pkg/yumrepository/handler.go | 2 +- internal/plugins/yum/plugin.go | 19 +--- 10 files changed, 199 insertions(+), 66 deletions(-) diff --git a/internal/pkg/beskar/plugin.go b/internal/pkg/beskar/plugin.go index 70a8793..89bc066 100644 --- a/internal/pkg/beskar/plugin.go +++ b/internal/pkg/beskar/plugin.go @@ -97,12 +97,16 @@ func (pm *pluginManager) setClientTLSConfig(tlsConfig *tls.Config) { } func (pm *pluginManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // We expect the request to be of the form /artifacts/{plugin_name}/... + // If it is not, we return a 404. matches := artifactsMatch.FindStringSubmatch(r.URL.Path) if len(matches) < 2 { w.WriteHeader(http.StatusNotFound) return } + // Check if the plugin is registered with a name matching the second path component. + // If it is not, we return a 404. pm.pluginsMutex.RLock() pl := pm.plugins[matches[1]] pm.pluginsMutex.RUnlock() @@ -112,6 +116,7 @@ func (pm *pluginManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // Forward the request to the plugin. (The in memory representation of the plugin not the plugin application itself) pl.ServeHTTP(w, r) } @@ -266,6 +271,9 @@ type plugin struct { func (p *plugin) ServeHTTP(w http.ResponseWriter, r *http.Request) { key := r.RemoteAddr + // If the request is for a repository, we need to check if the router has a decision for it. + // If it does, we need to redirect the request to the appropriate location. + // If it does not, we need to use the node hash to find the appropriate node to forward the request to. result, err := p.router.Load().Decision(r, p.registry) if err != nil { dcontext.GetLogger(r.Context()).Errorf("%s router decision error: %s", p.name, err) @@ -403,3 +411,7 @@ func loadPlugins(ctx context.Context) (func(), error) { return wg.Wait, nil } + +// Mountain Team - Lead Developer +// Fuzzball Team - +// Innovation Group - Tech Ambassador () diff --git a/internal/pkg/pluginsrv/webhandler.go b/internal/pkg/pluginsrv/webhandler.go index cea5fa8..06232e0 100644 --- a/internal/pkg/pluginsrv/webhandler.go +++ b/internal/pkg/pluginsrv/webhandler.go @@ -16,9 +16,9 @@ import ( "google.golang.org/protobuf/proto" ) -type webHandler[H repository.Handler] struct { +type webHandler struct { pluginInfo *pluginv1.Info - manager *repository.Manager[H] + manager *repository.Manager } func IsTLS(w http.ResponseWriter, r *http.Request) bool { @@ -29,7 +29,22 @@ func IsTLS(w http.ResponseWriter, r *http.Request) bool { return true } -func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { +// IsTLSMiddleware is a middleware that checks if the request is TLS. This is a convenience wrapper around IsTLS. +func IsTLSMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !IsTLS(w, r) { + return + } + next.ServeHTTP(w, r) + }) +} + +func (wh *webHandler) event(w http.ResponseWriter, r *http.Request) { + if wh.manager == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + if !IsTLS(w, r) { return } @@ -84,7 +99,7 @@ func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { } } -func (wh *webHandler[H]) info(w http.ResponseWriter, r *http.Request) { +func (wh *webHandler) info(w http.ResponseWriter, r *http.Request) { if !IsTLS(w, r) { return } diff --git a/internal/pkg/repository/handler.go b/internal/pkg/repository/handler.go index 6797612..60787da 100644 --- a/internal/pkg/repository/handler.go +++ b/internal/pkg/repository/handler.go @@ -29,13 +29,26 @@ func (hp HandlerParams) Remove(repository string) { hp.remove(repository) } +// Handler - Interface for handling events for a repository. type Handler interface { + // QueueEvent - Called when a new event is received. If store is true, the event should be stored in the database. + // Note: Avoid performing any long-running operations in this function. QueueEvent(event *eventv1.EventPayload, store bool) error + + // Started - Returns true if the handler has started. Started() bool + + // Start - Called when the handler should start processing events. + // This is your chance to set up any resources, e.g., database connections, run loops, etc. + // This will only be called once. Start(context.Context) + + // Stop - Called when the handler should stop processing events and clean up resources. Stop() } +// RepoHandler - A partial default implementation of the Handler interface that provides some common functionality. +// You can embed this in your own handler to get some default functionality, e.g., an event queue. type RepoHandler struct { Repository string Params *HandlerParams diff --git a/internal/pkg/repository/manager.go b/internal/pkg/repository/manager.go index f726aa2..724cdf7 100644 --- a/internal/pkg/repository/manager.go +++ b/internal/pkg/repository/manager.go @@ -11,20 +11,21 @@ import ( "go.ciq.dev/beskar/internal/pkg/log" ) -type Manager[H Handler] struct { +type HandlerMap = map[string]Handler + +type HandlerFactory = func(*slog.Logger, *RepoHandler) Handler + +type Manager struct { repositoryMutex sync.RWMutex - repositories map[string]H + repositories HandlerMap repositoryParams *HandlerParams - newHandler func(*slog.Logger, *RepoHandler) H + newHandler func(*slog.Logger, *RepoHandler) Handler } -func NewManager[H Handler]( - params *HandlerParams, - newHandler func(*slog.Logger, *RepoHandler) H, -) *Manager[H] { - m := &Manager[H]{ - repositories: make(map[string]H), +func NewManager(params *HandlerParams, newHandler HandlerFactory) *Manager { + m := &Manager{ + repositories: make(HandlerMap), repositoryParams: params, newHandler: newHandler, } @@ -33,13 +34,13 @@ func NewManager[H Handler]( return m } -func (m *Manager[H]) remove(repository string) { +func (m *Manager) remove(repository string) { m.repositoryMutex.Lock() delete(m.repositories, repository) m.repositoryMutex.Unlock() } -func (m *Manager[H]) Get(ctx context.Context, repository string) H { +func (m *Manager) Get(ctx context.Context, repository string) Handler { m.repositoryMutex.Lock() r, ok := m.repositories[repository] @@ -73,7 +74,7 @@ func (m *Manager[H]) Get(ctx context.Context, repository string) H { return rh } -func (m *Manager[H]) Has(repository string) bool { +func (m *Manager) Has(repository string) bool { m.repositoryMutex.RLock() _, ok := m.repositories[repository] m.repositoryMutex.RUnlock() @@ -81,10 +82,10 @@ func (m *Manager[H]) Has(repository string) bool { return ok } -func (m *Manager[H]) GetAll() map[string]H { +func (m *Manager) GetAll() HandlerMap { m.repositoryMutex.RLock() - handlers := make(map[string]H) + handlers := make(HandlerMap) for name, handler := range m.repositories { handlers[name] = handler } diff --git a/internal/plugins/static/api.go b/internal/plugins/static/api.go index 573e9fa..fc81d21 100644 --- a/internal/plugins/static/api.go +++ b/internal/plugins/static/api.go @@ -5,6 +5,7 @@ package static import ( "context" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" @@ -18,44 +19,83 @@ func checkRepository(repository string) error { return nil } +func (p *Plugin) getHandlerForRepository(ctx context.Context, repository string) (*staticrepository.Handler, error) { + h, ok := p.repositoryManager.Get(ctx, repository).(*staticrepository.Handler) + if !ok { + return nil, werror.Wrapf(gcode.ErrNotFound, "repository %q does not exist in the required form", repository) + } + + return h, nil +} + func (p *Plugin) DeleteRepository(ctx context.Context, repository string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.DeleteRepository(ctx) } func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryLogs(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryLogs(ctx, page) } func (p *Plugin) RemoveRepositoryFile(ctx context.Context, repository string, tag string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).RemoveRepositoryFile(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.RemoveRepositoryFile(ctx, tag) } func (p *Plugin) GetRepositoryFileByTag(ctx context.Context, repository string, tag string) (repositoryFile *apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryFileByTag(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryFileByTag(ctx, tag) } func (p *Plugin) GetRepositoryFileByName(ctx context.Context, repository string, name string) (repositoryFile *apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryFileByName(ctx, name) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryFileByName(ctx, name) } func (p *Plugin) ListRepositoryFiles(ctx context.Context, repository string, page *apiv1.Page) (repositoryFiles []*apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryFiles(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryFiles(ctx, page) } diff --git a/internal/plugins/static/pkg/staticrepository/handler.go b/internal/plugins/static/pkg/staticrepository/handler.go index b1af022..0939a86 100644 --- a/internal/plugins/static/pkg/staticrepository/handler.go +++ b/internal/plugins/static/pkg/staticrepository/handler.go @@ -32,7 +32,7 @@ type Handler struct { statusDB *staticdb.StatusDB } -func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) repository.Handler { return &Handler{ RepoHandler: repoHandler, repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), diff --git a/internal/plugins/static/plugin.go b/internal/plugins/static/plugin.go index 4aab58a..ded40d8 100644 --- a/internal/plugins/static/plugin.go +++ b/internal/plugins/static/plugin.go @@ -41,11 +41,11 @@ type Plugin struct { ctx context.Context config pluginsrv.Config - repositoryManager *repository.Manager[*staticrepository.Handler] + repositoryManager *repository.Manager handlerParams *repository.HandlerParams } -var _ pluginsrv.Service[*staticrepository.Handler] = &Plugin{} +var _ pluginsrv.Service = &Plugin{} func New(ctx context.Context, beskarStaticConfig *config.BeskarStaticConfig) (*Plugin, error) { logger, err := beskarStaticConfig.Log.Logger(log.ContextHandler) @@ -65,7 +65,7 @@ func New(ctx context.Context, beskarStaticConfig *config.BeskarStaticConfig) (*P Dir: filepath.Join(beskarStaticConfig.DataDir, "_repohandlers_"), }, } - plugin.repositoryManager = repository.NewManager[*staticrepository.Handler]( + plugin.repositoryManager = repository.NewManager( plugin.handlerParams, staticrepository.NewHandler, ) @@ -124,7 +124,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/static/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -143,15 +143,6 @@ func (p *Plugin) Context() context.Context { return p.ctx } -func (p *Plugin) RepositoryManager() *repository.Manager[*staticrepository.Handler] { +func (p *Plugin) RepositoryManager() *repository.Manager { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} diff --git a/internal/plugins/yum/api.go b/internal/plugins/yum/api.go index 0d69118..627fcc0 100644 --- a/internal/plugins/yum/api.go +++ b/internal/plugins/yum/api.go @@ -5,6 +5,7 @@ package yum import ( "context" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" @@ -18,86 +19,155 @@ func checkRepository(repository string) error { return nil } +func (p *Plugin) getHandlerForRepository(ctx context.Context, repository string) (*yumrepository.Handler, error) { + h, ok := p.repositoryManager.Get(ctx, repository).(*yumrepository.Handler) + if !ok { + return nil, werror.Wrapf(gcode.ErrNotFound, "repository %q does not exist in the required form", repository) + } + + return h, nil +} + func (p *Plugin) CreateRepository(ctx context.Context, repository string, properties *apiv1.RepositoryProperties) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).CreateRepository(ctx, properties) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.CreateRepository(ctx, properties) } func (p *Plugin) DeleteRepository(ctx context.Context, repository string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.DeleteRepository(ctx) } func (p *Plugin) UpdateRepository(ctx context.Context, repository string, properties *apiv1.RepositoryProperties) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).UpdateRepository(ctx, properties) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.UpdateRepository(ctx, properties) } func (p *Plugin) GetRepository(ctx context.Context, repository string) (properties *apiv1.RepositoryProperties, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepository(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepository(ctx) } func (p *Plugin) SyncRepository(ctx context.Context, repository string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.SyncRepository(ctx) } func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositorySyncStatus(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositorySyncStatus(ctx) } func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryLogs(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryLogs(ctx, page) } func (p *Plugin) RemoveRepositoryPackage(ctx context.Context, repository string, id string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).RemoveRepositoryPackage(ctx, id) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.RemoveRepositoryPackage(ctx, id) } func (p *Plugin) RemoveRepositoryPackageByTag(ctx context.Context, repository string, tag string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).RemoveRepositoryPackageByTag(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.RemoveRepositoryPackageByTag(ctx, tag) } func (p *Plugin) GetRepositoryPackage(ctx context.Context, repository string, id string) (repositoryPackage *apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryPackage(ctx, id) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryPackage(ctx, id) } func (p *Plugin) GetRepositoryPackageByTag(ctx context.Context, repository string, tag string) (repositoryPackage *apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryPackageByTag(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryPackageByTag(ctx, tag) } func (p *Plugin) ListRepositoryPackages(ctx context.Context, repository string, page *apiv1.Page) (repositoryPackages []*apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryPackages(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryPackages(ctx, page) } diff --git a/internal/plugins/yum/pkg/yumrepository/handler.go b/internal/plugins/yum/pkg/yumrepository/handler.go index 2bc4a61..c580388 100644 --- a/internal/plugins/yum/pkg/yumrepository/handler.go +++ b/internal/plugins/yum/pkg/yumrepository/handler.go @@ -52,7 +52,7 @@ type Handler struct { mirrorURLs []*url.URL } -func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) repository.Handler { return &Handler{ RepoHandler: repoHandler, repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), diff --git a/internal/plugins/yum/plugin.go b/internal/plugins/yum/plugin.go index b5953e0..cb205a8 100644 --- a/internal/plugins/yum/plugin.go +++ b/internal/plugins/yum/plugin.go @@ -41,11 +41,11 @@ type Plugin struct { ctx context.Context config pluginsrv.Config - repositoryManager *repository.Manager[*yumrepository.Handler] + repositoryManager *repository.Manager handlerParams *repository.HandlerParams } -var _ pluginsrv.Service[*yumrepository.Handler] = &Plugin{} +var _ pluginsrv.Service = &Plugin{} func New(ctx context.Context, beskarYumConfig *config.BeskarYumConfig) (*Plugin, error) { logger, err := beskarYumConfig.Log.Logger(log.ContextHandler) @@ -65,7 +65,7 @@ func New(ctx context.Context, beskarYumConfig *config.BeskarYumConfig) (*Plugin, Dir: filepath.Join(beskarYumConfig.DataDir, "_repohandlers_"), }, } - plugin.repositoryManager = repository.NewManager[*yumrepository.Handler]( + plugin.repositoryManager = repository.NewManager( plugin.handlerParams, yumrepository.NewHandler, ) @@ -127,7 +127,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/yum/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -146,15 +146,6 @@ func (p *Plugin) Context() context.Context { return p.ctx } -func (p *Plugin) RepositoryManager() *repository.Manager[*yumrepository.Handler] { +func (p *Plugin) RepositoryManager() *repository.Manager { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} From f94b00e07087fd5bf90c14d3edfc5f3b8fc169a7 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 22:59:51 -0500 Subject: [PATCH 03/30] ignore intellij project state files --- .gitignore | 2 ++ internal/plugins/ostree/api.go | 19 +++++++++++++++ pkg/plugins/ostree/api/v1/api.go | 40 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 internal/plugins/ostree/api.go create mode 100644 pkg/plugins/ostree/api/v1/api.go diff --git a/.gitignore b/.gitignore index 1f65aa4..d705dbd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ build/output .envrc.local vendor go.work.sum + +.idea diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go new file mode 100644 index 0000000..aca8687 --- /dev/null +++ b/internal/plugins/ostree/api.go @@ -0,0 +1,19 @@ +package ostree + +import ( + "context" + "errors" + "github.com/RussellLuo/kun/pkg/werror" + "github.com/RussellLuo/kun/pkg/werror/gcode" +) + +type apiService struct{} + +func newAPIService() *apiService { + return &apiService{} +} + +func (o *apiService) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { + //TODO implement me + return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) +} diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go new file mode 100644 index 0000000..9316106 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/api.go @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package apiv1 + +import ( + "context" + "regexp" +) + +const ( + RepositoryRegex = "^(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)$" + URLPath = "/artifacts/ostree/api/v1" +) + +var repositoryMatcher = regexp.MustCompile(RepositoryRegex) + +func RepositoryMatch(repository string) bool { + return repositoryMatcher.MatchString(repository) +} + +type Page struct { + Size int + Token string +} + +// OSTree is used for managing ostree repositories. +// This is the API documentation of OSTree. +// +//kun:oas title=OSTree Repository Management API +//kun:oas version=1.0.0 +//kun:oas basePath=/artifacts/ostree/api/v1 +//kun:oas docsPath=/doc/swagger.yaml +//kun:oas tags=static +type OSTree interface { + // Mirror a static repository. + //kun:op POST /repository/mirror + //kun:success statusCode=200 + MirrorRepository(ctx context.Context, repository string, depth int) (err error) +} From 60456e8f7c1032a6f561dc72d12b90d1ccb19154 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 23:01:44 -0500 Subject: [PATCH 04/30] refactors service to a non-generic form --- internal/pkg/pluginsrv/service.go | 48 +++++++++++++++++++------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/internal/pkg/pluginsrv/service.go b/internal/pkg/pluginsrv/service.go index 8c72615..59e81e1 100644 --- a/internal/pkg/pluginsrv/service.go +++ b/internal/pkg/pluginsrv/service.go @@ -34,14 +34,22 @@ type Config struct { Info *pluginv1.Info } -type Service[H repository.Handler] interface { +type Service interface { + // Start starts the service's HTTP server. Start(http.RoundTripper, *mtls.CAPEM, *gossip.BeskarMeta) error + + // Context returns the service's context. Context() context.Context + + // Config returns the service's configuration. Config() Config - RepositoryManager() *repository.Manager[H] + + // RepositoryManager returns the service's repository manager. + // For plugin's without a repository manager, this method should return nil. + RepositoryManager() *repository.Manager } -func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn error) { +func Serve(ln net.Listener, service Service) (errFn error) { ctx := service.Context() errCh := make(chan error) @@ -93,6 +101,23 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err } repoManager := service.RepositoryManager() + if repoManager != nil { + // Gracefully shutdown repository handlers + defer func() { + var wg sync.WaitGroup + for name, handler := range repoManager.GetAll() { + wg.Add(1) + + go func(name string, handler repository.Handler) { + logger.Info("stopping repository handler", "repository", name) + handler.Stop() + logger.Info("repository handler stopped", "repository", name) + wg.Done() + }(name, handler) + } + wg.Wait() + }() + } ticker := time.NewTicker(time.Second * 5) @@ -104,7 +129,7 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err case beskarMeta := <-beskarMetaCh: ticker.Stop() - wh := webHandler[H]{ + wh := webHandler{ pluginInfo: serviceConfig.Info, manager: repoManager, } @@ -139,21 +164,6 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err _ = server.Shutdown(ctx) - var wg sync.WaitGroup - - for name, handler := range repoManager.GetAll() { - wg.Add(1) - - go func(name string, handler repository.Handler) { - logger.Info("stopping repository handler", "repository", name) - handler.Stop() - logger.Info("repository handler stopped", "repository", name) - wg.Done() - }(name, handler) - } - - wg.Wait() - return serverErr } From e3c4d748f59b9a2c8627322cc2be2d400c10fbed Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 23:03:22 -0500 Subject: [PATCH 05/30] initial implementation of ostree plugin (without mirroring) refactors yum plugin executable to use new non-generic Serve refactors static plugin executable to use new non-generic Serve --- .idea/beskar.iml | 9 ++ build/mage/build.go | 14 +++ cmd/beskar-ostree/main.go | 68 +++++++++++ cmd/beskar-static/main.go | 3 +- cmd/beskar-yum/main.go | 3 +- internal/plugins/ostree/embedded/data.json | 27 +++++ internal/plugins/ostree/embedded/router.rego | 64 ++++++++++ .../ostree/pkg/config/beskar-ostree.go | 93 ++++++++++++++ .../pkg/config/default/beskar-ostree.yaml | 16 +++ internal/plugins/ostree/plugin.go | 113 ++++++++++++++++++ pkg/plugins/ostree/api/v1/endpoint.go | 49 ++++++++ pkg/plugins/ostree/api/v1/http.go | 58 +++++++++ pkg/plugins/ostree/api/v1/http_client.go | 84 +++++++++++++ pkg/plugins/ostree/api/v1/oas2.go | 73 +++++++++++ 14 files changed, 670 insertions(+), 4 deletions(-) create mode 100644 .idea/beskar.iml create mode 100644 cmd/beskar-ostree/main.go create mode 100644 internal/plugins/ostree/embedded/data.json create mode 100644 internal/plugins/ostree/embedded/router.rego create mode 100644 internal/plugins/ostree/pkg/config/beskar-ostree.go create mode 100644 internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml create mode 100644 internal/plugins/ostree/plugin.go create mode 100644 pkg/plugins/ostree/api/v1/endpoint.go create mode 100644 pkg/plugins/ostree/api/v1/http.go create mode 100644 pkg/plugins/ostree/api/v1/http_client.go create mode 100644 pkg/plugins/ostree/api/v1/oas2.go diff --git a/.idea/beskar.iml b/.idea/beskar.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/beskar.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/build/mage/build.go b/build/mage/build.go index 0ca1ccd..9784038 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -61,6 +61,7 @@ const ( beskarctlBinary = "beskarctl" beskarYUMBinary = "beskar-yum" beskarStaticBinary = "beskar-static" + beskarOSTreeBinary = "beskar-ostree" ) var binaries = map[string]binaryConfig{ @@ -103,6 +104,18 @@ var binaries = map[string]binaryConfig{ useProto: true, baseImage: "alpine:3.17", }, + beskarOSTreeBinary: { + configFiles: map[string]string{ + "internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml": "/etc/beskar/beskar-ostree.yaml", + }, + genAPI: &genAPI{ + path: "pkg/plugins/ostree/api/v1", + filename: "api.go", + interfaceName: "OSTree", + }, + useProto: true, + baseImage: "alpine:3.17", + }, } type Build mg.Namespace @@ -143,6 +156,7 @@ func (b Build) Plugins(ctx context.Context) { ctx, mg.F(b.Plugin, beskarYUMBinary), mg.F(b.Plugin, beskarStaticBinary), + mg.F(b.Plugin, beskarOSTreeBinary), ) } diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go new file mode 100644 index 0000000..1409d2f --- /dev/null +++ b/cmd/beskar-ostree/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "flag" + "fmt" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" + "go.ciq.dev/beskar/internal/plugins/ostree" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + "go.ciq.dev/beskar/pkg/sighandler" + "go.ciq.dev/beskar/pkg/version" + "log" + "net" + "os" + "syscall" +) + +var configDir string + +func serve(beskarOSTreeCmd *flag.FlagSet) error { + if err := beskarOSTreeCmd.Parse(os.Args[1:]); err != nil { + return err + } + + errCh := make(chan error) + + ctx, wait := sighandler.New(errCh, syscall.SIGTERM, syscall.SIGINT) + + beskarOSTreeConfig, err := config.ParseBeskarOSTreeConfig(configDir) + if err != nil { + return err + } + + ln, err := net.Listen("tcp", beskarOSTreeConfig.Addr) + if err != nil { + return err + } + defer ln.Close() + + plugin, err := ostree.New(ctx, beskarOSTreeConfig) + if err != nil { + return err + } + + go func() { + errCh <- pluginsrv.Serve(ln, plugin) + }() + + return wait(false) +} + +func main() { + beskarOSTreeCmd := flag.NewFlagSet("beskar-ostree", flag.ExitOnError) + beskarOSTreeCmd.StringVar(&configDir, "config-dir", "", "configuration directory") + + subCommand := "" + if len(os.Args) > 1 { + subCommand = os.Args[1] + } + + switch subCommand { + case "version": + fmt.Println(version.Semver) + default: + if err := serve(beskarOSTreeCmd); err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/beskar-static/main.go b/cmd/beskar-static/main.go index 9e4ca2a..9f68029 100644 --- a/cmd/beskar-static/main.go +++ b/cmd/beskar-static/main.go @@ -14,7 +14,6 @@ import ( "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/static" "go.ciq.dev/beskar/internal/plugins/static/pkg/config" - "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" ) @@ -47,7 +46,7 @@ func serve(beskarStaticCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve[*staticrepository.Handler](ln, plugin) + errCh <- pluginsrv.Serve(ln, plugin) }() return wait(false) diff --git a/cmd/beskar-yum/main.go b/cmd/beskar-yum/main.go index 30ec486..089a666 100644 --- a/cmd/beskar-yum/main.go +++ b/cmd/beskar-yum/main.go @@ -14,7 +14,6 @@ import ( "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/yum" "go.ciq.dev/beskar/internal/plugins/yum/pkg/config" - "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" ) @@ -47,7 +46,7 @@ func serve(beskarYumCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve[*yumrepository.Handler](ln, plugin) + errCh <- pluginsrv.Serve(ln, plugin) }() return wait(false) diff --git a/internal/plugins/ostree/embedded/data.json b/internal/plugins/ostree/embedded/data.json new file mode 100644 index 0000000..536198d --- /dev/null +++ b/internal/plugins/ostree/embedded/data.json @@ -0,0 +1,27 @@ +{ + "routes": [ + { + "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/file/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", + "methods": [ + "GET", + "HEAD" + ] + }, + { + "pattern": "^/artifacts/ostree/api/v1/doc/(.*)$", + "body": false + }, + { + "pattern": "^/artifacts/ostree/api/v1/(.*)$", + "body": true, + "body_key": "repository" + } + ], + "mediatype": { + "file": "application/vnd.ciq.ostree.v1.file" + }, + "tags": [ + "summary", + "config" + ] +} \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego new file mode 100644 index 0000000..42d8053 --- /dev/null +++ b/internal/plugins/ostree/embedded/router.rego @@ -0,0 +1,64 @@ +package router + +import future.keywords.if +import future.keywords.in + +default output = {"repository": "", "redirect_url": "", "found": false} + +filename_checksum(filename) = checksum if { + filename in data.tags + checksum := filename +} else = checksum { + checksum := crypto.md5(filename) +} + +blob_url(repo, filename) = url { + digest := oci.blob_digest(sprintf("%s:%s", [repo, filename_checksum(filename)]), "mediatype", data.mediatype.file) + url := { + "url": sprintf("/v2/%s/blobs/sha256:%s", [repo, digest]), + "found": digest != "", + } +} + +output = obj { + some index + input.method in data.routes[index].methods + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + redirect := blob_url( + sprintf("%s/files", [match[1]]), + match[2], + ) + obj := { + "repository": match[1], + "redirect_url": redirect.url, + "found": redirect.found + } +} else = obj if { + data.routes[index].body == true + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + repo := object.get({}, data.routes[index].body_key, "") + obj := { + "repository": repo, + "redirect_url": "", + "found": repo != "" + } +} else = obj { + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + obj := { + "repository": "", + "redirect_url": "", + "found": true + } +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go new file mode 100644 index 0000000..d692cfe --- /dev/null +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -0,0 +1,93 @@ +package config + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "github.com/distribution/distribution/v3/configuration" + "go.ciq.dev/beskar/internal/pkg/config" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" + "io" + "os" + "path/filepath" + "reflect" + "strings" +) + +const ( + BeskarOSTreeConfigFile = "beskar-ostree.yaml" + DefaultBeskarOSTreeDataDir = "/tmp/beskar-ostree" +) + +//go:embed default/beskar-ostree.yaml +var defaultBeskarOSTreeConfig string + +type BeskarOSTreeConfig struct { + Version string `yaml:"version"` + Log log.Config `yaml:"log"` + Addr string `yaml:"addr"` + Gossip gossip.Config `yaml:"gossip"` + Profiling bool `yaml:"profiling"` + DataDir string `yaml:"datadir"` + ConfigDirectory string `yaml:"-"` +} + +type BeskarOSTreeConfigV1 BeskarOSTreeConfig + +func ParseBeskarOSTreeConfig(dir string) (*BeskarOSTreeConfig, error) { + customDir := false + filename := filepath.Join(config.DefaultConfigDir, BeskarOSTreeConfigFile) + if dir != "" { + filename = filepath.Join(dir, BeskarOSTreeConfigFile) + customDir = true + } + + configDir := filepath.Dir(filename) + + var configReader io.Reader + + f, err := os.Open(filename) + if err != nil { + if !errors.Is(err, os.ErrNotExist) || customDir { + return nil, err + } + configReader = strings.NewReader(defaultBeskarOSTreeConfig) + configDir = "" + } else { + defer f.Close() + configReader = f + } + + configBuffer := new(bytes.Buffer) + if _, err := io.Copy(configBuffer, configReader); err != nil { + return nil, err + } + + configParser := configuration.NewParser("beskarostree", []configuration.VersionedParseInfo{ + { + Version: configuration.MajorMinorVersion(1, 0), + ParseAs: reflect.TypeOf(BeskarOSTreeConfigV1{}), + ConversionFunc: func(c interface{}) (interface{}, error) { + if v1, ok := c.(*BeskarOSTreeConfigV1); ok { + v1.ConfigDirectory = configDir + return (*BeskarOSTreeConfig)(v1), nil + } + return nil, fmt.Errorf("expected *BeskarOSTreeConfigV1, received %#v", c) + }, + }, + }) + + beskarOSTreeConfig := new(BeskarOSTreeConfig) + + if err := configParser.Parse(configBuffer.Bytes(), beskarOSTreeConfig); err != nil { + return nil, err + } + + if beskarOSTreeConfig.DataDir == "" { + beskarOSTreeConfig.DataDir = DefaultBeskarOSTreeDataDir + } + + return beskarOSTreeConfig, nil +} diff --git a/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml b/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml new file mode 100644 index 0000000..043fb9b --- /dev/null +++ b/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml @@ -0,0 +1,16 @@ +version: 1.0 + +addr: 0.0.0.0:5200 + +log: + level: debug + format: json + +profiling: true +datadir: /tmp/beskar-ostree + +gossip: + addr: 0.0.0.0:5201 + key: XD1IOhcp0HWFgZJ/HAaARqMKJwfMWtz284Yj7wxmerA= + peers: + - 127.0.0.1:5102 diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go new file mode 100644 index 0000000..8e6fbf1 --- /dev/null +++ b/internal/plugins/ostree/plugin.go @@ -0,0 +1,113 @@ +package ostree + +import ( + "context" + _ "embed" + "github.com/RussellLuo/kun/pkg/httpcodec" + "github.com/go-chi/chi" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" + "go.ciq.dev/beskar/internal/pkg/repository" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + pluginv1 "go.ciq.dev/beskar/pkg/api/plugin/v1" + "go.ciq.dev/beskar/pkg/mtls" + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" + "go.ciq.dev/beskar/pkg/version" + "net/http" + "net/http/pprof" +) + +const ( + PluginName = "ostree" + PluginAPIPathPattern = "/artifacts/ostree/api/v1" +) + +//go:embed embedded/router.rego +var routerRego []byte + +//go:embed embedded/data.json +var routerData []byte + +type Plugin struct { + ctx context.Context + config pluginsrv.Config +} + +func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*Plugin, error) { + logger, err := beskarOSTreeConfig.Log.Logger(log.ContextHandler) + if err != nil { + return nil, err + } + ctx = log.SetContextLogger(ctx, logger) + + apiSrv := newAPIService() + router := makeRouter(apiSrv, beskarOSTreeConfig.Profiling) + + return &Plugin{ + ctx: ctx, + config: pluginsrv.Config{ + Router: router, + Gossip: beskarOSTreeConfig.Gossip, + Info: &pluginv1.Info{ + Name: PluginName, + // Not registering media types so that Beskar doesn't send events. + // This plugin as no internal state so events are not needed. + Mediatypes: []string{}, + Version: version.Semver, + Router: &pluginv1.Router{ + Rego: routerRego, + Data: routerData, + }, + }, + }, + }, nil +} + +func (p *Plugin) Start(_ http.RoundTripper, _ *mtls.CAPEM, _ *gossip.BeskarMeta) error { + // Nothing to do here as this plugin has no internal state + // and router is already configured. + return nil +} + +func (p *Plugin) Context() context.Context { + return p.ctx +} + +func (p *Plugin) Config() pluginsrv.Config { + return p.config +} + +func (p *Plugin) RepositoryManager() *repository.Manager { + // this plugin has no internal state so no need for a repository manager + return nil +} + +func makeRouter(apiSrv *apiService, profilingEnabled bool) *chi.Mux { + router := chi.NewRouter() + + // for kubernetes probes + router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + if profilingEnabled { + router.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + router.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + router.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + router.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + router.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + router.Handle("/debug/pprof/{cmd}", http.HandlerFunc(pprof.Index)) // special handling for Gorilla mux + } + + router.Route( + PluginAPIPathPattern, + func(r chi.Router) { + r.Use(pluginsrv.IsTLSMiddleware) + r.Mount("/", apiv1.NewHTTPRouter( + apiSrv, + httpcodec.NewDefaultCodecs(nil), + )) + }, + ) + + return router +} diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go new file mode 100644 index 0000000..33252bf --- /dev/null +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -0,0 +1,49 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + + "github.com/RussellLuo/kun/pkg/httpoption" + "github.com/RussellLuo/validating/v3" + "github.com/go-kit/kit/endpoint" +) + +type MirrorRepositoryRequest struct { + Repository string `json:"repository"` + Depth int `json:"depth"` +} + +// ValidateMirrorRepositoryRequest creates a validator for MirrorRepositoryRequest. +func ValidateMirrorRepositoryRequest(newSchema func(*MirrorRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*MirrorRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type MirrorRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *MirrorRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *MirrorRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfMirrorRepository creates the endpoint for s.MirrorRepository. +func MakeEndpointOfMirrorRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*MirrorRepositoryRequest) + err := s.MirrorRepository( + ctx, + req.Repository, + req.Depth, + ) + return &MirrorRepositoryResponse{ + Err: err, + }, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http.go b/pkg/plugins/ostree/api/v1/http.go new file mode 100644 index 0000000..08d5f91 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/http.go @@ -0,0 +1,58 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + "net/http" + + "github.com/RussellLuo/kun/pkg/httpcodec" + "github.com/RussellLuo/kun/pkg/httpoption" + "github.com/RussellLuo/kun/pkg/oas2" + "github.com/go-chi/chi" + kithttp "github.com/go-kit/kit/transport/http" +) + +func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Option) chi.Router { + r := chi.NewRouter() + options := httpoption.NewOptions(opts...) + + r.Method("GET", "/doc/swagger.yaml", oas2.Handler(OASv2APIDoc, options.ResponseSchema())) + + var codec httpcodec.Codec + var validator httpoption.Validator + var kitOptions []kithttp.ServerOption + + codec = codecs.EncodeDecoder("MirrorRepository") + validator = options.RequestValidator("MirrorRepository") + r.Method( + "POST", "/repository/mirror", + kithttp.NewServer( + MakeEndpointOfMirrorRepository(svc), + decodeMirrorRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + return r +} + +func decodeMirrorRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req MirrorRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go new file mode 100644 index 0000000..bda1384 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -0,0 +1,84 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + "net/http" + "net/url" + "strings" + + "github.com/RussellLuo/kun/pkg/httpcodec" +) + +type HTTPClient struct { + codecs httpcodec.Codecs + httpClient *http.Client + scheme string + host string + pathPrefix string +} + +func NewHTTPClient(codecs httpcodec.Codecs, httpClient *http.Client, baseURL string) (*HTTPClient, error) { + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + return &HTTPClient{ + codecs: codecs, + httpClient: httpClient, + scheme: u.Scheme, + host: u.Host, + pathPrefix: strings.TrimSuffix(u.Path, "/"), + }, nil +} + +func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { + codec := c.codecs.EncodeDecoder("MirrorRepository") + + path := "/repository/mirror" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Depth int `json:"depth"` + }{ + Repository: repository, + Depth: depth, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go new file mode 100644 index 0000000..2a76e7e --- /dev/null +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -0,0 +1,73 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "reflect" + + "github.com/RussellLuo/kun/pkg/oas2" +) + +var ( + base = `swagger: "2.0" +info: + title: "OSTree Repository Management API" + version: "1.0.0" + description: "OSTree is used for managing ostree repositories.\nThis is the API documentation of OSTree.\n//" + license: + name: "MIT" +host: "example.com" +basePath: "/artifacts/ostree/api/v1" +schemes: + - "https" +consumes: + - "application/json" +produces: + - "application/json" +` + + paths = ` +paths: + /repository/mirror: + post: + description: "Mirror a static repository." + operationId: "MirrorRepository" + tags: + - static + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/MirrorRepositoryRequestBody" + %s +` +) + +func getResponses(schema oas2.Schema) []oas2.OASResponses { + return []oas2.OASResponses{ + oas2.GetOASResponses(schema, "MirrorRepository", 200, &MirrorRepositoryResponse{}), + } +} + +func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { + defs := make(map[string]oas2.Definition) + + oas2.AddDefinition(defs, "MirrorRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Depth int `json:"depth"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "MirrorRepository", 200, (&MirrorRepositoryResponse{}).Body()) + + return defs +} + +func OASv2APIDoc(schema oas2.Schema) string { + resps := getResponses(schema) + paths := oas2.GenPaths(resps, paths) + + defs := getDefinitions(schema) + definitions := oas2.GenDefinitions(defs) + + return base + paths + definitions +} From a0e014ef439604567f7a5c758b27ae4f59aefc48 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 8 Dec 2023 18:47:08 -0500 Subject: [PATCH 06/30] gofumpt refactors beskctl to use cobra adds beskarctl ostree sub-command tree adds beskarctl static file sub-command tree documents beskarctl --- cmd/beskar-ostree/main.go | 15 ++- cmd/beskarctl/README.md | 32 +++++++ cmd/beskarctl/ctl/error.go | 13 +++ cmd/beskarctl/ctl/helpers.go | 47 ++++++++++ cmd/beskarctl/ctl/root.go | 31 +++++++ cmd/beskarctl/main.go | 92 +------------------ cmd/beskarctl/ostree/push.go | 73 +++++++++++++++ cmd/beskarctl/ostree/root.go | 23 +++++ cmd/beskarctl/static/push.go | 46 ++++++++++ cmd/beskarctl/static/root.go | 24 +++++ cmd/beskarctl/yum/push.go | 47 ++++++++++ cmd/beskarctl/yum/pushmetadata.go | 55 +++++++++++ cmd/beskarctl/yum/root.go | 34 +++++++ internal/plugins/ostree/api.go | 2 +- internal/plugins/ostree/embedded/router.rego | 4 +- .../ostree/pkg/config/beskar-ostree.go | 15 ++- internal/plugins/ostree/plugin.go | 5 +- internal/plugins/static/api.go | 1 + internal/plugins/yum/api.go | 1 + pkg/orasostree/ostree.go | 65 +++++++++++++ 20 files changed, 520 insertions(+), 105 deletions(-) create mode 100644 cmd/beskarctl/README.md create mode 100644 cmd/beskarctl/ctl/error.go create mode 100644 cmd/beskarctl/ctl/helpers.go create mode 100644 cmd/beskarctl/ctl/root.go create mode 100644 cmd/beskarctl/ostree/push.go create mode 100644 cmd/beskarctl/ostree/root.go create mode 100644 cmd/beskarctl/static/push.go create mode 100644 cmd/beskarctl/static/root.go create mode 100644 cmd/beskarctl/yum/push.go create mode 100644 cmd/beskarctl/yum/pushmetadata.go create mode 100644 cmd/beskarctl/yum/root.go create mode 100644 pkg/orasostree/ostree.go diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go index 1409d2f..4286ca4 100644 --- a/cmd/beskar-ostree/main.go +++ b/cmd/beskar-ostree/main.go @@ -3,15 +3,16 @@ package main import ( "flag" "fmt" + "log" + "net" + "os" + "syscall" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/ostree" "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" - "log" - "net" - "os" - "syscall" ) var configDir string @@ -34,7 +35,11 @@ func serve(beskarOSTreeCmd *flag.FlagSet) error { if err != nil { return err } - defer ln.Close() + defer func() { + if err := ln.Close(); err != nil { + fmt.Println(err) + } + }() plugin, err := ostree.New(ctx, beskarOSTreeConfig) if err != nil { diff --git a/cmd/beskarctl/README.md b/cmd/beskarctl/README.md new file mode 100644 index 0000000..22f3db0 --- /dev/null +++ b/cmd/beskarctl/README.md @@ -0,0 +1,32 @@ +# beskarctl +`beskarctl` is a command line tool for interacting with Beskar Artifact Registries. + +## Installation +``` +go install go.ciq.dev/beskar/cmd/beskarctl@latest +``` + +## Usage +`beskarctl` is very similar to `kubectl` in that it provide various subcommands for interacting with Beskar repositories. +The following subcommands are available: + ``` +beskarctl yum [flags] +beskarctl static [flags] +beskarctl ostree [flags] + ``` +For more information on a specific subcommand, run `beskarctl --help`. + +## Adding a new subcommand +Adding a new subcommand is fairly straightforward. Feel free to use the existing subcommands as a template, e.g., +`cmd/beskarctl/static/`. The following steps should be followed: + +1. Create a new file in `cmd/beskarctl//root.go`. +2. Add a new `cobra.Command` to the `rootCmd` variable in `cmd/beskarctl//root.go`. +3. Add an accessor function to `cmd/beskarctl//root.go` that returns the new `cobra.Command`. +4. Register the new subcommand in `cmd/beskarctl/ctl/root.go` by calling the accessor function. + +### Implementation Notes +- The `cobra.Command` you create should not be exported. Rather, your package should export an accessor function that +returns the `cobra.Command`. The accessor function is your chance to set up any flags or subcommands that your +`cobra.Command` needs. Please avoid the use of init functi +- helper functions are available for common values such as `--repo` and `--registry`. See `cmd/beskarctl/ctl/helpers.go` \ No newline at end of file diff --git a/cmd/beskarctl/ctl/error.go b/cmd/beskarctl/ctl/error.go new file mode 100644 index 0000000..cce9285 --- /dev/null +++ b/cmd/beskarctl/ctl/error.go @@ -0,0 +1,13 @@ +package ctl + +import "fmt" + +type Err string + +func (e Err) Error() string { + return string(e) +} + +func Errf(str string, a ...any) Err { + return Err(fmt.Sprintf(str, a...)) +} diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go new file mode 100644 index 0000000..349ab44 --- /dev/null +++ b/cmd/beskarctl/ctl/helpers.go @@ -0,0 +1,47 @@ +package ctl + +import ( + "github.com/spf13/cobra" + "os" +) + +const ( + ErrMissingFlagRepo = Err("missing repo flag") + ErrMissingFlagRegistry = Err("missing registry flag") +) + +const ( + FlagNameRepo = "repo" + FlagNameRegistry = "registry" +) + +// RegisterFlags registers the flags that are common to all commands. +func RegisterFlags(cmd *cobra.Command) { + // Flags that are common to all commands. + cmd.PersistentFlags().String(FlagNameRepo, "", "The repository to operate on.") + cmd.PersistentFlags().String(FlagNameRegistry, "", "The registry to operate on.") +} + +// Repo returns the repository name from the command line. +// If the repository is not specified, the command will exit with an error. +func Repo() string { + repo, err := rootCmd.Flags().GetString(FlagNameRepo) + if err != nil || repo == "" { + rootCmd.PrintErrln(ErrMissingFlagRepo) + os.Exit(1) + } + + return repo +} + +// Registry returns the registry name from the command line. +// If the registry is not specified, the command will exit with an error. +func Registry() string { + registry, err := rootCmd.Flags().GetString(FlagNameRegistry) + if err != nil || registry == "" { + rootCmd.PrintErrln(ErrMissingFlagRegistry) + os.Exit(1) + } + + return registry +} diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go new file mode 100644 index 0000000..b057c89 --- /dev/null +++ b/cmd/beskarctl/ctl/root.go @@ -0,0 +1,31 @@ +package ctl + +import ( + "fmt" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ostree" + "go.ciq.dev/beskar/cmd/beskarctl/static" + "go.ciq.dev/beskar/cmd/beskarctl/yum" + "os" +) + +var rootCmd = &cobra.Command{ + Use: "beskarctl", + Short: "Operations related to beskar.", +} + +func Execute() { + RegisterFlags(rootCmd) + + rootCmd.AddCommand( + yum.RootCmd(), + static.RootCmd(), + ostree.RootCmd(), + ) + + err := rootCmd.Execute() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/cmd/beskarctl/main.go b/cmd/beskarctl/main.go index 360ad3a..31be082 100644 --- a/cmd/beskarctl/main.go +++ b/cmd/beskarctl/main.go @@ -3,96 +3,8 @@ package main -import ( - "flag" - "fmt" - "os" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "go.ciq.dev/beskar/pkg/oras" - "go.ciq.dev/beskar/pkg/orasrpm" - "go.ciq.dev/beskar/pkg/version" -) - -func fatal(format string, a ...any) { - fmt.Printf(format+"\n", a...) - os.Exit(1) -} +import "go.ciq.dev/beskar/cmd/beskarctl/ctl" func main() { - pushCmd := flag.NewFlagSet("push", flag.ExitOnError) - pushRepo := pushCmd.String("repo", "", "repo") - pushRegistry := pushCmd.String("registry", "", "registry") - - pushMetadataCmd := flag.NewFlagSet("push-metadata", flag.ExitOnError) - pushMetadataRepo := pushMetadataCmd.String("repo", "", "repo") - pushMetadataRegistry := pushMetadataCmd.String("registry", "", "registry") - pushMetadataType := pushMetadataCmd.String("type", "", "type") - - if len(os.Args) == 1 { - fatal("missing subcommand") - } - - switch os.Args[1] { - case "version": - fmt.Println(version.Semver) - case "push": - if err := pushCmd.Parse(os.Args[2:]); err != nil { - fatal("while parsing command arguments: %w", err) - } - rpm := pushCmd.Arg(0) - if rpm == "" { - fatal("an RPM package must be specified") - } else if pushRegistry == nil || *pushRegistry == "" { - fatal("a registry must be specified") - } else if pushRepo == nil || *pushRepo == "" { - fatal("a repo must be specified") - } - if err := push(rpm, *pushRepo, *pushRegistry); err != nil { - fatal("while pushing RPM package: %s", err) - } - case "push-metadata": - if err := pushMetadataCmd.Parse(os.Args[2:]); err != nil { - fatal("while parsing command arguments: %w", err) - } - metadata := pushMetadataCmd.Arg(0) - if metadata == "" { - fatal("a metadata file must be specified") - } else if pushMetadataRegistry == nil || *pushMetadataRegistry == "" { - fatal("a registry must be specified") - } else if pushMetadataRepo == nil || *pushMetadataRepo == "" { - fatal("a repo must be specified") - } else if pushMetadataType == nil || *pushMetadataType == "" { - fatal("a metadata type must be specified") - } - if err := pushMetadata(metadata, *pushMetadataType, *pushMetadataRepo, *pushMetadataRegistry); err != nil { - fatal("while pushing metadata: %s", err) - } - default: - fatal("unknown %q subcommand", os.Args[1]) - } -} - -func push(rpmPath string, repo, registry string) error { - pusher, err := orasrpm.NewRPMPusher(rpmPath, repo, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating RPM pusher: %w", err) - } - - fmt.Printf("Pushing %s to %s\n", rpmPath, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) -} - -func pushMetadata(metadataPath string, dataType, repo, registry string) error { - pusher, err := orasrpm.NewRPMExtraMetadataPusher(metadataPath, repo, dataType, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating RPM metadata pusher: %w", err) - } - - fmt.Printf("Pushing %s to %s\n", metadataPath, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + ctl.Execute() } diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go new file mode 100644 index 0000000..3644649 --- /dev/null +++ b/cmd/beskarctl/ostree/push.go @@ -0,0 +1,73 @@ +package ostree + +import ( + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasostree" + "os" + "path/filepath" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [directory]", + Short: "Push an ostree repository.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + if dir == "" { + return ctl.Err("a directory must be specified") + } + + if err := pushOSTreeRepository(dir, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing ostree repository: %s", err) + } + return nil + }, + } +) + +func PushCmd() *cobra.Command { + return pushCmd +} + +func pushOSTreeRepository(dir, repo, registry string) error { + // Prove that we were given the root directory of an ostree repository + // by checking for the existence of the summary file. + fileInfo, err := os.Stat(filepath.Join(dir, orasostree.KnownFileSummary)) + if err != nil || fileInfo.IsDir() { + return fmt.Errorf("%s file not found in %s", orasostree.KnownFileSummary, dir) + } + + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("while walking %s: %w", path, err) + } + + if info.IsDir() { + return nil + } + + if err := push(path, repo, registry); err != nil { + return fmt.Errorf("while pushing %s: %w", path, err) + } + + return nil + }) +} + +func push(filepath, repo, registry string) error { + pusher, err := orasostree.NewOSTreePusher(filepath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating StaticFile pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/ostree/root.go b/cmd/beskarctl/ostree/root.go new file mode 100644 index 0000000..6c8d70b --- /dev/null +++ b/cmd/beskarctl/ostree/root.go @@ -0,0 +1,23 @@ +package ostree + +import ( + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "ostree", + Aliases: []string{ + "o", + }, + Short: "Operations related to ostree repositories.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + ) + + return rootCmd +} diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go new file mode 100644 index 0000000..7b7115d --- /dev/null +++ b/cmd/beskarctl/static/push.go @@ -0,0 +1,46 @@ +package static + +import ( + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasfile" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [file]", + Short: "Push a file.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + file := args[0] + if file == "" { + return ctl.Err("file must be specified") + } + + if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing static file: %s", err) + } + return nil + }, + } +) + +func PushCmd() *cobra.Command { + return pushCmd +} + +func push(filepath, repo, registry string) error { + pusher, err := orasfile.NewStaticFilePusher(filepath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating StaticFile pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go new file mode 100644 index 0000000..2cc5406 --- /dev/null +++ b/cmd/beskarctl/static/root.go @@ -0,0 +1,24 @@ +package static + +import ( + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "static", + Aliases: []string{ + "file", + "s", + }, + Short: "Operations related to static files.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + ) + + return rootCmd +} diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go new file mode 100644 index 0000000..19fddcc --- /dev/null +++ b/cmd/beskarctl/yum/push.go @@ -0,0 +1,47 @@ +package yum + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasrpm" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [rpm filepath]", + Short: "Push a yum repository to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rpm := args[0] + if rpm == "" { + return ctl.Err("an RPM package must be specified") + } + + if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing RPM package: %s", err) + } + return nil + }, + } +) + +func PushCmd() *cobra.Command { + return pushCmd +} + +func push(rpmPath, repo, registry string) error { + pusher, err := orasrpm.NewRPMPusher(rpmPath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating RPM pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", rpmPath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go new file mode 100644 index 0000000..d8344af --- /dev/null +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -0,0 +1,55 @@ +package yum + +import ( + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasrpm" +) + +// yum push-metadata +var ( + pushMetadataCmd = &cobra.Command{ + Use: "push-metadata [metadata filepath]", + Short: "Push yum repository metadata to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + metadata := args[0] + if metadata == "" { + return ctl.Err("a metadata file must be specified") + } else if registry == "" { + return MissingRequiredFlagRegistry + } else if repo == "" { + return MissingRequiredFlagRepo + } else if pushMetadataType == "" { + return ctl.Err("a metadata type must be specified") + } + + if err := pushMetadata(metadata, pushMetadataType, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing metadata: %s", err) + } + return nil + }, + } + pushMetadataType string +) + +func PushMetadataCmd() *cobra.Command { + pushMetadataCmd.Flags().StringVarP(&pushMetadataType, "type", "t", "", "type") + return pushMetadataCmd +} + +func pushMetadata(metadataPath, dataType, repo, registry string) error { + pusher, err := orasrpm.NewRPMExtraMetadataPusher(metadataPath, repo, dataType, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating RPM metadata pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", metadataPath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/yum/root.go b/cmd/beskarctl/yum/root.go new file mode 100644 index 0000000..71a8912 --- /dev/null +++ b/cmd/beskarctl/yum/root.go @@ -0,0 +1,34 @@ +package yum + +import ( + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" +) + +const ( + MissingRequiredFlagRepo ctl.Err = "a repo must be specified" + MissingRequiredFlagRegistry ctl.Err = "a registry must be specified" +) + +var ( + repo string + registry string + rootCmd = &cobra.Command{ + Use: "yum", + Aliases: []string{ + "y", + "rpm", + "dnf", + }, + Short: "Operations related to yum repositories.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + PushMetadataCmd(), + ) + + return rootCmd +} diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index aca8687..15c8c26 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -3,6 +3,7 @@ package ostree import ( "context" "errors" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" ) @@ -14,6 +15,5 @@ func newAPIService() *apiService { } func (o *apiService) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { - //TODO implement me return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) } diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego index 42d8053..1db8b58 100644 --- a/internal/plugins/ostree/embedded/router.rego +++ b/internal/plugins/ostree/embedded/router.rego @@ -5,7 +5,7 @@ import future.keywords.in default output = {"repository": "", "redirect_url": "", "found": false} -filename_checksum(filename) = checksum if { +makeTag(filename) = checksum if { filename in data.tags checksum := filename } else = checksum { @@ -13,7 +13,7 @@ filename_checksum(filename) = checksum if { } blob_url(repo, filename) = url { - digest := oci.blob_digest(sprintf("%s:%s", [repo, filename_checksum(filename)]), "mediatype", data.mediatype.file) + digest := oci.blob_digest(sprintf("%s:%s", [repo, makeTag(filename)]), "mediatype", data.mediatype.file) url := { "url": sprintf("/v2/%s/blobs/sha256:%s", [repo, digest]), "found": digest != "", diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go index d692cfe..d8743ca 100644 --- a/internal/plugins/ostree/pkg/config/beskar-ostree.go +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -5,15 +5,16 @@ import ( _ "embed" "errors" "fmt" - "github.com/distribution/distribution/v3/configuration" - "go.ciq.dev/beskar/internal/pkg/config" - "go.ciq.dev/beskar/internal/pkg/gossip" - "go.ciq.dev/beskar/internal/pkg/log" "io" "os" "path/filepath" "reflect" "strings" + + "github.com/distribution/distribution/v3/configuration" + "go.ciq.dev/beskar/internal/pkg/config" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" ) const ( @@ -56,7 +57,11 @@ func ParseBeskarOSTreeConfig(dir string) (*BeskarOSTreeConfig, error) { configReader = strings.NewReader(defaultBeskarOSTreeConfig) configDir = "" } else { - defer f.Close() + defer func() { + if err := f.Close(); err != nil { + fmt.Println(err) + } + }() configReader = f } diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go index 8e6fbf1..2f7ed05 100644 --- a/internal/plugins/ostree/plugin.go +++ b/internal/plugins/ostree/plugin.go @@ -3,6 +3,9 @@ package ostree import ( "context" _ "embed" + "net/http" + "net/http/pprof" + "github.com/RussellLuo/kun/pkg/httpcodec" "github.com/go-chi/chi" "go.ciq.dev/beskar/internal/pkg/gossip" @@ -14,8 +17,6 @@ import ( "go.ciq.dev/beskar/pkg/mtls" apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" "go.ciq.dev/beskar/pkg/version" - "net/http" - "net/http/pprof" ) const ( diff --git a/internal/plugins/static/api.go b/internal/plugins/static/api.go index fc81d21..3d6717c 100644 --- a/internal/plugins/static/api.go +++ b/internal/plugins/static/api.go @@ -5,6 +5,7 @@ package static import ( "context" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "github.com/RussellLuo/kun/pkg/werror" diff --git a/internal/plugins/yum/api.go b/internal/plugins/yum/api.go index 627fcc0..572b3b9 100644 --- a/internal/plugins/yum/api.go +++ b/internal/plugins/yum/api.go @@ -5,6 +5,7 @@ package yum import ( "context" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "github.com/RussellLuo/kun/pkg/werror" diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go new file mode 100644 index 0000000..65241f7 --- /dev/null +++ b/pkg/orasostree/ostree.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package orasostree + +import ( + "crypto/md5" //nolint:gosec + "fmt" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "go.ciq.dev/beskar/pkg/oras" +) + +const ( + OSTreeConfigType = "application/vnd.ciq.ostree.file.v1.config+json" + OSTreeLayerType = "application/vnd.ciq.ostree.v1.file" + + KnownFileSummary = "summary" + KnownFileConfig = "config" +) + +func NewOSTreePusher(path, repo string, opts ...name.Option) (oras.Pusher, error) { + if !strings.HasPrefix(repo, "artifacts/") { + if !strings.HasPrefix(repo, "ostree/") { + repo = filepath.Join("static", repo) + } + + repo = filepath.Join("artifacts", repo) + } + + filename := filepath.Base(path) + //nolint:gosec + fileTag := makeTag(filename) + + rawRef := filepath.Join(repo, "files:"+fileTag) + ref, err := name.ParseReference(rawRef, opts...) + if err != nil { + return nil, fmt.Errorf("while parsing reference %s: %w", rawRef, err) + } + + return oras.NewGenericPusher( + ref, + oras.NewManifestConfig(OSTreeConfigType, nil), + oras.NewLocalFileLayer(path, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), + ), nil +} + +// specialTags +var specialTags = []string{ + KnownFileSummary, + KnownFileConfig, +} + +func makeTag(filename string) string { + for _, tag := range specialTags { + if strings.HasPrefix(filename, tag) { + return tag + } + } + + //nolint:gosec + return fmt.Sprintf("%x", md5.Sum([]byte(filename))) +} From 01244234590b294b124ed937f56dd0bba1d8bbce Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 15 Dec 2023 12:40:54 -0500 Subject: [PATCH 07/30] gofumpt fixes kun command typos adds summary.sig as known tag changes regex separator from file to repo refactors beskarctl to use cobra refactors beskarctl subcommands into packages implements beskarctl ostree push fixes ostree router.rego request body issue --- cmd/beskarctl/ctl/helpers.go | 3 +- cmd/beskarctl/ctl/root.go | 12 ++-- cmd/beskarctl/main.go | 13 ++++- cmd/beskarctl/ostree/push.go | 58 ++++++++++++++++---- cmd/beskarctl/ostree/root.go | 16 +++--- cmd/beskarctl/static/push.go | 35 ++++++------ cmd/beskarctl/static/root.go | 18 +++--- cmd/beskarctl/yum/push.go | 34 ++++++------ cmd/beskarctl/yum/pushmetadata.go | 1 + internal/plugins/ostree/README.md | 20 +++++++ internal/plugins/ostree/embedded/data.json | 3 +- internal/plugins/ostree/embedded/router.rego | 4 +- pkg/orasostree/ostree.go | 42 +++++++++----- pkg/plugins/ostree/api/v1/api.go | 4 +- pkg/plugins/ostree/api/v1/oas2.go | 4 +- 15 files changed, 170 insertions(+), 97 deletions(-) create mode 100644 internal/plugins/ostree/README.md diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go index 349ab44..7a34a53 100644 --- a/cmd/beskarctl/ctl/helpers.go +++ b/cmd/beskarctl/ctl/helpers.go @@ -1,8 +1,9 @@ package ctl import ( - "github.com/spf13/cobra" "os" + + "github.com/spf13/cobra" ) const ( diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go index b057c89..a44338d 100644 --- a/cmd/beskarctl/ctl/root.go +++ b/cmd/beskarctl/ctl/root.go @@ -2,11 +2,9 @@ package ctl import ( "fmt" - "github.com/spf13/cobra" - "go.ciq.dev/beskar/cmd/beskarctl/ostree" - "go.ciq.dev/beskar/cmd/beskarctl/static" - "go.ciq.dev/beskar/cmd/beskarctl/yum" "os" + + "github.com/spf13/cobra" ) var rootCmd = &cobra.Command{ @@ -14,13 +12,11 @@ var rootCmd = &cobra.Command{ Short: "Operations related to beskar.", } -func Execute() { +func Execute(cmds ...*cobra.Command) { RegisterFlags(rootCmd) rootCmd.AddCommand( - yum.RootCmd(), - static.RootCmd(), - ostree.RootCmd(), + cmds..., ) err := rootCmd.Execute() diff --git a/cmd/beskarctl/main.go b/cmd/beskarctl/main.go index 31be082..943cb8c 100644 --- a/cmd/beskarctl/main.go +++ b/cmd/beskarctl/main.go @@ -3,8 +3,17 @@ package main -import "go.ciq.dev/beskar/cmd/beskarctl/ctl" +import ( + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/cmd/beskarctl/ostree" + "go.ciq.dev/beskar/cmd/beskarctl/static" + "go.ciq.dev/beskar/cmd/beskarctl/yum" +) func main() { - ctl.Execute() + ctl.Execute( + yum.RootCmd(), + static.RootCmd(), + ostree.RootCmd(), + ) } diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go index 3644649..9edd759 100644 --- a/cmd/beskarctl/ostree/push.go +++ b/cmd/beskarctl/ostree/push.go @@ -1,7 +1,12 @@ package ostree import ( + "context" "fmt" + "os" + "path/filepath" + "strings" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -9,8 +14,7 @@ import ( "go.ciq.dev/beskar/cmd/beskarctl/ctl" "go.ciq.dev/beskar/pkg/oras" "go.ciq.dev/beskar/pkg/orasostree" - "os" - "path/filepath" + "golang.org/x/sync/errgroup" ) var ( @@ -30,12 +34,18 @@ var ( return nil }, } + jobCount int ) func PushCmd() *cobra.Command { + pushCmd.PersistentFlags().IntVarP(&jobCount, "jobs", "j", 10, "The repository to operate on.") return pushCmd } +// pushOSTreeRepository walks a local ostree repository and pushes each file to the given registry. +// dir is the root directory of the ostree repository, i.e., the directory containing the summary file. +// repo is the name of the ostree repository. +// registry is the registry to push to. func pushOSTreeRepository(dir, repo, registry string) error { // Prove that we were given the root directory of an ostree repository // by checking for the existence of the summary file. @@ -44,30 +54,58 @@ func pushOSTreeRepository(dir, repo, registry string) error { return fmt.Errorf("%s file not found in %s", orasostree.KnownFileSummary, dir) } - return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // Create a worker pool to push each file in the repository concurrently. + // ctx will be cancelled on error, and the error will be returned. + eg, ctx := errgroup.WithContext(context.Background()) + eg.SetLimit(jobCount) + + // Walk the directory tree, skipping directories and pushing each file. + if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + // If there was an error with the file, return it. if err != nil { return fmt.Errorf("while walking %s: %w", path, err) } - if info.IsDir() { + // Skip directories. + if d.IsDir() { return nil } - if err := push(path, repo, registry); err != nil { - return fmt.Errorf("while pushing %s: %w", path, err) + if ctx.Err() != nil { + // Skip remaining files because our context has been cancelled. + // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). + // This is because we would never be able to handle an error returned from the last job. + return filepath.SkipAll } + eg.Go(func() error { + if err := push(dir, path, repo, registry); err != nil { + return fmt.Errorf("while pushing %s: %w", path, err) + } + return nil + }) + return nil - }) + }); err != nil { + // We should only receive here if filepath.WalkDir() returns an error. + // Push errors are handled below. + return fmt.Errorf("while walking %s: %w", dir, err) + } + + // Wait for all workers to finish. + // If any worker returns an error, eg.Wait() will return that error. + return eg.Wait() } -func push(filepath, repo, registry string) error { - pusher, err := orasostree.NewOSTreePusher(filepath, repo, name.WithDefaultRegistry(registry)) +func push(repoRootDir, path, repo, registry string) error { + pusher, err := orasostree.NewOSTreePusher(repoRootDir, path, repo, name.WithDefaultRegistry(registry)) if err != nil { return fmt.Errorf("while creating StaticFile pusher: %w", err) } - fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + path = strings.TrimPrefix(path, repoRootDir) + path = strings.TrimPrefix(path, "/") + fmt.Printf("Pushing %s to %s\n", path, pusher.Reference()) return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) } diff --git a/cmd/beskarctl/ostree/root.go b/cmd/beskarctl/ostree/root.go index 6c8d70b..570c1cf 100644 --- a/cmd/beskarctl/ostree/root.go +++ b/cmd/beskarctl/ostree/root.go @@ -4,15 +4,13 @@ import ( "github.com/spf13/cobra" ) -var ( - rootCmd = &cobra.Command{ - Use: "ostree", - Aliases: []string{ - "o", - }, - Short: "Operations related to ostree repositories.", - } -) +var rootCmd = &cobra.Command{ + Use: "ostree", + Aliases: []string{ + "o", + }, + Short: "Operations related to ostree repositories.", +} func RootCmd() *cobra.Command { rootCmd.AddCommand( diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go index 7b7115d..4848f76 100644 --- a/cmd/beskarctl/static/push.go +++ b/cmd/beskarctl/static/push.go @@ -2,6 +2,7 @@ package static import ( "fmt" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -11,24 +12,22 @@ import ( "go.ciq.dev/beskar/pkg/orasfile" ) -var ( - pushCmd = &cobra.Command{ - Use: "push [file]", - Short: "Push a file.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - file := args[0] - if file == "" { - return ctl.Err("file must be specified") - } - - if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { - return ctl.Errf("while pushing static file: %s", err) - } - return nil - }, - } -) +var pushCmd = &cobra.Command{ + Use: "push [file]", + Short: "Push a file.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + file := args[0] + if file == "" { + return ctl.Err("file must be specified") + } + + if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing static file: %s", err) + } + return nil + }, +} func PushCmd() *cobra.Command { return pushCmd diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go index 2cc5406..8eb377f 100644 --- a/cmd/beskarctl/static/root.go +++ b/cmd/beskarctl/static/root.go @@ -4,16 +4,14 @@ import ( "github.com/spf13/cobra" ) -var ( - rootCmd = &cobra.Command{ - Use: "static", - Aliases: []string{ - "file", - "s", - }, - Short: "Operations related to static files.", - } -) +var rootCmd = &cobra.Command{ + Use: "static", + Aliases: []string{ + "file", + "s", + }, + Short: "Operations related to static files.", +} func RootCmd() *cobra.Command { rootCmd.AddCommand( diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go index 19fddcc..d211154 100644 --- a/cmd/beskarctl/yum/push.go +++ b/cmd/beskarctl/yum/push.go @@ -12,24 +12,22 @@ import ( "go.ciq.dev/beskar/pkg/orasrpm" ) -var ( - pushCmd = &cobra.Command{ - Use: "push [rpm filepath]", - Short: "Push a yum repository to a registry.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - rpm := args[0] - if rpm == "" { - return ctl.Err("an RPM package must be specified") - } - - if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { - return ctl.Errf("while pushing RPM package: %s", err) - } - return nil - }, - } -) +var pushCmd = &cobra.Command{ + Use: "push [rpm filepath]", + Short: "Push a yum repository to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rpm := args[0] + if rpm == "" { + return ctl.Err("an RPM package must be specified") + } + + if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing RPM package: %s", err) + } + return nil + }, +} func PushCmd() *cobra.Command { return pushCmd diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go index d8344af..043dbdc 100644 --- a/cmd/beskarctl/yum/pushmetadata.go +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -2,6 +2,7 @@ package yum import ( "fmt" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" diff --git a/internal/plugins/ostree/README.md b/internal/plugins/ostree/README.md new file mode 100644 index 0000000..8ed1e84 --- /dev/null +++ b/internal/plugins/ostree/README.md @@ -0,0 +1,20 @@ +# OSTree Plugin + +## Overview +The ostree plugin is responsible for mapping the ostree repository to the OCI registry. This is done in the router.rego +and no routing execution happens within the plugin itself at runtime. The plugin does, however, provide an API for mirroring +ostree repositories into beskar. + +## File Tagging +The ostree plugin maps the ostree repository filepaths to the OCI registry tags. Most files are simply mapped by hashing +the full filepath relative to the ostree root. For example, `objects/ab/abcd1234.filez` becomes `file:b8458bd029a97ca5e03f272a6b7bd0d1`. +There are a few exceptions to this rule, however. The following files are considered "special" and are tagged as follows: +1. `summary` -> `file:summary` +2. `summary.sig` -> `file:summary.sig` +3. `config` -> `file:config` + +There is no technical reason for this and is only done to make the mapping more human-readable in the case of "special" +files. + +## Mirroring +TBD \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/data.json b/internal/plugins/ostree/embedded/data.json index 536198d..337cbc3 100644 --- a/internal/plugins/ostree/embedded/data.json +++ b/internal/plugins/ostree/embedded/data.json @@ -1,7 +1,7 @@ { "routes": [ { - "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/file/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", + "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/repo/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", "methods": [ "GET", "HEAD" @@ -22,6 +22,7 @@ }, "tags": [ "summary", + "summary.sig", "config" ] } \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego index 1db8b58..08f365e 100644 --- a/internal/plugins/ostree/embedded/router.rego +++ b/internal/plugins/ostree/embedded/router.rego @@ -29,7 +29,7 @@ output = obj { 1 )[0] redirect := blob_url( - sprintf("%s/files", [match[1]]), + sprintf("%s/file", [match[1]]), match[2], ) obj := { @@ -44,7 +44,7 @@ output = obj { input.path, 1 )[0] - repo := object.get({}, data.routes[index].body_key, "") + repo := object.get(request.body(), data.routes[index].body_key, "") obj := { "repository": repo, "redirect_url": "", diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go index 65241f7..0a86011 100644 --- a/pkg/orasostree/ostree.go +++ b/pkg/orasostree/ostree.go @@ -14,48 +14,62 @@ import ( ) const ( + ArtifactsPathPrefix = "artifacts" + OSTreePathPrefix = "ostree" + OSTreeConfigType = "application/vnd.ciq.ostree.file.v1.config+json" OSTreeLayerType = "application/vnd.ciq.ostree.v1.file" - KnownFileSummary = "summary" - KnownFileConfig = "config" + KnownFileSummary = "summary" + KnownFileSummarySig = "summary.sig" + KnownFileConfig = "config" ) -func NewOSTreePusher(path, repo string, opts ...name.Option) (oras.Pusher, error) { - if !strings.HasPrefix(repo, "artifacts/") { - if !strings.HasPrefix(repo, "ostree/") { - repo = filepath.Join("static", repo) +func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras.Pusher, error) { + if !strings.HasPrefix(repo, ArtifactsPathPrefix+"/") { + if !strings.HasPrefix(repo, OSTreePathPrefix+"/") { + repo = filepath.Join(OSTreePathPrefix, repo) } - repo = filepath.Join("artifacts", repo) + repo = filepath.Join(ArtifactsPathPrefix, repo) } - filename := filepath.Base(path) - //nolint:gosec - fileTag := makeTag(filename) + // Sanitize the path to match the format of the tag. See internal/plugins/ostree/embedded/data.json. + // In this case the file path needs to be relative to the repository root and not contain a leading slash. + path = strings.TrimPrefix(path, repoRootDir) + path = strings.TrimPrefix(path, "/") - rawRef := filepath.Join(repo, "files:"+fileTag) + fileTag := makeTag(path) + rawRef := filepath.Join(repo, "file:"+fileTag) ref, err := name.ParseReference(rawRef, opts...) if err != nil { return nil, fmt.Errorf("while parsing reference %s: %w", rawRef, err) } + absolutePath := filepath.Join(repoRootDir, path) + return oras.NewGenericPusher( ref, oras.NewManifestConfig(OSTreeConfigType, nil), - oras.NewLocalFileLayer(path, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), + oras.NewLocalFileLayer(absolutePath, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), ), nil } -// specialTags +// specialTags are tags that are not md5 hashes of the filename. +// These files are meant to stand out in the registry. +// Note: Values are not limited to the repo's root directory, but at the moment on the following have been identified. var specialTags = []string{ KnownFileSummary, + KnownFileSummarySig, KnownFileConfig, } +// makeTag creates a tag for a file. +// If the filename starts with a special tag, the tag is returned as-is. +// Otherwise, the tag is the md5 hash of the filename. func makeTag(filename string) string { for _, tag := range specialTags { - if strings.HasPrefix(filename, tag) { + if filename == tag { return tag } } diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 9316106..8d9296c 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -31,9 +31,9 @@ type Page struct { //kun:oas version=1.0.0 //kun:oas basePath=/artifacts/ostree/api/v1 //kun:oas docsPath=/doc/swagger.yaml -//kun:oas tags=static +//kun:oas tags=ostree type OSTree interface { - // Mirror a static repository. + // Mirror an ostree repository. //kun:op POST /repository/mirror //kun:success statusCode=200 MirrorRepository(ctx context.Context, repository string, depth int) (err error) diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go index 2a76e7e..b3b5f4b 100644 --- a/pkg/plugins/ostree/api/v1/oas2.go +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -31,10 +31,10 @@ produces: paths: /repository/mirror: post: - description: "Mirror a static repository." + description: "Mirror an ostree repository." operationId: "MirrorRepository" tags: - - static + - ostree parameters: - name: body in: body From f34ec0cf7e4640ec22b8e56f1258b3f37b1270aa Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 16 Nov 2023 10:51:07 -0500 Subject: [PATCH 08/30] adds plugins.md as starting point for plugin how-to documentation links README.md to plugins.md --- README.md | 2 +- docs/plugins.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 docs/plugins.md diff --git a/README.md b/README.md index df2c668..0b8722b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ It's designed to support various artifacts and expose them through dedicated plu ### Features -* Modular/Extensible via plugins +* Modular/Extensible via [plugins](docs/plugins.md) * Support for YUM repositories (beskar-yum) * Support for static file repositories (beskar-static) diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 0000000..18c9952 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,17 @@ +# Beskar Plugins + +TODOS: +- [ ] Generate with Kun - See build/mage/build.go:86 for autogeneration +- [ ] Map artifact paths with a separator like `/artifacts/ostree/{repo_name}/separtor/path/to/{artifact_name}`. This will be translated to `/v2/%s/blobs/sha256:%s` +- [ ] Create router.rego & data.json so that Beskar knows how to route requests to plugin server(s) +- [ ] mediatypes may be needed for each file type + - [ ] `application/vnd.ciq.ostree.file.v1.file` + - [ ] `application/vnd.ciq.ostree.summary.v1.summary` + + + +See internal/plugins/yum/embedded/router.rego for example +/artifacts/ostree/{repo_name}/separtor/path/to/{artifact_name} + +/2/artifacts/ostree/{repo_name}/files:summary +/2/artifacts/ostree/{repo_name}/files:{sha256("/path/to/{artifact_name}")} \ No newline at end of file From 052611db378e31e314db3d5798db0ad6c339c075 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 22:58:16 -0500 Subject: [PATCH 09/30] adds IsTLSMiddleware for convenience refactors repository.Manager to non-generic form to allow nil without needing to satisfy generic constraints refactors code depending on generic repository.Manager to new non-generic form various comments --- internal/pkg/beskar/plugin.go | 12 +++ internal/pkg/pluginsrv/webhandler.go | 23 ++++- internal/pkg/repository/handler.go | 13 +++ internal/pkg/repository/manager.go | 29 +++--- internal/plugins/static/api.go | 52 ++++++++-- .../static/pkg/staticrepository/handler.go | 2 +- internal/plugins/static/plugin.go | 19 +--- internal/plugins/yum/api.go | 94 ++++++++++++++++--- .../plugins/yum/pkg/yumrepository/handler.go | 2 +- internal/plugins/yum/plugin.go | 19 +--- 10 files changed, 199 insertions(+), 66 deletions(-) diff --git a/internal/pkg/beskar/plugin.go b/internal/pkg/beskar/plugin.go index 70a8793..89bc066 100644 --- a/internal/pkg/beskar/plugin.go +++ b/internal/pkg/beskar/plugin.go @@ -97,12 +97,16 @@ func (pm *pluginManager) setClientTLSConfig(tlsConfig *tls.Config) { } func (pm *pluginManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // We expect the request to be of the form /artifacts/{plugin_name}/... + // If it is not, we return a 404. matches := artifactsMatch.FindStringSubmatch(r.URL.Path) if len(matches) < 2 { w.WriteHeader(http.StatusNotFound) return } + // Check if the plugin is registered with a name matching the second path component. + // If it is not, we return a 404. pm.pluginsMutex.RLock() pl := pm.plugins[matches[1]] pm.pluginsMutex.RUnlock() @@ -112,6 +116,7 @@ func (pm *pluginManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // Forward the request to the plugin. (The in memory representation of the plugin not the plugin application itself) pl.ServeHTTP(w, r) } @@ -266,6 +271,9 @@ type plugin struct { func (p *plugin) ServeHTTP(w http.ResponseWriter, r *http.Request) { key := r.RemoteAddr + // If the request is for a repository, we need to check if the router has a decision for it. + // If it does, we need to redirect the request to the appropriate location. + // If it does not, we need to use the node hash to find the appropriate node to forward the request to. result, err := p.router.Load().Decision(r, p.registry) if err != nil { dcontext.GetLogger(r.Context()).Errorf("%s router decision error: %s", p.name, err) @@ -403,3 +411,7 @@ func loadPlugins(ctx context.Context) (func(), error) { return wg.Wait, nil } + +// Mountain Team - Lead Developer +// Fuzzball Team - +// Innovation Group - Tech Ambassador () diff --git a/internal/pkg/pluginsrv/webhandler.go b/internal/pkg/pluginsrv/webhandler.go index cea5fa8..06232e0 100644 --- a/internal/pkg/pluginsrv/webhandler.go +++ b/internal/pkg/pluginsrv/webhandler.go @@ -16,9 +16,9 @@ import ( "google.golang.org/protobuf/proto" ) -type webHandler[H repository.Handler] struct { +type webHandler struct { pluginInfo *pluginv1.Info - manager *repository.Manager[H] + manager *repository.Manager } func IsTLS(w http.ResponseWriter, r *http.Request) bool { @@ -29,7 +29,22 @@ func IsTLS(w http.ResponseWriter, r *http.Request) bool { return true } -func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { +// IsTLSMiddleware is a middleware that checks if the request is TLS. This is a convenience wrapper around IsTLS. +func IsTLSMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !IsTLS(w, r) { + return + } + next.ServeHTTP(w, r) + }) +} + +func (wh *webHandler) event(w http.ResponseWriter, r *http.Request) { + if wh.manager == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + if !IsTLS(w, r) { return } @@ -84,7 +99,7 @@ func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { } } -func (wh *webHandler[H]) info(w http.ResponseWriter, r *http.Request) { +func (wh *webHandler) info(w http.ResponseWriter, r *http.Request) { if !IsTLS(w, r) { return } diff --git a/internal/pkg/repository/handler.go b/internal/pkg/repository/handler.go index 84fb100..ff9b4c9 100644 --- a/internal/pkg/repository/handler.go +++ b/internal/pkg/repository/handler.go @@ -31,13 +31,26 @@ func (hp HandlerParams) Remove(repository string) { hp.remove(repository) } +// Handler - Interface for handling events for a repository. type Handler interface { + // QueueEvent - Called when a new event is received. If store is true, the event should be stored in the database. + // Note: Avoid performing any long-running operations in this function. QueueEvent(event *eventv1.EventPayload, store bool) error + + // Started - Returns true if the handler has started. Started() bool + + // Start - Called when the handler should start processing events. + // This is your chance to set up any resources, e.g., database connections, run loops, etc. + // This will only be called once. Start(context.Context) + + // Stop - Called when the handler should stop processing events and clean up resources. Stop() } +// RepoHandler - A partial default implementation of the Handler interface that provides some common functionality. +// You can embed this in your own handler to get some default functionality, e.g., an event queue. type RepoHandler struct { Repository string Params *HandlerParams diff --git a/internal/pkg/repository/manager.go b/internal/pkg/repository/manager.go index f726aa2..724cdf7 100644 --- a/internal/pkg/repository/manager.go +++ b/internal/pkg/repository/manager.go @@ -11,20 +11,21 @@ import ( "go.ciq.dev/beskar/internal/pkg/log" ) -type Manager[H Handler] struct { +type HandlerMap = map[string]Handler + +type HandlerFactory = func(*slog.Logger, *RepoHandler) Handler + +type Manager struct { repositoryMutex sync.RWMutex - repositories map[string]H + repositories HandlerMap repositoryParams *HandlerParams - newHandler func(*slog.Logger, *RepoHandler) H + newHandler func(*slog.Logger, *RepoHandler) Handler } -func NewManager[H Handler]( - params *HandlerParams, - newHandler func(*slog.Logger, *RepoHandler) H, -) *Manager[H] { - m := &Manager[H]{ - repositories: make(map[string]H), +func NewManager(params *HandlerParams, newHandler HandlerFactory) *Manager { + m := &Manager{ + repositories: make(HandlerMap), repositoryParams: params, newHandler: newHandler, } @@ -33,13 +34,13 @@ func NewManager[H Handler]( return m } -func (m *Manager[H]) remove(repository string) { +func (m *Manager) remove(repository string) { m.repositoryMutex.Lock() delete(m.repositories, repository) m.repositoryMutex.Unlock() } -func (m *Manager[H]) Get(ctx context.Context, repository string) H { +func (m *Manager) Get(ctx context.Context, repository string) Handler { m.repositoryMutex.Lock() r, ok := m.repositories[repository] @@ -73,7 +74,7 @@ func (m *Manager[H]) Get(ctx context.Context, repository string) H { return rh } -func (m *Manager[H]) Has(repository string) bool { +func (m *Manager) Has(repository string) bool { m.repositoryMutex.RLock() _, ok := m.repositories[repository] m.repositoryMutex.RUnlock() @@ -81,10 +82,10 @@ func (m *Manager[H]) Has(repository string) bool { return ok } -func (m *Manager[H]) GetAll() map[string]H { +func (m *Manager) GetAll() HandlerMap { m.repositoryMutex.RLock() - handlers := make(map[string]H) + handlers := make(HandlerMap) for name, handler := range m.repositories { handlers[name] = handler } diff --git a/internal/plugins/static/api.go b/internal/plugins/static/api.go index ce0cb7a..83e9e16 100644 --- a/internal/plugins/static/api.go +++ b/internal/plugins/static/api.go @@ -5,6 +5,7 @@ package static import ( "context" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" @@ -18,44 +19,83 @@ func checkRepository(repository string) error { return nil } +func (p *Plugin) getHandlerForRepository(ctx context.Context, repository string) (*staticrepository.Handler, error) { + h, ok := p.repositoryManager.Get(ctx, repository).(*staticrepository.Handler) + if !ok { + return nil, werror.Wrapf(gcode.ErrNotFound, "repository %q does not exist in the required form", repository) + } + + return h, nil +} + func (p *Plugin) DeleteRepository(ctx context.Context, repository string, deleteFiles bool) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx, deleteFiles) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.DeleteRepository(ctx, deleteFiles) } func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryLogs(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryLogs(ctx, page) } func (p *Plugin) RemoveRepositoryFile(ctx context.Context, repository string, tag string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).RemoveRepositoryFile(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.RemoveRepositoryFile(ctx, tag) } func (p *Plugin) GetRepositoryFileByTag(ctx context.Context, repository string, tag string) (repositoryFile *apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryFileByTag(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryFileByTag(ctx, tag) } func (p *Plugin) GetRepositoryFileByName(ctx context.Context, repository string, name string) (repositoryFile *apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryFileByName(ctx, name) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryFileByName(ctx, name) } func (p *Plugin) ListRepositoryFiles(ctx context.Context, repository string, page *apiv1.Page) (repositoryFiles []*apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryFiles(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryFiles(ctx, page) } diff --git a/internal/plugins/static/pkg/staticrepository/handler.go b/internal/plugins/static/pkg/staticrepository/handler.go index 2e04eab..1f09859 100644 --- a/internal/plugins/static/pkg/staticrepository/handler.go +++ b/internal/plugins/static/pkg/staticrepository/handler.go @@ -35,7 +35,7 @@ type Handler struct { delete atomic.Bool } -func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) repository.Handler { return &Handler{ RepoHandler: repoHandler, repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), diff --git a/internal/plugins/static/plugin.go b/internal/plugins/static/plugin.go index 4aab58a..ded40d8 100644 --- a/internal/plugins/static/plugin.go +++ b/internal/plugins/static/plugin.go @@ -41,11 +41,11 @@ type Plugin struct { ctx context.Context config pluginsrv.Config - repositoryManager *repository.Manager[*staticrepository.Handler] + repositoryManager *repository.Manager handlerParams *repository.HandlerParams } -var _ pluginsrv.Service[*staticrepository.Handler] = &Plugin{} +var _ pluginsrv.Service = &Plugin{} func New(ctx context.Context, beskarStaticConfig *config.BeskarStaticConfig) (*Plugin, error) { logger, err := beskarStaticConfig.Log.Logger(log.ContextHandler) @@ -65,7 +65,7 @@ func New(ctx context.Context, beskarStaticConfig *config.BeskarStaticConfig) (*P Dir: filepath.Join(beskarStaticConfig.DataDir, "_repohandlers_"), }, } - plugin.repositoryManager = repository.NewManager[*staticrepository.Handler]( + plugin.repositoryManager = repository.NewManager( plugin.handlerParams, staticrepository.NewHandler, ) @@ -124,7 +124,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/static/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -143,15 +143,6 @@ func (p *Plugin) Context() context.Context { return p.ctx } -func (p *Plugin) RepositoryManager() *repository.Manager[*staticrepository.Handler] { +func (p *Plugin) RepositoryManager() *repository.Manager { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} diff --git a/internal/plugins/yum/api.go b/internal/plugins/yum/api.go index 2fe8534..59f09b3 100644 --- a/internal/plugins/yum/api.go +++ b/internal/plugins/yum/api.go @@ -5,6 +5,7 @@ package yum import ( "context" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" @@ -18,86 +19,155 @@ func checkRepository(repository string) error { return nil } +func (p *Plugin) getHandlerForRepository(ctx context.Context, repository string) (*yumrepository.Handler, error) { + h, ok := p.repositoryManager.Get(ctx, repository).(*yumrepository.Handler) + if !ok { + return nil, werror.Wrapf(gcode.ErrNotFound, "repository %q does not exist in the required form", repository) + } + + return h, nil +} + func (p *Plugin) CreateRepository(ctx context.Context, repository string, properties *apiv1.RepositoryProperties) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).CreateRepository(ctx, properties) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.CreateRepository(ctx, properties) } func (p *Plugin) DeleteRepository(ctx context.Context, repository string, deletePackages bool) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx, deletePackages) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.DeleteRepository(ctx, deletePackages) } func (p *Plugin) UpdateRepository(ctx context.Context, repository string, properties *apiv1.RepositoryProperties) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).UpdateRepository(ctx, properties) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.UpdateRepository(ctx, properties) } func (p *Plugin) GetRepository(ctx context.Context, repository string) (properties *apiv1.RepositoryProperties, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepository(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepository(ctx) } func (p *Plugin) SyncRepository(ctx context.Context, repository string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.SyncRepository(ctx) } func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositorySyncStatus(ctx) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositorySyncStatus(ctx) } func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryLogs(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryLogs(ctx, page) } func (p *Plugin) RemoveRepositoryPackage(ctx context.Context, repository string, id string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).RemoveRepositoryPackage(ctx, id) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.RemoveRepositoryPackage(ctx, id) } func (p *Plugin) RemoveRepositoryPackageByTag(ctx context.Context, repository string, tag string) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).RemoveRepositoryPackageByTag(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return err + } + + return h.RemoveRepositoryPackageByTag(ctx, tag) } func (p *Plugin) GetRepositoryPackage(ctx context.Context, repository string, id string) (repositoryPackage *apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryPackage(ctx, id) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryPackage(ctx, id) } func (p *Plugin) GetRepositoryPackageByTag(ctx context.Context, repository string, tag string) (repositoryPackage *apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).GetRepositoryPackageByTag(ctx, tag) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.GetRepositoryPackageByTag(ctx, tag) } func (p *Plugin) ListRepositoryPackages(ctx context.Context, repository string, page *apiv1.Page) (repositoryPackages []*apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - return p.repositoryManager.Get(ctx, repository).ListRepositoryPackages(ctx, page) + h, err := p.getHandlerForRepository(ctx, repository) + if err != nil { + return nil, err + } + + return h.ListRepositoryPackages(ctx, page) } diff --git a/internal/plugins/yum/pkg/yumrepository/handler.go b/internal/plugins/yum/pkg/yumrepository/handler.go index 5f0d728..9fea298 100644 --- a/internal/plugins/yum/pkg/yumrepository/handler.go +++ b/internal/plugins/yum/pkg/yumrepository/handler.go @@ -52,7 +52,7 @@ type Handler struct { delete atomic.Bool } -func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) repository.Handler { return &Handler{ RepoHandler: repoHandler, repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), diff --git a/internal/plugins/yum/plugin.go b/internal/plugins/yum/plugin.go index b5953e0..cb205a8 100644 --- a/internal/plugins/yum/plugin.go +++ b/internal/plugins/yum/plugin.go @@ -41,11 +41,11 @@ type Plugin struct { ctx context.Context config pluginsrv.Config - repositoryManager *repository.Manager[*yumrepository.Handler] + repositoryManager *repository.Manager handlerParams *repository.HandlerParams } -var _ pluginsrv.Service[*yumrepository.Handler] = &Plugin{} +var _ pluginsrv.Service = &Plugin{} func New(ctx context.Context, beskarYumConfig *config.BeskarYumConfig) (*Plugin, error) { logger, err := beskarYumConfig.Log.Logger(log.ContextHandler) @@ -65,7 +65,7 @@ func New(ctx context.Context, beskarYumConfig *config.BeskarYumConfig) (*Plugin, Dir: filepath.Join(beskarYumConfig.DataDir, "_repohandlers_"), }, } - plugin.repositoryManager = repository.NewManager[*yumrepository.Handler]( + plugin.repositoryManager = repository.NewManager( plugin.handlerParams, yumrepository.NewHandler, ) @@ -127,7 +127,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/yum/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -146,15 +146,6 @@ func (p *Plugin) Context() context.Context { return p.ctx } -func (p *Plugin) RepositoryManager() *repository.Manager[*yumrepository.Handler] { +func (p *Plugin) RepositoryManager() *repository.Manager { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} From 46978c50c426e0ae0a7d3e806ff74ea0a00b97f7 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 22:59:51 -0500 Subject: [PATCH 10/30] ignore intellij project state files --- .gitignore | 2 ++ internal/plugins/ostree/api.go | 19 +++++++++++++++ pkg/plugins/ostree/api/v1/api.go | 40 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 internal/plugins/ostree/api.go create mode 100644 pkg/plugins/ostree/api/v1/api.go diff --git a/.gitignore b/.gitignore index 1f65aa4..d705dbd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ build/output .envrc.local vendor go.work.sum + +.idea diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go new file mode 100644 index 0000000..aca8687 --- /dev/null +++ b/internal/plugins/ostree/api.go @@ -0,0 +1,19 @@ +package ostree + +import ( + "context" + "errors" + "github.com/RussellLuo/kun/pkg/werror" + "github.com/RussellLuo/kun/pkg/werror/gcode" +) + +type apiService struct{} + +func newAPIService() *apiService { + return &apiService{} +} + +func (o *apiService) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { + //TODO implement me + return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) +} diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go new file mode 100644 index 0000000..9316106 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/api.go @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package apiv1 + +import ( + "context" + "regexp" +) + +const ( + RepositoryRegex = "^(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)$" + URLPath = "/artifacts/ostree/api/v1" +) + +var repositoryMatcher = regexp.MustCompile(RepositoryRegex) + +func RepositoryMatch(repository string) bool { + return repositoryMatcher.MatchString(repository) +} + +type Page struct { + Size int + Token string +} + +// OSTree is used for managing ostree repositories. +// This is the API documentation of OSTree. +// +//kun:oas title=OSTree Repository Management API +//kun:oas version=1.0.0 +//kun:oas basePath=/artifacts/ostree/api/v1 +//kun:oas docsPath=/doc/swagger.yaml +//kun:oas tags=static +type OSTree interface { + // Mirror a static repository. + //kun:op POST /repository/mirror + //kun:success statusCode=200 + MirrorRepository(ctx context.Context, repository string, depth int) (err error) +} From 4375255d67bd593865e8d9fe7fe24bd140f00ea0 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 23:01:44 -0500 Subject: [PATCH 11/30] refactors service to a non-generic form --- internal/pkg/pluginsrv/service.go | 48 +++++++++++++++++++------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/internal/pkg/pluginsrv/service.go b/internal/pkg/pluginsrv/service.go index 8c72615..59e81e1 100644 --- a/internal/pkg/pluginsrv/service.go +++ b/internal/pkg/pluginsrv/service.go @@ -34,14 +34,22 @@ type Config struct { Info *pluginv1.Info } -type Service[H repository.Handler] interface { +type Service interface { + // Start starts the service's HTTP server. Start(http.RoundTripper, *mtls.CAPEM, *gossip.BeskarMeta) error + + // Context returns the service's context. Context() context.Context + + // Config returns the service's configuration. Config() Config - RepositoryManager() *repository.Manager[H] + + // RepositoryManager returns the service's repository manager. + // For plugin's without a repository manager, this method should return nil. + RepositoryManager() *repository.Manager } -func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn error) { +func Serve(ln net.Listener, service Service) (errFn error) { ctx := service.Context() errCh := make(chan error) @@ -93,6 +101,23 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err } repoManager := service.RepositoryManager() + if repoManager != nil { + // Gracefully shutdown repository handlers + defer func() { + var wg sync.WaitGroup + for name, handler := range repoManager.GetAll() { + wg.Add(1) + + go func(name string, handler repository.Handler) { + logger.Info("stopping repository handler", "repository", name) + handler.Stop() + logger.Info("repository handler stopped", "repository", name) + wg.Done() + }(name, handler) + } + wg.Wait() + }() + } ticker := time.NewTicker(time.Second * 5) @@ -104,7 +129,7 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err case beskarMeta := <-beskarMetaCh: ticker.Stop() - wh := webHandler[H]{ + wh := webHandler{ pluginInfo: serviceConfig.Info, manager: repoManager, } @@ -139,21 +164,6 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err _ = server.Shutdown(ctx) - var wg sync.WaitGroup - - for name, handler := range repoManager.GetAll() { - wg.Add(1) - - go func(name string, handler repository.Handler) { - logger.Info("stopping repository handler", "repository", name) - handler.Stop() - logger.Info("repository handler stopped", "repository", name) - wg.Done() - }(name, handler) - } - - wg.Wait() - return serverErr } From 95adf48d0251608c78adfe62eac866bf42faae6d Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Wed, 6 Dec 2023 23:03:22 -0500 Subject: [PATCH 12/30] initial implementation of ostree plugin (without mirroring) refactors yum plugin executable to use new non-generic Serve refactors static plugin executable to use new non-generic Serve --- .idea/beskar.iml | 9 ++ build/mage/build.go | 14 +++ cmd/beskar-ostree/main.go | 68 +++++++++++ cmd/beskar-static/main.go | 3 +- cmd/beskar-yum/main.go | 3 +- internal/plugins/ostree/embedded/data.json | 27 +++++ internal/plugins/ostree/embedded/router.rego | 64 ++++++++++ .../ostree/pkg/config/beskar-ostree.go | 93 ++++++++++++++ .../pkg/config/default/beskar-ostree.yaml | 16 +++ internal/plugins/ostree/plugin.go | 113 ++++++++++++++++++ pkg/plugins/ostree/api/v1/endpoint.go | 49 ++++++++ pkg/plugins/ostree/api/v1/http.go | 58 +++++++++ pkg/plugins/ostree/api/v1/http_client.go | 84 +++++++++++++ pkg/plugins/ostree/api/v1/oas2.go | 73 +++++++++++ 14 files changed, 670 insertions(+), 4 deletions(-) create mode 100644 .idea/beskar.iml create mode 100644 cmd/beskar-ostree/main.go create mode 100644 internal/plugins/ostree/embedded/data.json create mode 100644 internal/plugins/ostree/embedded/router.rego create mode 100644 internal/plugins/ostree/pkg/config/beskar-ostree.go create mode 100644 internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml create mode 100644 internal/plugins/ostree/plugin.go create mode 100644 pkg/plugins/ostree/api/v1/endpoint.go create mode 100644 pkg/plugins/ostree/api/v1/http.go create mode 100644 pkg/plugins/ostree/api/v1/http_client.go create mode 100644 pkg/plugins/ostree/api/v1/oas2.go diff --git a/.idea/beskar.iml b/.idea/beskar.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/beskar.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/build/mage/build.go b/build/mage/build.go index 65dad68..dbd1f98 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -61,6 +61,7 @@ const ( beskarctlBinary = "beskarctl" beskarYUMBinary = "beskar-yum" beskarStaticBinary = "beskar-static" + beskarOSTreeBinary = "beskar-ostree" ) var binaries = map[string]binaryConfig{ @@ -103,6 +104,18 @@ var binaries = map[string]binaryConfig{ useProto: true, baseImage: BaseImage, }, + beskarOSTreeBinary: { + configFiles: map[string]string{ + "internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml": "/etc/beskar/beskar-ostree.yaml", + }, + genAPI: &genAPI{ + path: "pkg/plugins/ostree/api/v1", + filename: "api.go", + interfaceName: "OSTree", + }, + useProto: true, + baseImage: "alpine:3.17", + }, } type Build mg.Namespace @@ -143,6 +156,7 @@ func (b Build) Plugins(ctx context.Context) { ctx, mg.F(b.Plugin, beskarYUMBinary), mg.F(b.Plugin, beskarStaticBinary), + mg.F(b.Plugin, beskarOSTreeBinary), ) } diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go new file mode 100644 index 0000000..1409d2f --- /dev/null +++ b/cmd/beskar-ostree/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "flag" + "fmt" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" + "go.ciq.dev/beskar/internal/plugins/ostree" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + "go.ciq.dev/beskar/pkg/sighandler" + "go.ciq.dev/beskar/pkg/version" + "log" + "net" + "os" + "syscall" +) + +var configDir string + +func serve(beskarOSTreeCmd *flag.FlagSet) error { + if err := beskarOSTreeCmd.Parse(os.Args[1:]); err != nil { + return err + } + + errCh := make(chan error) + + ctx, wait := sighandler.New(errCh, syscall.SIGTERM, syscall.SIGINT) + + beskarOSTreeConfig, err := config.ParseBeskarOSTreeConfig(configDir) + if err != nil { + return err + } + + ln, err := net.Listen("tcp", beskarOSTreeConfig.Addr) + if err != nil { + return err + } + defer ln.Close() + + plugin, err := ostree.New(ctx, beskarOSTreeConfig) + if err != nil { + return err + } + + go func() { + errCh <- pluginsrv.Serve(ln, plugin) + }() + + return wait(false) +} + +func main() { + beskarOSTreeCmd := flag.NewFlagSet("beskar-ostree", flag.ExitOnError) + beskarOSTreeCmd.StringVar(&configDir, "config-dir", "", "configuration directory") + + subCommand := "" + if len(os.Args) > 1 { + subCommand = os.Args[1] + } + + switch subCommand { + case "version": + fmt.Println(version.Semver) + default: + if err := serve(beskarOSTreeCmd); err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/beskar-static/main.go b/cmd/beskar-static/main.go index 9e4ca2a..9f68029 100644 --- a/cmd/beskar-static/main.go +++ b/cmd/beskar-static/main.go @@ -14,7 +14,6 @@ import ( "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/static" "go.ciq.dev/beskar/internal/plugins/static/pkg/config" - "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" ) @@ -47,7 +46,7 @@ func serve(beskarStaticCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve[*staticrepository.Handler](ln, plugin) + errCh <- pluginsrv.Serve(ln, plugin) }() return wait(false) diff --git a/cmd/beskar-yum/main.go b/cmd/beskar-yum/main.go index 30ec486..089a666 100644 --- a/cmd/beskar-yum/main.go +++ b/cmd/beskar-yum/main.go @@ -14,7 +14,6 @@ import ( "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/yum" "go.ciq.dev/beskar/internal/plugins/yum/pkg/config" - "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" ) @@ -47,7 +46,7 @@ func serve(beskarYumCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve[*yumrepository.Handler](ln, plugin) + errCh <- pluginsrv.Serve(ln, plugin) }() return wait(false) diff --git a/internal/plugins/ostree/embedded/data.json b/internal/plugins/ostree/embedded/data.json new file mode 100644 index 0000000..536198d --- /dev/null +++ b/internal/plugins/ostree/embedded/data.json @@ -0,0 +1,27 @@ +{ + "routes": [ + { + "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/file/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", + "methods": [ + "GET", + "HEAD" + ] + }, + { + "pattern": "^/artifacts/ostree/api/v1/doc/(.*)$", + "body": false + }, + { + "pattern": "^/artifacts/ostree/api/v1/(.*)$", + "body": true, + "body_key": "repository" + } + ], + "mediatype": { + "file": "application/vnd.ciq.ostree.v1.file" + }, + "tags": [ + "summary", + "config" + ] +} \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego new file mode 100644 index 0000000..42d8053 --- /dev/null +++ b/internal/plugins/ostree/embedded/router.rego @@ -0,0 +1,64 @@ +package router + +import future.keywords.if +import future.keywords.in + +default output = {"repository": "", "redirect_url": "", "found": false} + +filename_checksum(filename) = checksum if { + filename in data.tags + checksum := filename +} else = checksum { + checksum := crypto.md5(filename) +} + +blob_url(repo, filename) = url { + digest := oci.blob_digest(sprintf("%s:%s", [repo, filename_checksum(filename)]), "mediatype", data.mediatype.file) + url := { + "url": sprintf("/v2/%s/blobs/sha256:%s", [repo, digest]), + "found": digest != "", + } +} + +output = obj { + some index + input.method in data.routes[index].methods + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + redirect := blob_url( + sprintf("%s/files", [match[1]]), + match[2], + ) + obj := { + "repository": match[1], + "redirect_url": redirect.url, + "found": redirect.found + } +} else = obj if { + data.routes[index].body == true + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + repo := object.get({}, data.routes[index].body_key, "") + obj := { + "repository": repo, + "redirect_url": "", + "found": repo != "" + } +} else = obj { + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + obj := { + "repository": "", + "redirect_url": "", + "found": true + } +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go new file mode 100644 index 0000000..d692cfe --- /dev/null +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -0,0 +1,93 @@ +package config + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "github.com/distribution/distribution/v3/configuration" + "go.ciq.dev/beskar/internal/pkg/config" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" + "io" + "os" + "path/filepath" + "reflect" + "strings" +) + +const ( + BeskarOSTreeConfigFile = "beskar-ostree.yaml" + DefaultBeskarOSTreeDataDir = "/tmp/beskar-ostree" +) + +//go:embed default/beskar-ostree.yaml +var defaultBeskarOSTreeConfig string + +type BeskarOSTreeConfig struct { + Version string `yaml:"version"` + Log log.Config `yaml:"log"` + Addr string `yaml:"addr"` + Gossip gossip.Config `yaml:"gossip"` + Profiling bool `yaml:"profiling"` + DataDir string `yaml:"datadir"` + ConfigDirectory string `yaml:"-"` +} + +type BeskarOSTreeConfigV1 BeskarOSTreeConfig + +func ParseBeskarOSTreeConfig(dir string) (*BeskarOSTreeConfig, error) { + customDir := false + filename := filepath.Join(config.DefaultConfigDir, BeskarOSTreeConfigFile) + if dir != "" { + filename = filepath.Join(dir, BeskarOSTreeConfigFile) + customDir = true + } + + configDir := filepath.Dir(filename) + + var configReader io.Reader + + f, err := os.Open(filename) + if err != nil { + if !errors.Is(err, os.ErrNotExist) || customDir { + return nil, err + } + configReader = strings.NewReader(defaultBeskarOSTreeConfig) + configDir = "" + } else { + defer f.Close() + configReader = f + } + + configBuffer := new(bytes.Buffer) + if _, err := io.Copy(configBuffer, configReader); err != nil { + return nil, err + } + + configParser := configuration.NewParser("beskarostree", []configuration.VersionedParseInfo{ + { + Version: configuration.MajorMinorVersion(1, 0), + ParseAs: reflect.TypeOf(BeskarOSTreeConfigV1{}), + ConversionFunc: func(c interface{}) (interface{}, error) { + if v1, ok := c.(*BeskarOSTreeConfigV1); ok { + v1.ConfigDirectory = configDir + return (*BeskarOSTreeConfig)(v1), nil + } + return nil, fmt.Errorf("expected *BeskarOSTreeConfigV1, received %#v", c) + }, + }, + }) + + beskarOSTreeConfig := new(BeskarOSTreeConfig) + + if err := configParser.Parse(configBuffer.Bytes(), beskarOSTreeConfig); err != nil { + return nil, err + } + + if beskarOSTreeConfig.DataDir == "" { + beskarOSTreeConfig.DataDir = DefaultBeskarOSTreeDataDir + } + + return beskarOSTreeConfig, nil +} diff --git a/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml b/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml new file mode 100644 index 0000000..043fb9b --- /dev/null +++ b/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml @@ -0,0 +1,16 @@ +version: 1.0 + +addr: 0.0.0.0:5200 + +log: + level: debug + format: json + +profiling: true +datadir: /tmp/beskar-ostree + +gossip: + addr: 0.0.0.0:5201 + key: XD1IOhcp0HWFgZJ/HAaARqMKJwfMWtz284Yj7wxmerA= + peers: + - 127.0.0.1:5102 diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go new file mode 100644 index 0000000..8e6fbf1 --- /dev/null +++ b/internal/plugins/ostree/plugin.go @@ -0,0 +1,113 @@ +package ostree + +import ( + "context" + _ "embed" + "github.com/RussellLuo/kun/pkg/httpcodec" + "github.com/go-chi/chi" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" + "go.ciq.dev/beskar/internal/pkg/repository" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + pluginv1 "go.ciq.dev/beskar/pkg/api/plugin/v1" + "go.ciq.dev/beskar/pkg/mtls" + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" + "go.ciq.dev/beskar/pkg/version" + "net/http" + "net/http/pprof" +) + +const ( + PluginName = "ostree" + PluginAPIPathPattern = "/artifacts/ostree/api/v1" +) + +//go:embed embedded/router.rego +var routerRego []byte + +//go:embed embedded/data.json +var routerData []byte + +type Plugin struct { + ctx context.Context + config pluginsrv.Config +} + +func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*Plugin, error) { + logger, err := beskarOSTreeConfig.Log.Logger(log.ContextHandler) + if err != nil { + return nil, err + } + ctx = log.SetContextLogger(ctx, logger) + + apiSrv := newAPIService() + router := makeRouter(apiSrv, beskarOSTreeConfig.Profiling) + + return &Plugin{ + ctx: ctx, + config: pluginsrv.Config{ + Router: router, + Gossip: beskarOSTreeConfig.Gossip, + Info: &pluginv1.Info{ + Name: PluginName, + // Not registering media types so that Beskar doesn't send events. + // This plugin as no internal state so events are not needed. + Mediatypes: []string{}, + Version: version.Semver, + Router: &pluginv1.Router{ + Rego: routerRego, + Data: routerData, + }, + }, + }, + }, nil +} + +func (p *Plugin) Start(_ http.RoundTripper, _ *mtls.CAPEM, _ *gossip.BeskarMeta) error { + // Nothing to do here as this plugin has no internal state + // and router is already configured. + return nil +} + +func (p *Plugin) Context() context.Context { + return p.ctx +} + +func (p *Plugin) Config() pluginsrv.Config { + return p.config +} + +func (p *Plugin) RepositoryManager() *repository.Manager { + // this plugin has no internal state so no need for a repository manager + return nil +} + +func makeRouter(apiSrv *apiService, profilingEnabled bool) *chi.Mux { + router := chi.NewRouter() + + // for kubernetes probes + router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + if profilingEnabled { + router.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + router.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + router.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + router.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + router.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + router.Handle("/debug/pprof/{cmd}", http.HandlerFunc(pprof.Index)) // special handling for Gorilla mux + } + + router.Route( + PluginAPIPathPattern, + func(r chi.Router) { + r.Use(pluginsrv.IsTLSMiddleware) + r.Mount("/", apiv1.NewHTTPRouter( + apiSrv, + httpcodec.NewDefaultCodecs(nil), + )) + }, + ) + + return router +} diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go new file mode 100644 index 0000000..33252bf --- /dev/null +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -0,0 +1,49 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + + "github.com/RussellLuo/kun/pkg/httpoption" + "github.com/RussellLuo/validating/v3" + "github.com/go-kit/kit/endpoint" +) + +type MirrorRepositoryRequest struct { + Repository string `json:"repository"` + Depth int `json:"depth"` +} + +// ValidateMirrorRepositoryRequest creates a validator for MirrorRepositoryRequest. +func ValidateMirrorRepositoryRequest(newSchema func(*MirrorRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*MirrorRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type MirrorRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *MirrorRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *MirrorRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfMirrorRepository creates the endpoint for s.MirrorRepository. +func MakeEndpointOfMirrorRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*MirrorRepositoryRequest) + err := s.MirrorRepository( + ctx, + req.Repository, + req.Depth, + ) + return &MirrorRepositoryResponse{ + Err: err, + }, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http.go b/pkg/plugins/ostree/api/v1/http.go new file mode 100644 index 0000000..08d5f91 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/http.go @@ -0,0 +1,58 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + "net/http" + + "github.com/RussellLuo/kun/pkg/httpcodec" + "github.com/RussellLuo/kun/pkg/httpoption" + "github.com/RussellLuo/kun/pkg/oas2" + "github.com/go-chi/chi" + kithttp "github.com/go-kit/kit/transport/http" +) + +func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Option) chi.Router { + r := chi.NewRouter() + options := httpoption.NewOptions(opts...) + + r.Method("GET", "/doc/swagger.yaml", oas2.Handler(OASv2APIDoc, options.ResponseSchema())) + + var codec httpcodec.Codec + var validator httpoption.Validator + var kitOptions []kithttp.ServerOption + + codec = codecs.EncodeDecoder("MirrorRepository") + validator = options.RequestValidator("MirrorRepository") + r.Method( + "POST", "/repository/mirror", + kithttp.NewServer( + MakeEndpointOfMirrorRepository(svc), + decodeMirrorRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + return r +} + +func decodeMirrorRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req MirrorRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go new file mode 100644 index 0000000..bda1384 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -0,0 +1,84 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + "net/http" + "net/url" + "strings" + + "github.com/RussellLuo/kun/pkg/httpcodec" +) + +type HTTPClient struct { + codecs httpcodec.Codecs + httpClient *http.Client + scheme string + host string + pathPrefix string +} + +func NewHTTPClient(codecs httpcodec.Codecs, httpClient *http.Client, baseURL string) (*HTTPClient, error) { + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + return &HTTPClient{ + codecs: codecs, + httpClient: httpClient, + scheme: u.Scheme, + host: u.Host, + pathPrefix: strings.TrimSuffix(u.Path, "/"), + }, nil +} + +func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { + codec := c.codecs.EncodeDecoder("MirrorRepository") + + path := "/repository/mirror" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Depth int `json:"depth"` + }{ + Repository: repository, + Depth: depth, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go new file mode 100644 index 0000000..2a76e7e --- /dev/null +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -0,0 +1,73 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "reflect" + + "github.com/RussellLuo/kun/pkg/oas2" +) + +var ( + base = `swagger: "2.0" +info: + title: "OSTree Repository Management API" + version: "1.0.0" + description: "OSTree is used for managing ostree repositories.\nThis is the API documentation of OSTree.\n//" + license: + name: "MIT" +host: "example.com" +basePath: "/artifacts/ostree/api/v1" +schemes: + - "https" +consumes: + - "application/json" +produces: + - "application/json" +` + + paths = ` +paths: + /repository/mirror: + post: + description: "Mirror a static repository." + operationId: "MirrorRepository" + tags: + - static + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/MirrorRepositoryRequestBody" + %s +` +) + +func getResponses(schema oas2.Schema) []oas2.OASResponses { + return []oas2.OASResponses{ + oas2.GetOASResponses(schema, "MirrorRepository", 200, &MirrorRepositoryResponse{}), + } +} + +func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { + defs := make(map[string]oas2.Definition) + + oas2.AddDefinition(defs, "MirrorRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Depth int `json:"depth"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "MirrorRepository", 200, (&MirrorRepositoryResponse{}).Body()) + + return defs +} + +func OASv2APIDoc(schema oas2.Schema) string { + resps := getResponses(schema) + paths := oas2.GenPaths(resps, paths) + + defs := getDefinitions(schema) + definitions := oas2.GenDefinitions(defs) + + return base + paths + definitions +} From 29f5dbb8348ed02f80aac1d08d7a0ace4c435609 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 8 Dec 2023 18:47:08 -0500 Subject: [PATCH 13/30] gofumpt refactors beskctl to use cobra adds beskarctl ostree sub-command tree adds beskarctl static file sub-command tree documents beskarctl --- cmd/beskar-ostree/main.go | 15 ++- cmd/beskarctl/README.md | 32 +++++++ cmd/beskarctl/ctl/error.go | 13 +++ cmd/beskarctl/ctl/helpers.go | 47 ++++++++++ cmd/beskarctl/ctl/root.go | 31 +++++++ cmd/beskarctl/main.go | 92 +------------------ cmd/beskarctl/ostree/push.go | 73 +++++++++++++++ cmd/beskarctl/ostree/root.go | 23 +++++ cmd/beskarctl/static/push.go | 46 ++++++++++ cmd/beskarctl/static/root.go | 24 +++++ cmd/beskarctl/yum/push.go | 47 ++++++++++ cmd/beskarctl/yum/pushmetadata.go | 55 +++++++++++ cmd/beskarctl/yum/root.go | 34 +++++++ internal/plugins/ostree/api.go | 2 +- internal/plugins/ostree/embedded/router.rego | 4 +- .../ostree/pkg/config/beskar-ostree.go | 15 ++- internal/plugins/ostree/plugin.go | 5 +- internal/plugins/static/api.go | 1 + internal/plugins/yum/api.go | 1 + pkg/orasostree/ostree.go | 65 +++++++++++++ 20 files changed, 520 insertions(+), 105 deletions(-) create mode 100644 cmd/beskarctl/README.md create mode 100644 cmd/beskarctl/ctl/error.go create mode 100644 cmd/beskarctl/ctl/helpers.go create mode 100644 cmd/beskarctl/ctl/root.go create mode 100644 cmd/beskarctl/ostree/push.go create mode 100644 cmd/beskarctl/ostree/root.go create mode 100644 cmd/beskarctl/static/push.go create mode 100644 cmd/beskarctl/static/root.go create mode 100644 cmd/beskarctl/yum/push.go create mode 100644 cmd/beskarctl/yum/pushmetadata.go create mode 100644 cmd/beskarctl/yum/root.go create mode 100644 pkg/orasostree/ostree.go diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go index 1409d2f..4286ca4 100644 --- a/cmd/beskar-ostree/main.go +++ b/cmd/beskar-ostree/main.go @@ -3,15 +3,16 @@ package main import ( "flag" "fmt" + "log" + "net" + "os" + "syscall" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/ostree" "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" - "log" - "net" - "os" - "syscall" ) var configDir string @@ -34,7 +35,11 @@ func serve(beskarOSTreeCmd *flag.FlagSet) error { if err != nil { return err } - defer ln.Close() + defer func() { + if err := ln.Close(); err != nil { + fmt.Println(err) + } + }() plugin, err := ostree.New(ctx, beskarOSTreeConfig) if err != nil { diff --git a/cmd/beskarctl/README.md b/cmd/beskarctl/README.md new file mode 100644 index 0000000..22f3db0 --- /dev/null +++ b/cmd/beskarctl/README.md @@ -0,0 +1,32 @@ +# beskarctl +`beskarctl` is a command line tool for interacting with Beskar Artifact Registries. + +## Installation +``` +go install go.ciq.dev/beskar/cmd/beskarctl@latest +``` + +## Usage +`beskarctl` is very similar to `kubectl` in that it provide various subcommands for interacting with Beskar repositories. +The following subcommands are available: + ``` +beskarctl yum [flags] +beskarctl static [flags] +beskarctl ostree [flags] + ``` +For more information on a specific subcommand, run `beskarctl --help`. + +## Adding a new subcommand +Adding a new subcommand is fairly straightforward. Feel free to use the existing subcommands as a template, e.g., +`cmd/beskarctl/static/`. The following steps should be followed: + +1. Create a new file in `cmd/beskarctl//root.go`. +2. Add a new `cobra.Command` to the `rootCmd` variable in `cmd/beskarctl//root.go`. +3. Add an accessor function to `cmd/beskarctl//root.go` that returns the new `cobra.Command`. +4. Register the new subcommand in `cmd/beskarctl/ctl/root.go` by calling the accessor function. + +### Implementation Notes +- The `cobra.Command` you create should not be exported. Rather, your package should export an accessor function that +returns the `cobra.Command`. The accessor function is your chance to set up any flags or subcommands that your +`cobra.Command` needs. Please avoid the use of init functi +- helper functions are available for common values such as `--repo` and `--registry`. See `cmd/beskarctl/ctl/helpers.go` \ No newline at end of file diff --git a/cmd/beskarctl/ctl/error.go b/cmd/beskarctl/ctl/error.go new file mode 100644 index 0000000..cce9285 --- /dev/null +++ b/cmd/beskarctl/ctl/error.go @@ -0,0 +1,13 @@ +package ctl + +import "fmt" + +type Err string + +func (e Err) Error() string { + return string(e) +} + +func Errf(str string, a ...any) Err { + return Err(fmt.Sprintf(str, a...)) +} diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go new file mode 100644 index 0000000..349ab44 --- /dev/null +++ b/cmd/beskarctl/ctl/helpers.go @@ -0,0 +1,47 @@ +package ctl + +import ( + "github.com/spf13/cobra" + "os" +) + +const ( + ErrMissingFlagRepo = Err("missing repo flag") + ErrMissingFlagRegistry = Err("missing registry flag") +) + +const ( + FlagNameRepo = "repo" + FlagNameRegistry = "registry" +) + +// RegisterFlags registers the flags that are common to all commands. +func RegisterFlags(cmd *cobra.Command) { + // Flags that are common to all commands. + cmd.PersistentFlags().String(FlagNameRepo, "", "The repository to operate on.") + cmd.PersistentFlags().String(FlagNameRegistry, "", "The registry to operate on.") +} + +// Repo returns the repository name from the command line. +// If the repository is not specified, the command will exit with an error. +func Repo() string { + repo, err := rootCmd.Flags().GetString(FlagNameRepo) + if err != nil || repo == "" { + rootCmd.PrintErrln(ErrMissingFlagRepo) + os.Exit(1) + } + + return repo +} + +// Registry returns the registry name from the command line. +// If the registry is not specified, the command will exit with an error. +func Registry() string { + registry, err := rootCmd.Flags().GetString(FlagNameRegistry) + if err != nil || registry == "" { + rootCmd.PrintErrln(ErrMissingFlagRegistry) + os.Exit(1) + } + + return registry +} diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go new file mode 100644 index 0000000..b057c89 --- /dev/null +++ b/cmd/beskarctl/ctl/root.go @@ -0,0 +1,31 @@ +package ctl + +import ( + "fmt" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ostree" + "go.ciq.dev/beskar/cmd/beskarctl/static" + "go.ciq.dev/beskar/cmd/beskarctl/yum" + "os" +) + +var rootCmd = &cobra.Command{ + Use: "beskarctl", + Short: "Operations related to beskar.", +} + +func Execute() { + RegisterFlags(rootCmd) + + rootCmd.AddCommand( + yum.RootCmd(), + static.RootCmd(), + ostree.RootCmd(), + ) + + err := rootCmd.Execute() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/cmd/beskarctl/main.go b/cmd/beskarctl/main.go index 360ad3a..31be082 100644 --- a/cmd/beskarctl/main.go +++ b/cmd/beskarctl/main.go @@ -3,96 +3,8 @@ package main -import ( - "flag" - "fmt" - "os" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "go.ciq.dev/beskar/pkg/oras" - "go.ciq.dev/beskar/pkg/orasrpm" - "go.ciq.dev/beskar/pkg/version" -) - -func fatal(format string, a ...any) { - fmt.Printf(format+"\n", a...) - os.Exit(1) -} +import "go.ciq.dev/beskar/cmd/beskarctl/ctl" func main() { - pushCmd := flag.NewFlagSet("push", flag.ExitOnError) - pushRepo := pushCmd.String("repo", "", "repo") - pushRegistry := pushCmd.String("registry", "", "registry") - - pushMetadataCmd := flag.NewFlagSet("push-metadata", flag.ExitOnError) - pushMetadataRepo := pushMetadataCmd.String("repo", "", "repo") - pushMetadataRegistry := pushMetadataCmd.String("registry", "", "registry") - pushMetadataType := pushMetadataCmd.String("type", "", "type") - - if len(os.Args) == 1 { - fatal("missing subcommand") - } - - switch os.Args[1] { - case "version": - fmt.Println(version.Semver) - case "push": - if err := pushCmd.Parse(os.Args[2:]); err != nil { - fatal("while parsing command arguments: %w", err) - } - rpm := pushCmd.Arg(0) - if rpm == "" { - fatal("an RPM package must be specified") - } else if pushRegistry == nil || *pushRegistry == "" { - fatal("a registry must be specified") - } else if pushRepo == nil || *pushRepo == "" { - fatal("a repo must be specified") - } - if err := push(rpm, *pushRepo, *pushRegistry); err != nil { - fatal("while pushing RPM package: %s", err) - } - case "push-metadata": - if err := pushMetadataCmd.Parse(os.Args[2:]); err != nil { - fatal("while parsing command arguments: %w", err) - } - metadata := pushMetadataCmd.Arg(0) - if metadata == "" { - fatal("a metadata file must be specified") - } else if pushMetadataRegistry == nil || *pushMetadataRegistry == "" { - fatal("a registry must be specified") - } else if pushMetadataRepo == nil || *pushMetadataRepo == "" { - fatal("a repo must be specified") - } else if pushMetadataType == nil || *pushMetadataType == "" { - fatal("a metadata type must be specified") - } - if err := pushMetadata(metadata, *pushMetadataType, *pushMetadataRepo, *pushMetadataRegistry); err != nil { - fatal("while pushing metadata: %s", err) - } - default: - fatal("unknown %q subcommand", os.Args[1]) - } -} - -func push(rpmPath string, repo, registry string) error { - pusher, err := orasrpm.NewRPMPusher(rpmPath, repo, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating RPM pusher: %w", err) - } - - fmt.Printf("Pushing %s to %s\n", rpmPath, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) -} - -func pushMetadata(metadataPath string, dataType, repo, registry string) error { - pusher, err := orasrpm.NewRPMExtraMetadataPusher(metadataPath, repo, dataType, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating RPM metadata pusher: %w", err) - } - - fmt.Printf("Pushing %s to %s\n", metadataPath, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + ctl.Execute() } diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go new file mode 100644 index 0000000..3644649 --- /dev/null +++ b/cmd/beskarctl/ostree/push.go @@ -0,0 +1,73 @@ +package ostree + +import ( + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasostree" + "os" + "path/filepath" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [directory]", + Short: "Push an ostree repository.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + if dir == "" { + return ctl.Err("a directory must be specified") + } + + if err := pushOSTreeRepository(dir, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing ostree repository: %s", err) + } + return nil + }, + } +) + +func PushCmd() *cobra.Command { + return pushCmd +} + +func pushOSTreeRepository(dir, repo, registry string) error { + // Prove that we were given the root directory of an ostree repository + // by checking for the existence of the summary file. + fileInfo, err := os.Stat(filepath.Join(dir, orasostree.KnownFileSummary)) + if err != nil || fileInfo.IsDir() { + return fmt.Errorf("%s file not found in %s", orasostree.KnownFileSummary, dir) + } + + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("while walking %s: %w", path, err) + } + + if info.IsDir() { + return nil + } + + if err := push(path, repo, registry); err != nil { + return fmt.Errorf("while pushing %s: %w", path, err) + } + + return nil + }) +} + +func push(filepath, repo, registry string) error { + pusher, err := orasostree.NewOSTreePusher(filepath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating StaticFile pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/ostree/root.go b/cmd/beskarctl/ostree/root.go new file mode 100644 index 0000000..6c8d70b --- /dev/null +++ b/cmd/beskarctl/ostree/root.go @@ -0,0 +1,23 @@ +package ostree + +import ( + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "ostree", + Aliases: []string{ + "o", + }, + Short: "Operations related to ostree repositories.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + ) + + return rootCmd +} diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go new file mode 100644 index 0000000..7b7115d --- /dev/null +++ b/cmd/beskarctl/static/push.go @@ -0,0 +1,46 @@ +package static + +import ( + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasfile" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [file]", + Short: "Push a file.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + file := args[0] + if file == "" { + return ctl.Err("file must be specified") + } + + if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing static file: %s", err) + } + return nil + }, + } +) + +func PushCmd() *cobra.Command { + return pushCmd +} + +func push(filepath, repo, registry string) error { + pusher, err := orasfile.NewStaticFilePusher(filepath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating StaticFile pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go new file mode 100644 index 0000000..2cc5406 --- /dev/null +++ b/cmd/beskarctl/static/root.go @@ -0,0 +1,24 @@ +package static + +import ( + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "static", + Aliases: []string{ + "file", + "s", + }, + Short: "Operations related to static files.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + ) + + return rootCmd +} diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go new file mode 100644 index 0000000..19fddcc --- /dev/null +++ b/cmd/beskarctl/yum/push.go @@ -0,0 +1,47 @@ +package yum + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasrpm" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [rpm filepath]", + Short: "Push a yum repository to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rpm := args[0] + if rpm == "" { + return ctl.Err("an RPM package must be specified") + } + + if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing RPM package: %s", err) + } + return nil + }, + } +) + +func PushCmd() *cobra.Command { + return pushCmd +} + +func push(rpmPath, repo, registry string) error { + pusher, err := orasrpm.NewRPMPusher(rpmPath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating RPM pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", rpmPath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go new file mode 100644 index 0000000..d8344af --- /dev/null +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -0,0 +1,55 @@ +package yum + +import ( + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasrpm" +) + +// yum push-metadata +var ( + pushMetadataCmd = &cobra.Command{ + Use: "push-metadata [metadata filepath]", + Short: "Push yum repository metadata to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + metadata := args[0] + if metadata == "" { + return ctl.Err("a metadata file must be specified") + } else if registry == "" { + return MissingRequiredFlagRegistry + } else if repo == "" { + return MissingRequiredFlagRepo + } else if pushMetadataType == "" { + return ctl.Err("a metadata type must be specified") + } + + if err := pushMetadata(metadata, pushMetadataType, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing metadata: %s", err) + } + return nil + }, + } + pushMetadataType string +) + +func PushMetadataCmd() *cobra.Command { + pushMetadataCmd.Flags().StringVarP(&pushMetadataType, "type", "t", "", "type") + return pushMetadataCmd +} + +func pushMetadata(metadataPath, dataType, repo, registry string) error { + pusher, err := orasrpm.NewRPMExtraMetadataPusher(metadataPath, repo, dataType, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating RPM metadata pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", metadataPath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/yum/root.go b/cmd/beskarctl/yum/root.go new file mode 100644 index 0000000..71a8912 --- /dev/null +++ b/cmd/beskarctl/yum/root.go @@ -0,0 +1,34 @@ +package yum + +import ( + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" +) + +const ( + MissingRequiredFlagRepo ctl.Err = "a repo must be specified" + MissingRequiredFlagRegistry ctl.Err = "a registry must be specified" +) + +var ( + repo string + registry string + rootCmd = &cobra.Command{ + Use: "yum", + Aliases: []string{ + "y", + "rpm", + "dnf", + }, + Short: "Operations related to yum repositories.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + PushMetadataCmd(), + ) + + return rootCmd +} diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index aca8687..15c8c26 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -3,6 +3,7 @@ package ostree import ( "context" "errors" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" ) @@ -14,6 +15,5 @@ func newAPIService() *apiService { } func (o *apiService) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { - //TODO implement me return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) } diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego index 42d8053..1db8b58 100644 --- a/internal/plugins/ostree/embedded/router.rego +++ b/internal/plugins/ostree/embedded/router.rego @@ -5,7 +5,7 @@ import future.keywords.in default output = {"repository": "", "redirect_url": "", "found": false} -filename_checksum(filename) = checksum if { +makeTag(filename) = checksum if { filename in data.tags checksum := filename } else = checksum { @@ -13,7 +13,7 @@ filename_checksum(filename) = checksum if { } blob_url(repo, filename) = url { - digest := oci.blob_digest(sprintf("%s:%s", [repo, filename_checksum(filename)]), "mediatype", data.mediatype.file) + digest := oci.blob_digest(sprintf("%s:%s", [repo, makeTag(filename)]), "mediatype", data.mediatype.file) url := { "url": sprintf("/v2/%s/blobs/sha256:%s", [repo, digest]), "found": digest != "", diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go index d692cfe..d8743ca 100644 --- a/internal/plugins/ostree/pkg/config/beskar-ostree.go +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -5,15 +5,16 @@ import ( _ "embed" "errors" "fmt" - "github.com/distribution/distribution/v3/configuration" - "go.ciq.dev/beskar/internal/pkg/config" - "go.ciq.dev/beskar/internal/pkg/gossip" - "go.ciq.dev/beskar/internal/pkg/log" "io" "os" "path/filepath" "reflect" "strings" + + "github.com/distribution/distribution/v3/configuration" + "go.ciq.dev/beskar/internal/pkg/config" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" ) const ( @@ -56,7 +57,11 @@ func ParseBeskarOSTreeConfig(dir string) (*BeskarOSTreeConfig, error) { configReader = strings.NewReader(defaultBeskarOSTreeConfig) configDir = "" } else { - defer f.Close() + defer func() { + if err := f.Close(); err != nil { + fmt.Println(err) + } + }() configReader = f } diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go index 8e6fbf1..2f7ed05 100644 --- a/internal/plugins/ostree/plugin.go +++ b/internal/plugins/ostree/plugin.go @@ -3,6 +3,9 @@ package ostree import ( "context" _ "embed" + "net/http" + "net/http/pprof" + "github.com/RussellLuo/kun/pkg/httpcodec" "github.com/go-chi/chi" "go.ciq.dev/beskar/internal/pkg/gossip" @@ -14,8 +17,6 @@ import ( "go.ciq.dev/beskar/pkg/mtls" apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" "go.ciq.dev/beskar/pkg/version" - "net/http" - "net/http/pprof" ) const ( diff --git a/internal/plugins/static/api.go b/internal/plugins/static/api.go index 83e9e16..3249b6f 100644 --- a/internal/plugins/static/api.go +++ b/internal/plugins/static/api.go @@ -5,6 +5,7 @@ package static import ( "context" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "github.com/RussellLuo/kun/pkg/werror" diff --git a/internal/plugins/yum/api.go b/internal/plugins/yum/api.go index 59f09b3..1a7f8e2 100644 --- a/internal/plugins/yum/api.go +++ b/internal/plugins/yum/api.go @@ -5,6 +5,7 @@ package yum import ( "context" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "github.com/RussellLuo/kun/pkg/werror" diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go new file mode 100644 index 0000000..65241f7 --- /dev/null +++ b/pkg/orasostree/ostree.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package orasostree + +import ( + "crypto/md5" //nolint:gosec + "fmt" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "go.ciq.dev/beskar/pkg/oras" +) + +const ( + OSTreeConfigType = "application/vnd.ciq.ostree.file.v1.config+json" + OSTreeLayerType = "application/vnd.ciq.ostree.v1.file" + + KnownFileSummary = "summary" + KnownFileConfig = "config" +) + +func NewOSTreePusher(path, repo string, opts ...name.Option) (oras.Pusher, error) { + if !strings.HasPrefix(repo, "artifacts/") { + if !strings.HasPrefix(repo, "ostree/") { + repo = filepath.Join("static", repo) + } + + repo = filepath.Join("artifacts", repo) + } + + filename := filepath.Base(path) + //nolint:gosec + fileTag := makeTag(filename) + + rawRef := filepath.Join(repo, "files:"+fileTag) + ref, err := name.ParseReference(rawRef, opts...) + if err != nil { + return nil, fmt.Errorf("while parsing reference %s: %w", rawRef, err) + } + + return oras.NewGenericPusher( + ref, + oras.NewManifestConfig(OSTreeConfigType, nil), + oras.NewLocalFileLayer(path, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), + ), nil +} + +// specialTags +var specialTags = []string{ + KnownFileSummary, + KnownFileConfig, +} + +func makeTag(filename string) string { + for _, tag := range specialTags { + if strings.HasPrefix(filename, tag) { + return tag + } + } + + //nolint:gosec + return fmt.Sprintf("%x", md5.Sum([]byte(filename))) +} From 80d309d6d9a0014c01957e38ac29a6a9cd0ebfac Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 15 Dec 2023 12:40:54 -0500 Subject: [PATCH 14/30] gofumpt fixes kun command typos adds summary.sig as known tag changes regex separator from file to repo refactors beskarctl to use cobra refactors beskarctl subcommands into packages implements beskarctl ostree push fixes ostree router.rego request body issue --- cmd/beskarctl/ctl/helpers.go | 3 +- cmd/beskarctl/ctl/root.go | 12 ++-- cmd/beskarctl/main.go | 13 ++++- cmd/beskarctl/ostree/push.go | 58 ++++++++++++++++---- cmd/beskarctl/ostree/root.go | 16 +++--- cmd/beskarctl/static/push.go | 35 ++++++------ cmd/beskarctl/static/root.go | 18 +++--- cmd/beskarctl/yum/push.go | 34 ++++++------ cmd/beskarctl/yum/pushmetadata.go | 1 + internal/plugins/ostree/README.md | 20 +++++++ internal/plugins/ostree/embedded/data.json | 3 +- internal/plugins/ostree/embedded/router.rego | 4 +- pkg/orasostree/ostree.go | 42 +++++++++----- pkg/plugins/ostree/api/v1/api.go | 4 +- pkg/plugins/ostree/api/v1/oas2.go | 4 +- 15 files changed, 170 insertions(+), 97 deletions(-) create mode 100644 internal/plugins/ostree/README.md diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go index 349ab44..7a34a53 100644 --- a/cmd/beskarctl/ctl/helpers.go +++ b/cmd/beskarctl/ctl/helpers.go @@ -1,8 +1,9 @@ package ctl import ( - "github.com/spf13/cobra" "os" + + "github.com/spf13/cobra" ) const ( diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go index b057c89..a44338d 100644 --- a/cmd/beskarctl/ctl/root.go +++ b/cmd/beskarctl/ctl/root.go @@ -2,11 +2,9 @@ package ctl import ( "fmt" - "github.com/spf13/cobra" - "go.ciq.dev/beskar/cmd/beskarctl/ostree" - "go.ciq.dev/beskar/cmd/beskarctl/static" - "go.ciq.dev/beskar/cmd/beskarctl/yum" "os" + + "github.com/spf13/cobra" ) var rootCmd = &cobra.Command{ @@ -14,13 +12,11 @@ var rootCmd = &cobra.Command{ Short: "Operations related to beskar.", } -func Execute() { +func Execute(cmds ...*cobra.Command) { RegisterFlags(rootCmd) rootCmd.AddCommand( - yum.RootCmd(), - static.RootCmd(), - ostree.RootCmd(), + cmds..., ) err := rootCmd.Execute() diff --git a/cmd/beskarctl/main.go b/cmd/beskarctl/main.go index 31be082..943cb8c 100644 --- a/cmd/beskarctl/main.go +++ b/cmd/beskarctl/main.go @@ -3,8 +3,17 @@ package main -import "go.ciq.dev/beskar/cmd/beskarctl/ctl" +import ( + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/cmd/beskarctl/ostree" + "go.ciq.dev/beskar/cmd/beskarctl/static" + "go.ciq.dev/beskar/cmd/beskarctl/yum" +) func main() { - ctl.Execute() + ctl.Execute( + yum.RootCmd(), + static.RootCmd(), + ostree.RootCmd(), + ) } diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go index 3644649..9edd759 100644 --- a/cmd/beskarctl/ostree/push.go +++ b/cmd/beskarctl/ostree/push.go @@ -1,7 +1,12 @@ package ostree import ( + "context" "fmt" + "os" + "path/filepath" + "strings" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -9,8 +14,7 @@ import ( "go.ciq.dev/beskar/cmd/beskarctl/ctl" "go.ciq.dev/beskar/pkg/oras" "go.ciq.dev/beskar/pkg/orasostree" - "os" - "path/filepath" + "golang.org/x/sync/errgroup" ) var ( @@ -30,12 +34,18 @@ var ( return nil }, } + jobCount int ) func PushCmd() *cobra.Command { + pushCmd.PersistentFlags().IntVarP(&jobCount, "jobs", "j", 10, "The repository to operate on.") return pushCmd } +// pushOSTreeRepository walks a local ostree repository and pushes each file to the given registry. +// dir is the root directory of the ostree repository, i.e., the directory containing the summary file. +// repo is the name of the ostree repository. +// registry is the registry to push to. func pushOSTreeRepository(dir, repo, registry string) error { // Prove that we were given the root directory of an ostree repository // by checking for the existence of the summary file. @@ -44,30 +54,58 @@ func pushOSTreeRepository(dir, repo, registry string) error { return fmt.Errorf("%s file not found in %s", orasostree.KnownFileSummary, dir) } - return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // Create a worker pool to push each file in the repository concurrently. + // ctx will be cancelled on error, and the error will be returned. + eg, ctx := errgroup.WithContext(context.Background()) + eg.SetLimit(jobCount) + + // Walk the directory tree, skipping directories and pushing each file. + if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + // If there was an error with the file, return it. if err != nil { return fmt.Errorf("while walking %s: %w", path, err) } - if info.IsDir() { + // Skip directories. + if d.IsDir() { return nil } - if err := push(path, repo, registry); err != nil { - return fmt.Errorf("while pushing %s: %w", path, err) + if ctx.Err() != nil { + // Skip remaining files because our context has been cancelled. + // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). + // This is because we would never be able to handle an error returned from the last job. + return filepath.SkipAll } + eg.Go(func() error { + if err := push(dir, path, repo, registry); err != nil { + return fmt.Errorf("while pushing %s: %w", path, err) + } + return nil + }) + return nil - }) + }); err != nil { + // We should only receive here if filepath.WalkDir() returns an error. + // Push errors are handled below. + return fmt.Errorf("while walking %s: %w", dir, err) + } + + // Wait for all workers to finish. + // If any worker returns an error, eg.Wait() will return that error. + return eg.Wait() } -func push(filepath, repo, registry string) error { - pusher, err := orasostree.NewOSTreePusher(filepath, repo, name.WithDefaultRegistry(registry)) +func push(repoRootDir, path, repo, registry string) error { + pusher, err := orasostree.NewOSTreePusher(repoRootDir, path, repo, name.WithDefaultRegistry(registry)) if err != nil { return fmt.Errorf("while creating StaticFile pusher: %w", err) } - fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + path = strings.TrimPrefix(path, repoRootDir) + path = strings.TrimPrefix(path, "/") + fmt.Printf("Pushing %s to %s\n", path, pusher.Reference()) return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) } diff --git a/cmd/beskarctl/ostree/root.go b/cmd/beskarctl/ostree/root.go index 6c8d70b..570c1cf 100644 --- a/cmd/beskarctl/ostree/root.go +++ b/cmd/beskarctl/ostree/root.go @@ -4,15 +4,13 @@ import ( "github.com/spf13/cobra" ) -var ( - rootCmd = &cobra.Command{ - Use: "ostree", - Aliases: []string{ - "o", - }, - Short: "Operations related to ostree repositories.", - } -) +var rootCmd = &cobra.Command{ + Use: "ostree", + Aliases: []string{ + "o", + }, + Short: "Operations related to ostree repositories.", +} func RootCmd() *cobra.Command { rootCmd.AddCommand( diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go index 7b7115d..4848f76 100644 --- a/cmd/beskarctl/static/push.go +++ b/cmd/beskarctl/static/push.go @@ -2,6 +2,7 @@ package static import ( "fmt" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -11,24 +12,22 @@ import ( "go.ciq.dev/beskar/pkg/orasfile" ) -var ( - pushCmd = &cobra.Command{ - Use: "push [file]", - Short: "Push a file.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - file := args[0] - if file == "" { - return ctl.Err("file must be specified") - } - - if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { - return ctl.Errf("while pushing static file: %s", err) - } - return nil - }, - } -) +var pushCmd = &cobra.Command{ + Use: "push [file]", + Short: "Push a file.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + file := args[0] + if file == "" { + return ctl.Err("file must be specified") + } + + if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing static file: %s", err) + } + return nil + }, +} func PushCmd() *cobra.Command { return pushCmd diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go index 2cc5406..8eb377f 100644 --- a/cmd/beskarctl/static/root.go +++ b/cmd/beskarctl/static/root.go @@ -4,16 +4,14 @@ import ( "github.com/spf13/cobra" ) -var ( - rootCmd = &cobra.Command{ - Use: "static", - Aliases: []string{ - "file", - "s", - }, - Short: "Operations related to static files.", - } -) +var rootCmd = &cobra.Command{ + Use: "static", + Aliases: []string{ + "file", + "s", + }, + Short: "Operations related to static files.", +} func RootCmd() *cobra.Command { rootCmd.AddCommand( diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go index 19fddcc..d211154 100644 --- a/cmd/beskarctl/yum/push.go +++ b/cmd/beskarctl/yum/push.go @@ -12,24 +12,22 @@ import ( "go.ciq.dev/beskar/pkg/orasrpm" ) -var ( - pushCmd = &cobra.Command{ - Use: "push [rpm filepath]", - Short: "Push a yum repository to a registry.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - rpm := args[0] - if rpm == "" { - return ctl.Err("an RPM package must be specified") - } - - if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { - return ctl.Errf("while pushing RPM package: %s", err) - } - return nil - }, - } -) +var pushCmd = &cobra.Command{ + Use: "push [rpm filepath]", + Short: "Push a yum repository to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rpm := args[0] + if rpm == "" { + return ctl.Err("an RPM package must be specified") + } + + if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing RPM package: %s", err) + } + return nil + }, +} func PushCmd() *cobra.Command { return pushCmd diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go index d8344af..043dbdc 100644 --- a/cmd/beskarctl/yum/pushmetadata.go +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -2,6 +2,7 @@ package yum import ( "fmt" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" diff --git a/internal/plugins/ostree/README.md b/internal/plugins/ostree/README.md new file mode 100644 index 0000000..8ed1e84 --- /dev/null +++ b/internal/plugins/ostree/README.md @@ -0,0 +1,20 @@ +# OSTree Plugin + +## Overview +The ostree plugin is responsible for mapping the ostree repository to the OCI registry. This is done in the router.rego +and no routing execution happens within the plugin itself at runtime. The plugin does, however, provide an API for mirroring +ostree repositories into beskar. + +## File Tagging +The ostree plugin maps the ostree repository filepaths to the OCI registry tags. Most files are simply mapped by hashing +the full filepath relative to the ostree root. For example, `objects/ab/abcd1234.filez` becomes `file:b8458bd029a97ca5e03f272a6b7bd0d1`. +There are a few exceptions to this rule, however. The following files are considered "special" and are tagged as follows: +1. `summary` -> `file:summary` +2. `summary.sig` -> `file:summary.sig` +3. `config` -> `file:config` + +There is no technical reason for this and is only done to make the mapping more human-readable in the case of "special" +files. + +## Mirroring +TBD \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/data.json b/internal/plugins/ostree/embedded/data.json index 536198d..337cbc3 100644 --- a/internal/plugins/ostree/embedded/data.json +++ b/internal/plugins/ostree/embedded/data.json @@ -1,7 +1,7 @@ { "routes": [ { - "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/file/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", + "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/repo/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", "methods": [ "GET", "HEAD" @@ -22,6 +22,7 @@ }, "tags": [ "summary", + "summary.sig", "config" ] } \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego index 1db8b58..08f365e 100644 --- a/internal/plugins/ostree/embedded/router.rego +++ b/internal/plugins/ostree/embedded/router.rego @@ -29,7 +29,7 @@ output = obj { 1 )[0] redirect := blob_url( - sprintf("%s/files", [match[1]]), + sprintf("%s/file", [match[1]]), match[2], ) obj := { @@ -44,7 +44,7 @@ output = obj { input.path, 1 )[0] - repo := object.get({}, data.routes[index].body_key, "") + repo := object.get(request.body(), data.routes[index].body_key, "") obj := { "repository": repo, "redirect_url": "", diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go index 65241f7..0a86011 100644 --- a/pkg/orasostree/ostree.go +++ b/pkg/orasostree/ostree.go @@ -14,48 +14,62 @@ import ( ) const ( + ArtifactsPathPrefix = "artifacts" + OSTreePathPrefix = "ostree" + OSTreeConfigType = "application/vnd.ciq.ostree.file.v1.config+json" OSTreeLayerType = "application/vnd.ciq.ostree.v1.file" - KnownFileSummary = "summary" - KnownFileConfig = "config" + KnownFileSummary = "summary" + KnownFileSummarySig = "summary.sig" + KnownFileConfig = "config" ) -func NewOSTreePusher(path, repo string, opts ...name.Option) (oras.Pusher, error) { - if !strings.HasPrefix(repo, "artifacts/") { - if !strings.HasPrefix(repo, "ostree/") { - repo = filepath.Join("static", repo) +func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras.Pusher, error) { + if !strings.HasPrefix(repo, ArtifactsPathPrefix+"/") { + if !strings.HasPrefix(repo, OSTreePathPrefix+"/") { + repo = filepath.Join(OSTreePathPrefix, repo) } - repo = filepath.Join("artifacts", repo) + repo = filepath.Join(ArtifactsPathPrefix, repo) } - filename := filepath.Base(path) - //nolint:gosec - fileTag := makeTag(filename) + // Sanitize the path to match the format of the tag. See internal/plugins/ostree/embedded/data.json. + // In this case the file path needs to be relative to the repository root and not contain a leading slash. + path = strings.TrimPrefix(path, repoRootDir) + path = strings.TrimPrefix(path, "/") - rawRef := filepath.Join(repo, "files:"+fileTag) + fileTag := makeTag(path) + rawRef := filepath.Join(repo, "file:"+fileTag) ref, err := name.ParseReference(rawRef, opts...) if err != nil { return nil, fmt.Errorf("while parsing reference %s: %w", rawRef, err) } + absolutePath := filepath.Join(repoRootDir, path) + return oras.NewGenericPusher( ref, oras.NewManifestConfig(OSTreeConfigType, nil), - oras.NewLocalFileLayer(path, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), + oras.NewLocalFileLayer(absolutePath, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), ), nil } -// specialTags +// specialTags are tags that are not md5 hashes of the filename. +// These files are meant to stand out in the registry. +// Note: Values are not limited to the repo's root directory, but at the moment on the following have been identified. var specialTags = []string{ KnownFileSummary, + KnownFileSummarySig, KnownFileConfig, } +// makeTag creates a tag for a file. +// If the filename starts with a special tag, the tag is returned as-is. +// Otherwise, the tag is the md5 hash of the filename. func makeTag(filename string) string { for _, tag := range specialTags { - if strings.HasPrefix(filename, tag) { + if filename == tag { return tag } } diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 9316106..8d9296c 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -31,9 +31,9 @@ type Page struct { //kun:oas version=1.0.0 //kun:oas basePath=/artifacts/ostree/api/v1 //kun:oas docsPath=/doc/swagger.yaml -//kun:oas tags=static +//kun:oas tags=ostree type OSTree interface { - // Mirror a static repository. + // Mirror an ostree repository. //kun:op POST /repository/mirror //kun:success statusCode=200 MirrorRepository(ctx context.Context, repository string, depth int) (err error) diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go index 2a76e7e..b3b5f4b 100644 --- a/pkg/plugins/ostree/api/v1/oas2.go +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -31,10 +31,10 @@ produces: paths: /repository/mirror: post: - description: "Mirror a static repository." + description: "Mirror an ostree repository." operationId: "MirrorRepository" tags: - - static + - ostree parameters: - name: body in: body From d75a5b11a28474d9753a8742da19ec8d50e62624 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 15 Dec 2023 14:50:10 -0500 Subject: [PATCH 15/30] various changes from PR review fixes for linter --- cmd/beskarctl/ostree/push.go | 14 +++++++++++--- internal/pkg/beskar/plugin.go | 4 ---- internal/pkg/pluginsrv/service.go | 4 ++-- internal/pkg/pluginsrv/webhandler.go | 10 +--------- internal/plugins/ostree/api.go | 2 +- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go index 9edd759..191b462 100644 --- a/cmd/beskarctl/ostree/push.go +++ b/cmd/beskarctl/ostree/push.go @@ -38,7 +38,13 @@ var ( ) func PushCmd() *cobra.Command { - pushCmd.PersistentFlags().IntVarP(&jobCount, "jobs", "j", 10, "The repository to operate on.") + pushCmd.Flags().IntVarP( + &jobCount, + "jobs", + "j", + 10, + "The number of concurrent jobs to use for pushing the repository.", + ) return pushCmd } @@ -50,8 +56,10 @@ func pushOSTreeRepository(dir, repo, registry string) error { // Prove that we were given the root directory of an ostree repository // by checking for the existence of the summary file. fileInfo, err := os.Stat(filepath.Join(dir, orasostree.KnownFileSummary)) - if err != nil || fileInfo.IsDir() { + if os.IsNotExist(err) || fileInfo.IsDir() { return fmt.Errorf("%s file not found in %s", orasostree.KnownFileSummary, dir) + } else if err != nil { + return fmt.Errorf("error accessing %s in %s: %w", orasostree.KnownFileSummary, dir, err) } // Create a worker pool to push each file in the repository concurrently. @@ -100,7 +108,7 @@ func pushOSTreeRepository(dir, repo, registry string) error { func push(repoRootDir, path, repo, registry string) error { pusher, err := orasostree.NewOSTreePusher(repoRootDir, path, repo, name.WithDefaultRegistry(registry)) if err != nil { - return fmt.Errorf("while creating StaticFile pusher: %w", err) + return fmt.Errorf("while creating OSTree pusher: %w", err) } path = strings.TrimPrefix(path, repoRootDir) diff --git a/internal/pkg/beskar/plugin.go b/internal/pkg/beskar/plugin.go index 89bc066..c77e34a 100644 --- a/internal/pkg/beskar/plugin.go +++ b/internal/pkg/beskar/plugin.go @@ -411,7 +411,3 @@ func loadPlugins(ctx context.Context) (func(), error) { return wg.Wait, nil } - -// Mountain Team - Lead Developer -// Fuzzball Team - -// Innovation Group - Tech Ambassador () diff --git a/internal/pkg/pluginsrv/service.go b/internal/pkg/pluginsrv/service.go index 59e81e1..9ec2dfc 100644 --- a/internal/pkg/pluginsrv/service.go +++ b/internal/pkg/pluginsrv/service.go @@ -134,8 +134,8 @@ func Serve(ln net.Listener, service Service) (errFn error) { manager: repoManager, } - serviceConfig.Router.HandleFunc("/event", http.HandlerFunc(wh.event)) - serviceConfig.Router.HandleFunc("/info", http.HandlerFunc(wh.info)) + serviceConfig.Router.With(IsTLSMiddleware).HandleFunc("/event", wh.event) + serviceConfig.Router.With(IsTLSMiddleware).HandleFunc("/info", wh.info) transport, err := getBeskarTransport(caPEM, beskarMeta) if err != nil { diff --git a/internal/pkg/pluginsrv/webhandler.go b/internal/pkg/pluginsrv/webhandler.go index 06232e0..a81a5e7 100644 --- a/internal/pkg/pluginsrv/webhandler.go +++ b/internal/pkg/pluginsrv/webhandler.go @@ -41,11 +41,7 @@ func IsTLSMiddleware(next http.Handler) http.Handler { func (wh *webHandler) event(w http.ResponseWriter, r *http.Request) { if wh.manager == nil { - w.WriteHeader(http.StatusNotImplemented) - return - } - - if !IsTLS(w, r) { + w.WriteHeader(http.StatusInternalServerError) return } @@ -100,10 +96,6 @@ func (wh *webHandler) event(w http.ResponseWriter, r *http.Request) { } func (wh *webHandler) info(w http.ResponseWriter, r *http.Request) { - if !IsTLS(w, r) { - return - } - if r.Method != http.MethodGet { w.WriteHeader(http.StatusNotImplemented) return diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index 15c8c26..1d4250e 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -14,6 +14,6 @@ func newAPIService() *apiService { return &apiService{} } -func (o *apiService) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { +func (o *apiService) MirrorRepository(_ context.Context, _ string, _ int) (err error) { return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) } From 7b00e6d6e1d6c7250bfbf79c98e7e0c704571121 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 15 Dec 2023 17:12:00 -0500 Subject: [PATCH 16/30] more fixes for linter adds ostree helm chart regens ostree api for future work --- .github/workflows/release.yml | 16 +- README.md | 7 + charts/beskar-ostree/.helmignore | 23 ++ charts/beskar-ostree/Chart.yaml | 12 ++ charts/beskar-ostree/LICENSE | 202 ++++++++++++++++++ charts/beskar-ostree/templates/NOTES.txt | 0 charts/beskar-ostree/templates/_helpers.tpl | 150 +++++++++++++ charts/beskar-ostree/templates/configmap.yaml | 10 + charts/beskar-ostree/templates/hpa.yaml | 28 +++ charts/beskar-ostree/templates/pvc.yaml | 26 +++ charts/beskar-ostree/templates/role.yaml | 26 +++ charts/beskar-ostree/templates/secret.yaml | 24 +++ charts/beskar-ostree/templates/service.yaml | 59 +++++ .../templates/serviceaccount.yaml | 14 ++ .../beskar-ostree/templates/statefulset.yaml | 75 +++++++ charts/beskar-ostree/values.yaml | 130 +++++++++++ cmd/beskar-ostree/main.go | 3 + cmd/beskarctl/ctl/error.go | 3 + cmd/beskarctl/ctl/helpers.go | 3 + cmd/beskarctl/ctl/root.go | 3 + cmd/beskarctl/ostree/push.go | 3 + cmd/beskarctl/ostree/root.go | 3 + cmd/beskarctl/static/push.go | 3 + cmd/beskarctl/static/root.go | 3 + cmd/beskarctl/yum/push.go | 3 + cmd/beskarctl/yum/pushmetadata.go | 3 + cmd/beskarctl/yum/root.go | 3 + internal/plugins/ostree/api.go | 9 +- .../ostree/pkg/config/beskar-ostree.go | 3 + internal/plugins/ostree/plugin.go | 3 + pkg/plugins/ostree/api/v1/api.go | 9 +- pkg/plugins/ostree/api/v1/endpoint.go | 6 +- pkg/plugins/ostree/api/v1/http_client.go | 8 +- pkg/plugins/ostree/api/v1/oas2.go | 4 +- 34 files changed, 865 insertions(+), 12 deletions(-) create mode 100644 charts/beskar-ostree/.helmignore create mode 100644 charts/beskar-ostree/Chart.yaml create mode 100644 charts/beskar-ostree/LICENSE create mode 100644 charts/beskar-ostree/templates/NOTES.txt create mode 100644 charts/beskar-ostree/templates/_helpers.tpl create mode 100644 charts/beskar-ostree/templates/configmap.yaml create mode 100644 charts/beskar-ostree/templates/hpa.yaml create mode 100644 charts/beskar-ostree/templates/pvc.yaml create mode 100644 charts/beskar-ostree/templates/role.yaml create mode 100644 charts/beskar-ostree/templates/secret.yaml create mode 100644 charts/beskar-ostree/templates/service.yaml create mode 100644 charts/beskar-ostree/templates/serviceaccount.yaml create mode 100644 charts/beskar-ostree/templates/statefulset.yaml create mode 100644 charts/beskar-ostree/values.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40531f2..c102f03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,4 +56,18 @@ jobs: - name: Release beskar-static image run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" - name: Release beskar-static helm chart - run: ./scripts/mage ci:chart ghcr.io/ctrliq/helm-charts/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file + run: ./scripts/mage ci:chart ghcr.io/ctrliq/helm-charts/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" + + release-beskar-ostree: + name: release beskar-ostree + needs: lint + runs-on: ubuntu-22.04 + steps: + - uses: actions/setup-go@v3 + with: + go-version: '1.20' + - uses: actions/checkout@v3 + - name: Release beskar-ostree image + run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-ostree:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" + - name: Release beskar-ostree helm chart + run: ./scripts/mage ci:chart ghcr.io/ctrliq/helm-charts/beskar-ostree:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/README.md b/README.md index 0b8722b..29b8aec 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ It's designed to support various artifacts and expose them through dedicated plu * Modular/Extensible via [plugins](docs/plugins.md) * Support for YUM repositories (beskar-yum) * Support for static file repositories (beskar-static) +* Support for OSTree repositories ([beskar-ostree](internal/plugins/ostree/README.md)) ### Docker images @@ -41,6 +42,12 @@ For beskar-static helm chart: helm pull oci://ghcr.io/ctrliq/helm-charts/beskar-static --version 0.0.1 --untar ``` +For beskar-static helm chart: + +``` +helm pull oci://ghcr.io/ctrliq/helm-charts/beskar-ostree --version 0.0.1 --untar +``` + ### Compilation Binaries are not provided as part of releases, you can compile it yourself by running: diff --git a/charts/beskar-ostree/.helmignore b/charts/beskar-ostree/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/beskar-ostree/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/beskar-ostree/Chart.yaml b/charts/beskar-ostree/Chart.yaml new file mode 100644 index 0000000..44625d7 --- /dev/null +++ b/charts/beskar-ostree/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +description: A Helm chart for Beskar OSTree Repository Plugin +name: beskar-ostree +version: 0.0.1 +appVersion: 0.0.1 +home: https://github.com/ctrliq/beskar +maintainers: +- email: dev@ciq.com + name: CtrlIQ Inc. + url: https://github.com/ctrliq/beskar +sources: +- https://github.com/ctrliq/beskar diff --git a/charts/beskar-ostree/LICENSE b/charts/beskar-ostree/LICENSE new file mode 100644 index 0000000..393b7a3 --- /dev/null +++ b/charts/beskar-ostree/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright The Helm Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/charts/beskar-ostree/templates/NOTES.txt b/charts/beskar-ostree/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/charts/beskar-ostree/templates/_helpers.tpl b/charts/beskar-ostree/templates/_helpers.tpl new file mode 100644 index 0000000..e827ba5 --- /dev/null +++ b/charts/beskar-ostree/templates/_helpers.tpl @@ -0,0 +1,150 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "beskar-ostree.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "beskar-ostree.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "beskar-ostree.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "beskar-ostree.labels" -}} +helm.sh/chart: {{ include "beskar-ostree.chart" . }} +{{ include "beskar-ostree.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "beskar-ostree.selectorLabels" -}} +app.kubernetes.io/name: {{ include "beskar-ostree.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "beskar-ostree.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "beskar-ostree.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "beskar-ostree.envs" -}} +- name: BESKAROSTREE_GOSSIP_KEY + valueFrom: + secretKeyRef: + name: beskar-gossip-secret + key: gossipKey +{{- if eq .Values.configData.storage.driver "filesystem" }} +- name: BESKAROSTREE_STORAGE_FILESYSTEM_DIRECTORY + value: {{ .Values.configData.storage.filesystem.directory }} +{{- else if eq .Values.configData.storage.driver "azure" }} +- name: BESKAROSTREE_STORAGE_AZURE_ACCOUNTNAME + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: azureAccountName +- name: BESKAROSTREE_STORAGE_AZURE_ACCOUNTKEY + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: azureAccountKey +{{- else if eq .Values.configData.storage.driver "s3" }} + {{- if and .Values.secrets.s3.secretKey .Values.secrets.s3.accessKey }} +- name: BESKAROSTREE_STORAGE_S3_ACCESSKEYID + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: s3AccessKey +- name: BESKAROSTREE_STORAGE_S3_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: s3SecretKey + {{- end }} +{{- else if eq .Values.configData.storage.driver "gcs" }} +- name: BESKAROSTREE_STORAGE_GCS_KEYFILE + value: /etc/gcs-keyfile +{{- end -}} + +{{- with .Values.extraEnvVars }} +{{ toYaml . }} +{{- end -}} + +{{- end -}} + +{{- define "beskar-ostree.volumeMounts" -}} +- name: config + mountPath: "/etc/beskar" + +{{- if eq .Values.configData.storage.driver "filesystem" }} +- name: data + mountPath: {{ .Values.configData.storage.filesystem.directory }} +{{- else if eq .Values.configData.storage.driver "gcs" }} +- name: gcs + mountPath: "/etc/gcs-keyfile" + subPath: gcsKeyfile + readOnly: true +{{- end }} + +{{- with .Values.extraVolumeMounts }} +{{ toYaml . }} +{{- end }} + +{{- end -}} + +{{- define "beskar-ostree.volumes" -}} +- name: config + configMap: + name: {{ template "beskar-ostree.fullname" . }}-config + +{{- if eq .Values.configData.storage.driver "filesystem" }} +- name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "beskar-ostree.fullname" . }}{{- end }} + {{- else }} + emptyDir: {} + {{- end -}} +{{- else if eq .Values.configData.storage.driver "gcs" }} +- name: gcs + secret: + secretName: {{ template "beskar-ostree.fullname" . }}-secret +{{- end }} + +{{- with .Values.extraVolumes }} +{{ toYaml . }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/configmap.yaml b/charts/beskar-ostree/templates/configmap.yaml new file mode 100644 index 0000000..3426be5 --- /dev/null +++ b/charts/beskar-ostree/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "beskar-ostree.fullname" . }}-config + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +data: + beskar-ostree.yaml: |- +{{ toYaml .Values.configData | indent 4 }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/hpa.yaml b/charts/beskar-ostree/templates/hpa.yaml new file mode 100644 index 0000000..022bfd1 --- /dev/null +++ b/charts/beskar-ostree/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "beskar-ostree.fullname" . }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "beskar-ostree.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/beskar-ostree/templates/pvc.yaml b/charts/beskar-ostree/templates/pvc.yaml new file mode 100644 index 0000000..77c0621 --- /dev/null +++ b/charts/beskar-ostree/templates/pvc.yaml @@ -0,0 +1,26 @@ +{{- if .Values.persistence.enabled }} +{{- if not .Values.persistence.existingClaim -}} +{{- if eq .Values.configData.storage.driver "filesystem" }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "beskar-ostree.fullname" . }} + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/beskar-ostree/templates/role.yaml b/charts/beskar-ostree/templates/role.yaml new file mode 100644 index 0000000..5e2a930 --- /dev/null +++ b/charts/beskar-ostree/templates/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "beskar-ostree.fullname" . }} +rules: + - apiGroups: + - '' + resources: + - endpoints + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "beskar-ostree.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name | default (include "beskar-ostree.fullname" .) }} + apiGroup: "" + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ template "beskar-ostree.fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/charts/beskar-ostree/templates/secret.yaml b/charts/beskar-ostree/templates/secret.yaml new file mode 100644 index 0000000..fc5e2c8 --- /dev/null +++ b/charts/beskar-ostree/templates/secret.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "beskar-ostree.fullname" . }}-secret + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +type: Opaque +data: + {{- if eq .Values.configData.storage.driver "azure" }} + {{- if and .Values.secrets.azure.accountName .Values.secrets.azure.accountKey .Values.secrets.azure.container }} + azureAccountName: {{ .Values.secrets.azure.accountName | b64enc | quote }} + azureAccountKey: {{ .Values.secrets.azure.accountKey | b64enc | quote }} + {{- end }} + {{- else if eq .Values.configData.storage.driver "s3" }} + {{- if and .Values.secrets.s3.secretKey .Values.secrets.s3.accessKey }} + s3AccessKey: {{ .Values.secrets.s3.accessKey | b64enc | quote }} + s3SecretKey: {{ .Values.secrets.s3.secretKey | b64enc | quote }} + {{- end }} + {{- else if eq .Values.configData.storage.driver "gcs" }} + gcsKeyfile: {{ .Values.secrets.gcs.keyfile | b64enc | quote }} + {{- end }} + registryUsername: {{ .Values.secrets.registry.username | b64enc | quote }} + registryPassword: {{ .Values.secrets.registry.password | b64enc | quote }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/service.yaml b/charts/beskar-ostree/templates/service.yaml new file mode 100644 index 0000000..378d4da --- /dev/null +++ b/charts/beskar-ostree/templates/service.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "beskar-ostree.fullname" . }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.service.type }} +{{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: + {{ toYaml .Values.service.sessionAffinityConfig | nindent 4 }} + {{- end -}} +{{- end }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: http + selector: + {{- include "beskar-ostree.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "beskar-ostree.fullname" . }}-gossip + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} + go.ciq.dev/beskar-gossip: "true" +{{- if .Values.gossip.annotations }} + annotations: +{{ toYaml .Values.gossip.annotations | indent 4 }} +{{- end }} +spec: + type: ClusterIP +{{- if .Values.gossip.sessionAffinity }} + sessionAffinity: {{ .Values.gossip.sessionAffinity }} + {{- if .Values.gossip.sessionAffinityConfig }} + sessionAffinityConfig: + {{ toYaml .Values.gossip.sessionAffinityConfig | nindent 4 }} + {{- end -}} +{{- end }} + ports: + - port: {{ .Values.gossip.port }} + protocol: TCP + name: gossip-tcp + targetPort: {{ .Values.gossip.port }} + - port: {{ .Values.gossip.port }} + protocol: UDP + name: gossip-udp + targetPort: {{ .Values.gossip.port }} + selector: + {{- include "beskar-ostree.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/serviceaccount.yaml b/charts/beskar-ostree/templates/serviceaccount.yaml new file mode 100644 index 0000000..e6ad1a9 --- /dev/null +++ b/charts/beskar-ostree/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: +{{- if .Values.serviceAccount.name }} + name: {{ .Values.serviceAccount.name }} +{{- else }} + name: {{ include "beskar-ostree.fullname" . }} +{{- end }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/statefulset.yaml b/charts/beskar-ostree/templates/statefulset.yaml new file mode 100644 index 0000000..36947fb --- /dev/null +++ b/charts/beskar-ostree/templates/statefulset.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "beskar-ostree.fullname" . }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +spec: + serviceName: {{ .Chart.Name }} + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "beskar-ostree.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "beskar-ostree.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{ toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.name | default (include "beskar-ostree.fullname" .) }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /usr/bin/beskar-ostree + - -config-dir=/etc/beskar + ports: + - containerPort: {{ .Values.service.port }} + name: http + protocol: TCP + - containerPort: {{ .Values.gossip.port }} + name: gossip-tcp + protocol: TCP + - containerPort: {{ .Values.gossip.port }} + name: gossip-udp + protocol: UDP + livenessProbe: + tcpSocket: + port: http + readinessProbe: + tcpSocket: + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: {{ include "beskar-ostree.envs" . | nindent 12 }} + volumeMounts: {{ include "beskar-ostree.volumeMounts" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: {{ include "beskar-ostree.volumes" . | nindent 8 }} diff --git a/charts/beskar-ostree/values.yaml b/charts/beskar-ostree/values.yaml new file mode 100644 index 0000000..223c957 --- /dev/null +++ b/charts/beskar-ostree/values.yaml @@ -0,0 +1,130 @@ +# Default values for beskar-ostree. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/ctrliq/beskar-ostree + # Overrides the image tag whose default is the chart appVersion. + tag: 0.0.1 + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # sessionAffinity: None + # sessionAffinityConfig: {} + type: ClusterIP + port: 5200 + annotations: {} + +gossip: + # sessionAffinity: None + # sessionAffinityConfig: {} + port: 5201 + annotations: {} + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +extraVolumeMounts: [] + +extraVolumes: [] + +extraEnvVars: [] + +persistence: + accessMode: 'ReadWriteOnce' + enabled: false + size: 10Gi + # storageClass: '-' + +secrets: + registry: + username: beskar + password: beskar + + s3: + accessKey: "" + secretKey: "" + + gcs: + keyfile: "" + + azure: + accountName: "" + # base64_encoded_account_key + accountKey: "" + +configData: + version: "1.0" + addr: :5200 + profiling: false + datadir: /tmp/beskar-ostree + + log: + level: debug + format: json + + gossip: + addr: :5201 + + storage: + driver: filesystem + prefix: "" + s3: + endpoint: 127.0.0.1:9100 + bucket: beskar-ostree + region: us-east-1 + filesystem: + directory: /tmp/beskar-ostree + gcs: + bucket: beskar-ostree + azure: + container: beskar-ostree \ No newline at end of file diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go index 4286ca4..477bcda 100644 --- a/cmd/beskar-ostree/main.go +++ b/cmd/beskar-ostree/main.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package main import ( diff --git a/cmd/beskarctl/ctl/error.go b/cmd/beskarctl/ctl/error.go index cce9285..cd98fd2 100644 --- a/cmd/beskarctl/ctl/error.go +++ b/cmd/beskarctl/ctl/error.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ctl import "fmt" diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go index 7a34a53..eb86300 100644 --- a/cmd/beskarctl/ctl/helpers.go +++ b/cmd/beskarctl/ctl/helpers.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ctl import ( diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go index a44338d..d0afdf1 100644 --- a/cmd/beskarctl/ctl/root.go +++ b/cmd/beskarctl/ctl/root.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ctl import ( diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go index 191b462..b063eb5 100644 --- a/cmd/beskarctl/ostree/push.go +++ b/cmd/beskarctl/ostree/push.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostree import ( diff --git a/cmd/beskarctl/ostree/root.go b/cmd/beskarctl/ostree/root.go index 570c1cf..f6934d1 100644 --- a/cmd/beskarctl/ostree/root.go +++ b/cmd/beskarctl/ostree/root.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostree import ( diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go index 4848f76..616a61b 100644 --- a/cmd/beskarctl/static/push.go +++ b/cmd/beskarctl/static/push.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package static import ( diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go index 8eb377f..8bcae1c 100644 --- a/cmd/beskarctl/static/root.go +++ b/cmd/beskarctl/static/root.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package static import ( diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go index d211154..a3cb9c0 100644 --- a/cmd/beskarctl/yum/push.go +++ b/cmd/beskarctl/yum/push.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package yum import ( diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go index 043dbdc..09d1023 100644 --- a/cmd/beskarctl/yum/pushmetadata.go +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package yum import ( diff --git a/cmd/beskarctl/yum/root.go b/cmd/beskarctl/yum/root.go index 71a8912..29fe668 100644 --- a/cmd/beskarctl/yum/root.go +++ b/cmd/beskarctl/yum/root.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package yum import ( diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index 1d4250e..1b5cf9e 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -1,8 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostree import ( "context" "errors" + "fmt" + + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" @@ -14,6 +20,7 @@ func newAPIService() *apiService { return &apiService{} } -func (o *apiService) MirrorRepository(_ context.Context, _ string, _ int) (err error) { +func (o *apiService) MirrorRepository(_ context.Context, repository string, properties *apiv1.OSTreeRepositoryProperties) (err error) { + fmt.Printf("Repo: %s,\nProperties: %v\n", repository, properties) return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) } diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go index d8743ca..7663530 100644 --- a/internal/plugins/ostree/pkg/config/beskar-ostree.go +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package config import ( diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go index 2f7ed05..1af8d3c 100644 --- a/internal/plugins/ostree/plugin.go +++ b/internal/plugins/ostree/plugin.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostree import ( diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 8d9296c..7f9d224 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -24,6 +24,13 @@ type Page struct { Token string } +type OSTreeRepositoryProperties struct { + RemoteURL string `json:"remote_url"` + Branch string `json:"branch"` + Depth int `json:"depth"` + Mirror bool `json:"mirror"` +} + // OSTree is used for managing ostree repositories. // This is the API documentation of OSTree. // @@ -36,5 +43,5 @@ type OSTree interface { // Mirror an ostree repository. //kun:op POST /repository/mirror //kun:success statusCode=200 - MirrorRepository(ctx context.Context, repository string, depth int) (err error) + MirrorRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) } diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go index 33252bf..99c6dd7 100644 --- a/pkg/plugins/ostree/api/v1/endpoint.go +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -12,8 +12,8 @@ import ( ) type MirrorRepositoryRequest struct { - Repository string `json:"repository"` - Depth int `json:"depth"` + Repository string `json:"repository"` + Properties *OSTreeRepositoryProperties `json:"properties"` } // ValidateMirrorRepositoryRequest creates a validator for MirrorRepositoryRequest. @@ -40,7 +40,7 @@ func MakeEndpointOfMirrorRepository(s OSTree) endpoint.Endpoint { err := s.MirrorRepository( ctx, req.Repository, - req.Depth, + req.Properties, ) return &MirrorRepositoryResponse{ Err: err, diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go index bda1384..2d50a80 100644 --- a/pkg/plugins/ostree/api/v1/http_client.go +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -34,7 +34,7 @@ func NewHTTPClient(codecs httpcodec.Codecs, httpClient *http.Client, baseURL str }, nil } -func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, depth int) (err error) { +func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) { codec := c.codecs.EncodeDecoder("MirrorRepository") path := "/repository/mirror" @@ -45,11 +45,11 @@ func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, de } reqBody := struct { - Repository string `json:"repository"` - Depth int `json:"depth"` + Repository string `json:"repository"` + Properties *OSTreeRepositoryProperties `json:"properties"` }{ Repository: repository, - Depth: depth, + Properties: properties, } reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) if err != nil { diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go index b3b5f4b..369a082 100644 --- a/pkg/plugins/ostree/api/v1/oas2.go +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -54,8 +54,8 @@ func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { defs := make(map[string]oas2.Definition) oas2.AddDefinition(defs, "MirrorRepositoryRequestBody", reflect.ValueOf(&struct { - Repository string `json:"repository"` - Depth int `json:"depth"` + Repository string `json:"repository"` + Properties *OSTreeRepositoryProperties `json:"properties"` }{})) oas2.AddResponseDefinitions(defs, schema, "MirrorRepository", 200, (&MirrorRepositoryResponse{}).Body()) From 70da0f2e16772c628a24c328723caa708bf71760 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 11 Jan 2024 12:45:46 -0500 Subject: [PATCH 17/30] adds libostree testdata to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d705dbd..faae1fa 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ vendor go.work.sum .idea + +internal/plugins/ostree/pkg/libostree/testdata \ No newline at end of file From aac24f23d8bd862203dd6e9829d0b1f9b15a2f89 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 11 Jan 2024 12:47:14 -0500 Subject: [PATCH 18/30] libostree finished ostree plugin finished (needs testing) --- build/mage/build.go | 54 ++- build/mage/dockerfiles/ostree.dockerfile | 5 + cmd/beskar-ostree/main.go | 3 +- cmd/beskar-static/main.go | 3 +- cmd/beskar-yum/main.go | 3 +- cmd/beskarctl/ostree/push.go | 81 +---- docs/plugins.md | 171 +++++++++- internal/pkg/pluginsrv/service.go | 8 +- internal/pkg/pluginsrv/webhandler.go | 8 +- internal/pkg/repository/handler.go | 12 + internal/pkg/repository/manager.go | 29 +- internal/plugins/ostree/api.go | 55 ++- .../plugins/ostree/pkg/libostree/README.md | 22 ++ .../plugins/ostree/pkg/libostree/errors.go | 11 + .../ostree/pkg/libostree/generate-testdata.sh | 26 ++ .../ostree/pkg/libostree/glib_helpers.go | 21 ++ .../ostree/pkg/libostree/glib_helpers.go.h | 11 + .../plugins/ostree/pkg/libostree/options.go | 66 ++++ .../plugins/ostree/pkg/libostree/options.go.h | 14 + .../plugins/ostree/pkg/libostree/ostree.go | 1 + internal/plugins/ostree/pkg/libostree/pull.go | 200 +++++++++++ .../plugins/ostree/pkg/libostree/pull.go.h | 34 ++ .../plugins/ostree/pkg/libostree/pull_test.go | 164 +++++++++ internal/plugins/ostree/pkg/libostree/repo.go | 318 ++++++++++++++++++ .../ostree/pkg/ostreerepository/api.go | 215 ++++++++++++ .../ostree/pkg/ostreerepository/local.go | 86 +++++ .../pkg/ostreerepository/ostreerepository.go | 120 +++++++ .../ostree/pkg/ostreerepository/sync.go | 35 ++ internal/plugins/ostree/plugin.go | 97 +++--- internal/plugins/static/api.go | 53 +-- .../static/pkg/staticrepository/api.go | 14 +- .../static/pkg/staticrepository/handler.go | 2 +- internal/plugins/static/plugin.go | 19 +- internal/plugins/yum/api.go | 97 +----- internal/plugins/yum/pkg/yumrepository/api.go | 20 +- .../plugins/yum/pkg/yumrepository/handler.go | 2 +- internal/plugins/yum/plugin.go | 8 +- pkg/orasostree/ostree.go | 20 +- pkg/orasostree/push.go | 85 +++++ pkg/plugins/ostree/api/v1/api.go | 63 +++- pkg/plugins/ostree/api/v1/endpoint.go | 170 +++++++++- pkg/plugins/ostree/api/v1/http.go | 134 +++++++- pkg/plugins/ostree/api/v1/http_client.go | 203 ++++++++++- pkg/plugins/ostree/api/v1/oas2.go | 85 ++++- pkg/utils/time.go | 12 + 45 files changed, 2470 insertions(+), 390 deletions(-) create mode 100644 build/mage/dockerfiles/ostree.dockerfile create mode 100644 internal/plugins/ostree/pkg/libostree/README.md create mode 100644 internal/plugins/ostree/pkg/libostree/errors.go create mode 100755 internal/plugins/ostree/pkg/libostree/generate-testdata.sh create mode 100644 internal/plugins/ostree/pkg/libostree/glib_helpers.go create mode 100644 internal/plugins/ostree/pkg/libostree/glib_helpers.go.h create mode 100644 internal/plugins/ostree/pkg/libostree/options.go create mode 100644 internal/plugins/ostree/pkg/libostree/options.go.h create mode 100644 internal/plugins/ostree/pkg/libostree/ostree.go create mode 100644 internal/plugins/ostree/pkg/libostree/pull.go create mode 100644 internal/plugins/ostree/pkg/libostree/pull.go.h create mode 100644 internal/plugins/ostree/pkg/libostree/pull_test.go create mode 100644 internal/plugins/ostree/pkg/libostree/repo.go create mode 100644 internal/plugins/ostree/pkg/ostreerepository/api.go create mode 100644 internal/plugins/ostree/pkg/ostreerepository/local.go create mode 100644 internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go create mode 100644 internal/plugins/ostree/pkg/ostreerepository/sync.go create mode 100644 pkg/orasostree/push.go create mode 100644 pkg/utils/time.go diff --git a/build/mage/build.go b/build/mage/build.go index e5019e7..6f94e4e 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -55,14 +55,16 @@ type binaryConfig struct { buildTags []string baseImage string integrationTest *integrationTest + buildEnv map[string]string + buildExecStmts [][]string } const ( - beskarBinary = "beskar" - beskarctlBinary = "beskarctl" - beskarYUMBinary = "beskar-yum" - beskarStaticBinary = "beskar-static" - beskarOSTreeBinary = "beskar-ostree" + BeskarBinary = "beskar" + BeskarctlBinary = "beskarctl" + BeskarYUMBinary = "beskar-yum" + BeskarStaticBinary = "beskar-static" + BeskarOSTreeBinary = "beskar-ostree" ) var binaries = map[string]binaryConfig{ @@ -131,7 +133,7 @@ var binaries = map[string]binaryConfig{ }, }, }, - beskarOSTreeBinary: { + BeskarOSTreeBinary: { configFiles: map[string]string{ "internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml": "/etc/beskar/beskar-ostree.yaml", }, @@ -140,8 +142,30 @@ var binaries = map[string]binaryConfig{ filename: "api.go", interfaceName: "OSTree", }, - useProto: true, - baseImage: "alpine:3.17", + useProto: true, + execStmts: [][]string{ + { + "apk", "add", "ostree", "ostree-dev", + }, + }, + buildExecStmts: [][]string{ + { + "apk", "add", "build-base", + }, + { + "apk", "add", "ostree", "ostree-dev", + }, + }, + buildEnv: map[string]string{ + "CGO_ENABLED": "1", + }, + excludedPlatforms: map[dagger.Platform]struct{}{ + "linux/arm64": {}, + "linux/s390x": {}, + "linux/ppc64le": {}, + "linux/arm/v6": {}, + "linux/arm/v7": {}, + }, }, } @@ -181,9 +205,9 @@ func (b Build) Beskarctl(ctx context.Context) error { func (b Build) Plugins(ctx context.Context) { mg.CtxDeps( ctx, - mg.F(b.Plugin, beskarYUMBinary), - mg.F(b.Plugin, beskarStaticBinary), - mg.F(b.Plugin, beskarOSTreeBinary), + mg.F(b.Plugin, BeskarYUMBinary), + mg.F(b.Plugin, BeskarStaticBinary), + mg.F(b.Plugin, BeskarOSTreeBinary), ) } @@ -288,6 +312,14 @@ func (b Build) build(ctx context.Context, name string) error { golang = golang.WithEnvVariable(key, value) } + for key, value := range binaryConfig.buildEnv { + golang = golang.WithEnvVariable(key, value) + } + + for _, execStmt := range binaryConfig.buildExecStmts { + golang = golang.WithExec(execStmt) + } + path := filepath.Join("/output", binary) inputCmd := filepath.Join("cmd", name) diff --git a/build/mage/dockerfiles/ostree.dockerfile b/build/mage/dockerfiles/ostree.dockerfile new file mode 100644 index 0000000..2ceeb5f --- /dev/null +++ b/build/mage/dockerfiles/ostree.dockerfile @@ -0,0 +1,5 @@ +FROM rockylinux:8-minimal as Builder + +RUN microdnf update && \ + microdnf -y install ostree ostree-devel + diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go index 477bcda..4ae53d4 100644 --- a/cmd/beskar-ostree/main.go +++ b/cmd/beskar-ostree/main.go @@ -6,6 +6,7 @@ package main import ( "flag" "fmt" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/ostreerepository" "log" "net" "os" @@ -50,7 +51,7 @@ func serve(beskarOSTreeCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve(ln, plugin) + errCh <- pluginsrv.Serve[*ostreerepository.Handler](ln, plugin) }() return wait(false) diff --git a/cmd/beskar-static/main.go b/cmd/beskar-static/main.go index 9f68029..9f8e7f4 100644 --- a/cmd/beskar-static/main.go +++ b/cmd/beskar-static/main.go @@ -6,6 +6,7 @@ package main import ( "flag" "fmt" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "log" "net" "os" @@ -46,7 +47,7 @@ func serve(beskarStaticCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve(ln, plugin) + errCh <- pluginsrv.Serve[*staticrepository.Handler](ln, plugin) }() return wait(false) diff --git a/cmd/beskar-yum/main.go b/cmd/beskar-yum/main.go index 089a666..f4b5a6c 100644 --- a/cmd/beskar-yum/main.go +++ b/cmd/beskar-yum/main.go @@ -6,6 +6,7 @@ package main import ( "flag" "fmt" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "log" "net" "os" @@ -46,7 +47,7 @@ func serve(beskarYumCmd *flag.FlagSet) error { } go func() { - errCh <- pluginsrv.Serve(ln, plugin) + errCh <- pluginsrv.Serve[*yumrepository.Handler](ln, plugin) }() return wait(false) diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/push.go index b063eb5..2006703 100644 --- a/cmd/beskarctl/ostree/push.go +++ b/cmd/beskarctl/ostree/push.go @@ -5,19 +5,10 @@ package ostree import ( "context" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/cobra" "go.ciq.dev/beskar/cmd/beskarctl/ctl" - "go.ciq.dev/beskar/pkg/oras" "go.ciq.dev/beskar/pkg/orasostree" - "golang.org/x/sync/errgroup" ) var ( @@ -31,7 +22,7 @@ var ( return ctl.Err("a directory must be specified") } - if err := pushOSTreeRepository(dir, ctl.Repo(), ctl.Registry()); err != nil { + if err := orasostree.PushOSTreeRepository(context.Background(), dir, ctl.Repo(), jobCount, name.WithDefaultRegistry(ctl.Registry())); err != nil { return ctl.Errf("while pushing ostree repository: %s", err) } return nil @@ -50,73 +41,3 @@ func PushCmd() *cobra.Command { ) return pushCmd } - -// pushOSTreeRepository walks a local ostree repository and pushes each file to the given registry. -// dir is the root directory of the ostree repository, i.e., the directory containing the summary file. -// repo is the name of the ostree repository. -// registry is the registry to push to. -func pushOSTreeRepository(dir, repo, registry string) error { - // Prove that we were given the root directory of an ostree repository - // by checking for the existence of the summary file. - fileInfo, err := os.Stat(filepath.Join(dir, orasostree.KnownFileSummary)) - if os.IsNotExist(err) || fileInfo.IsDir() { - return fmt.Errorf("%s file not found in %s", orasostree.KnownFileSummary, dir) - } else if err != nil { - return fmt.Errorf("error accessing %s in %s: %w", orasostree.KnownFileSummary, dir, err) - } - - // Create a worker pool to push each file in the repository concurrently. - // ctx will be cancelled on error, and the error will be returned. - eg, ctx := errgroup.WithContext(context.Background()) - eg.SetLimit(jobCount) - - // Walk the directory tree, skipping directories and pushing each file. - if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - // If there was an error with the file, return it. - if err != nil { - return fmt.Errorf("while walking %s: %w", path, err) - } - - // Skip directories. - if d.IsDir() { - return nil - } - - if ctx.Err() != nil { - // Skip remaining files because our context has been cancelled. - // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). - // This is because we would never be able to handle an error returned from the last job. - return filepath.SkipAll - } - - eg.Go(func() error { - if err := push(dir, path, repo, registry); err != nil { - return fmt.Errorf("while pushing %s: %w", path, err) - } - return nil - }) - - return nil - }); err != nil { - // We should only receive here if filepath.WalkDir() returns an error. - // Push errors are handled below. - return fmt.Errorf("while walking %s: %w", dir, err) - } - - // Wait for all workers to finish. - // If any worker returns an error, eg.Wait() will return that error. - return eg.Wait() -} - -func push(repoRootDir, path, repo, registry string) error { - pusher, err := orasostree.NewOSTreePusher(repoRootDir, path, repo, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating OSTree pusher: %w", err) - } - - path = strings.TrimPrefix(path, repoRootDir) - path = strings.TrimPrefix(path, "/") - fmt.Printf("Pushing %s to %s\n", path, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) -} diff --git a/docs/plugins.md b/docs/plugins.md index 18c9952..322cdb8 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -1,17 +1,166 @@ # Beskar Plugins +A Beskar plugin is a binary that is deployed alongside Beskar and is responsible for managing a specific type of +artifact. For example, the `yum` plugin is responsible for managing RPMs. In it's very basic form a plugin is responsible +for mapping an incoming request to an artifact in the registry. Plugins may contain additional logic to support other actions +such as uploading, deleting, etc. For example, the `yum` plugin supports mirroring a remote repository. -TODOS: -- [ ] Generate with Kun - See build/mage/build.go:86 for autogeneration -- [ ] Map artifact paths with a separator like `/artifacts/ostree/{repo_name}/separtor/path/to/{artifact_name}`. This will be translated to `/v2/%s/blobs/sha256:%s` -- [ ] Create router.rego & data.json so that Beskar knows how to route requests to plugin server(s) -- [ ] mediatypes may be needed for each file type - - [ ] `application/vnd.ciq.ostree.file.v1.file` - - [ ] `application/vnd.ciq.ostree.summary.v1.summary` +## How To Use This Document +This document is intended to be a guide for writing a Beskar plugin. It is not intended to be a complete reference. Use +the information provided here to get started and then refer to the code for more details. `internal/plugins/static` is a +simple plugin to use as a reference. It is recommended that you read through the code and then use it as a starting point +for your own plugin. +## Plugin Architecture +A Beskar plugin is written in Go and will be deployed so that it can be accessed by Beskar. There are a few mechanisms that +Beskar uses to discover and communicate with plugins. The first of which is a gossip protocol that is used to discover +plugins. The second is the Events API that is used to keep plugins in sync with Beskar, such as when an artifact is uploaded +or deleted. The third is the plugin service that is used to serve the plugin's API. We will cover these in more detail below, +but luckily Beskar provides a few interfaces, as well as a series of helper methods, to make writing a plugin easier. +### Plugin Discovery and API Request Routing +Beskar uses [a gossip protocol](https://github.com/hashicorp/memberlist) to discover plugins. Early in its startup process a plugin will register itself +with a known peer, generally one of the main Beskar instances, and the plugin's info will be shared with the rest of the cluster. +This info includes the plugin's name, version, and the address of the plugin's API. Beskar will then use this info to route +requests to the plugin's API using a [Rego policy](https://www.openpolicyagent.org/) provided by the plugin. -See internal/plugins/yum/embedded/router.rego for example -/artifacts/ostree/{repo_name}/separtor/path/to/{artifact_name} +**Note that you do not need to do anything special to register your plugin. Beskar will handle this for you.** All you need +to do is provide the plugin's info, which includes the rego policy, and a router. We will cover this in more detail later. + +### Repository Handler +In some cases your plugin may need to be informed when an artifact is uploaded or deleted. This is accomplished by +implementing the [Handler interface](../internal/pkg/repository/handler.go). The object you implement will be used to receive events from Beskar and will +enable your plugin to keep its internal state up to date. + +#### Implementation Notes +When implementing your `repository.Handler` there are a few things to keep in mind. + +First, the `QueueEvent()` method is not intended to be used to perform long-running operations. Instead, you should +queue the event for processing in another goroutine. The static plugin provides a good example of this by spinning +up a goroutine in its `Start()` that listens for events and processes them, while the `QueueEvent()` method simply queues +the event for processing in the run loop. + +Second, Beskar provides a [RepoHandler struct](../internal/pkg/repository/handler.go) that partially implements the +`Handler` interface and provides some helper methods that reduce your implementation burden to only `Start()` and +`QueueEvent()`. This is exemplified below as well as in the [Static plugin](../internal/plugins/static/pkg/staticrepository/handler.go). + +Third, we recommend that you create a constructor for your handler that conforms to the `repository.HandlerFactory` type. +This will come in handy later when creating the plugin service. + +#### Example Implementation of `repository.Handler` +``` + +type ExampleHandler struct { + *repository.RepoHandler +} + +func NewExampleHandler(*slog.Logger, repoHandler *repository.RepoHandler) *ExampleHandler { + return &ExampleHandler{ + RepoHandler: repoHandler, + } +} + +func (h *ExampleHandler) Start(ctx context.Context) { + // Process stored events + // Start goroutine to dequeue and process new events +} + +func (h *ExampleHandler) QueueEvent(event *eventv1.EventPayload, store bool) error { + // Store event if store is true + // Queue event for processing + return nil +} +``` + +#### Plugins without internal state +Not all plugins will have internal state, for example, the [Static plugin](../internal/plugins/ostree/plugin.go). simply +maps an incoming request to an artifact in the registry. In these cases, it is not required to implement a +`repository.Handler`. You can simply return `nil` from the `RepositoryManager()` method of your plugin service and leave +your plugin's `Info.MediaTypes` empty. This will tell Beskar that your plugin does not need to receive events. More on +this in the next section. + + +### Plugin Service +The [Plugin Service](../internal/pkg/pluginsrv/service.go) is responsible for serving the plugin's API, registering your +`repository.Handler` and providing the info Beskar needs about your plugin. We recommend that your implementation of +`pluginsrv.Service` have a constructor that accepts a config object and returns a pointer to your service. For example: +``` + +//go:embed embedded/router.rego +var routerRego []byte + +//go:embed embedded/data.json +var routerData []byte + +const ( + // PluginName is the name of the plugin + PluginName = "example" +) + +type ExamplePlugin struct { + ctx context.Context + config pluginsrv.Config + + repositoryManager *repository.Manager + handlerParams *repository.HandlerParams +} + +type ExamplePluginConfig struct { + Gossip gossip.Config +} + +func NewExamplePlugin(ctx context.Context, exampleConfig ExamplePluginConfig) (*ExamplePlugin, error) { + config := pluginsrv.Config{} + + router := chi.NewRouter() + // for kubernetes probes + router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + config.Router = router + config.Gossip = exampleConfig.Gossip + config.Info = &pluginv1.Info{ + Name: PluginName, + Version: version.Semver, + Mediatypes: []string{ + "application/vnd.ciq.example.file.v1.config+json", + }, + Router: &pluginv1.Router{ + Rego: routerRego, + Data: routerData, + }, + } + + plugin := ExamplePlugin{ + ctx: ctx, + config: config, + } + + plugin.repositoryManager = repository.NewManager(plugin.handlerParams, NewExampleHandler) + + return &plugin, nil +} + +func (p *ExamplePlugin) Start(http.RoundTripper, *mtls.CAPEM, *gossip.BeskarMeta) error { + // Register handlers with p.config.Router + return nil +} + +func (p *ExamplePlugin) Context() context.Context { + return p.ctx +} + +func (p *ExamplePlugin) Config() Config { + return p.config +} + +func (p *ExamplePlugin) RepositoryManager() *repository.Manager { + return nil +} +``` + + +#### Your Plugin's API +The `Start(...)` method is called when the server is about to serve your plugin's api and is your chance to register your +plugin's handlers with the server. + +The `Config()` method is used to return your plugin's configuration. This is used by Beskar to generate the plugin's -/2/artifacts/ostree/{repo_name}/files:summary -/2/artifacts/ostree/{repo_name}/files:{sha256("/path/to/{artifact_name}")} \ No newline at end of file diff --git a/internal/pkg/pluginsrv/service.go b/internal/pkg/pluginsrv/service.go index 948df16..bdd8ab0 100644 --- a/internal/pkg/pluginsrv/service.go +++ b/internal/pkg/pluginsrv/service.go @@ -36,7 +36,7 @@ type Config struct { Info *pluginv1.Info } -type Service interface { +type Service[H repository.Handler] interface { // Start starts the service's HTTP server. Start(http.RoundTripper, *mtls.CAPEM, *gossip.BeskarMeta) error @@ -48,10 +48,10 @@ type Service interface { // RepositoryManager returns the service's repository manager. // For plugin's without a repository manager, this method should return nil. - RepositoryManager() *repository.Manager + RepositoryManager() *repository.Manager[H] } -func Serve(ln net.Listener, service Service) (errFn error) { +func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn error) { ctx := service.Context() errCh := make(chan error) @@ -132,7 +132,7 @@ func Serve(ln net.Listener, service Service) (errFn error) { case beskarMeta := <-beskarMetaCh: ticker.Stop() - wh := webHandler{ + wh := webHandler[H]{ pluginInfo: serviceConfig.Info, manager: repoManager, } diff --git a/internal/pkg/pluginsrv/webhandler.go b/internal/pkg/pluginsrv/webhandler.go index a81a5e7..b1f3ef0 100644 --- a/internal/pkg/pluginsrv/webhandler.go +++ b/internal/pkg/pluginsrv/webhandler.go @@ -16,9 +16,9 @@ import ( "google.golang.org/protobuf/proto" ) -type webHandler struct { +type webHandler[H repository.Handler] struct { pluginInfo *pluginv1.Info - manager *repository.Manager + manager *repository.Manager[H] } func IsTLS(w http.ResponseWriter, r *http.Request) bool { @@ -39,7 +39,7 @@ func IsTLSMiddleware(next http.Handler) http.Handler { }) } -func (wh *webHandler) event(w http.ResponseWriter, r *http.Request) { +func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { if wh.manager == nil { w.WriteHeader(http.StatusInternalServerError) return @@ -95,7 +95,7 @@ func (wh *webHandler) event(w http.ResponseWriter, r *http.Request) { } } -func (wh *webHandler) info(w http.ResponseWriter, r *http.Request) { +func (wh *webHandler[H]) info(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.WriteHeader(http.StatusNotImplemented) return diff --git a/internal/pkg/repository/handler.go b/internal/pkg/repository/handler.go index ff9b4c9..df076f4 100644 --- a/internal/pkg/repository/handler.go +++ b/internal/pkg/repository/handler.go @@ -6,9 +6,12 @@ package repository import ( "context" "errors" + "go.ciq.dev/beskar/internal/pkg/gossip" "io" + "net" "os" "path/filepath" + "strconv" "sync" "sync/atomic" "time" @@ -25,12 +28,21 @@ type HandlerParams struct { RemoteOptions []remote.Option NameOptions []name.Option remove func(string) + BeskarMeta *gossip.BeskarMeta } func (hp HandlerParams) Remove(repository string) { hp.remove(repository) } +func (hp HandlerParams) GetBeskarServiceHostPort() string { + return net.JoinHostPort(hp.BeskarMeta.Hostname, strconv.Itoa(int(hp.BeskarMeta.ServicePort))) +} + +func (hp HandlerParams) GetBeskarRegistryHostPort() string { + return net.JoinHostPort(hp.BeskarMeta.Hostname, strconv.Itoa(int(hp.BeskarMeta.RegistryPort))) +} + // Handler - Interface for handling events for a repository. type Handler interface { // QueueEvent - Called when a new event is received. If store is true, the event should be stored in the database. diff --git a/internal/pkg/repository/manager.go b/internal/pkg/repository/manager.go index 724cdf7..f726aa2 100644 --- a/internal/pkg/repository/manager.go +++ b/internal/pkg/repository/manager.go @@ -11,21 +11,20 @@ import ( "go.ciq.dev/beskar/internal/pkg/log" ) -type HandlerMap = map[string]Handler - -type HandlerFactory = func(*slog.Logger, *RepoHandler) Handler - -type Manager struct { +type Manager[H Handler] struct { repositoryMutex sync.RWMutex - repositories HandlerMap + repositories map[string]H repositoryParams *HandlerParams - newHandler func(*slog.Logger, *RepoHandler) Handler + newHandler func(*slog.Logger, *RepoHandler) H } -func NewManager(params *HandlerParams, newHandler HandlerFactory) *Manager { - m := &Manager{ - repositories: make(HandlerMap), +func NewManager[H Handler]( + params *HandlerParams, + newHandler func(*slog.Logger, *RepoHandler) H, +) *Manager[H] { + m := &Manager[H]{ + repositories: make(map[string]H), repositoryParams: params, newHandler: newHandler, } @@ -34,13 +33,13 @@ func NewManager(params *HandlerParams, newHandler HandlerFactory) *Manager { return m } -func (m *Manager) remove(repository string) { +func (m *Manager[H]) remove(repository string) { m.repositoryMutex.Lock() delete(m.repositories, repository) m.repositoryMutex.Unlock() } -func (m *Manager) Get(ctx context.Context, repository string) Handler { +func (m *Manager[H]) Get(ctx context.Context, repository string) H { m.repositoryMutex.Lock() r, ok := m.repositories[repository] @@ -74,7 +73,7 @@ func (m *Manager) Get(ctx context.Context, repository string) Handler { return rh } -func (m *Manager) Has(repository string) bool { +func (m *Manager[H]) Has(repository string) bool { m.repositoryMutex.RLock() _, ok := m.repositories[repository] m.repositoryMutex.RUnlock() @@ -82,10 +81,10 @@ func (m *Manager) Has(repository string) bool { return ok } -func (m *Manager) GetAll() HandlerMap { +func (m *Manager[H]) GetAll() map[string]H { m.repositoryMutex.RLock() - handlers := make(HandlerMap) + handlers := make(map[string]H) for name, handler := range m.repositories { handlers[name] = handler } diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index 1b5cf9e..adecfa2 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -5,22 +5,55 @@ package ostree import ( "context" - "errors" - "fmt" - - apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" - "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" + + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" ) -type apiService struct{} +func checkRepository(repository string) error { + if !apiv1.RepositoryMatch(repository) { + return werror.Wrapf(gcode.ErrInvalidArgument, "invalid repository name, must match expression %q", apiv1.RepositoryRegex) + } + return nil +} + +func (p *Plugin) CreateRepository(ctx context.Context, repository string, properties *apiv1.OSTreeRepositoryProperties) (err error) { + if err := checkRepository(repository); err != nil { + return err + } -func newAPIService() *apiService { - return &apiService{} + return p.repositoryManager.Get(ctx, repository).CreateRepository(ctx, properties) } -func (o *apiService) MirrorRepository(_ context.Context, repository string, properties *apiv1.OSTreeRepositoryProperties) (err error) { - fmt.Printf("Repo: %s,\nProperties: %v\n", repository, properties) - return werror.Wrap(gcode.ErrNotImplemented, errors.New("repository mirroring not yet supported")) +func (p *Plugin) DeleteRepository(ctx context.Context, repository string) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx) +} + +func (p *Plugin) AddRemote(ctx context.Context, repository string, properties *apiv1.OSTreeRemoteProperties) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).AddRemote(ctx, properties) +} + +func (p *Plugin) SyncRepository(ctx context.Context, repository string, request *apiv1.OSTreeRepositorySyncRequest) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, request) +} + +func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { + if err := checkRepository(repository); err != nil { + return nil, err + } + + return p.repositoryManager.Get(ctx, repository).GetRepositorySyncStatus(ctx) } diff --git a/internal/plugins/ostree/pkg/libostree/README.md b/internal/plugins/ostree/pkg/libostree/README.md new file mode 100644 index 0000000..14186b8 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/README.md @@ -0,0 +1,22 @@ +# ostree + +ostree is a wrapper around [libostree](https://github.com/ostreedev/ostree) that aims to provide an idiomatic API. + + +### Notes +1. A minimal glib implementation exists within the ostree pkg. This is to avoid a dependency on glib for the time being. + - This implementation is not complete and will be expanded as needed. + - The glib implementation is not intended to be used outside of the ostree pkg. + - `GCancellable` is not implemented. Just send nil. +2. Not all of libostree is wrapped. Only the parts that are needed for beskar are wrapped. Which is basically everything + need to perform pull operations. + - `OstreeAsyncProgress` is not implemented. Just send nil. + + +### Developer Warnings +- `glib/gobject` are used here and add a layer of complexity to the code, specifically with regard to memory management. +glib/gobject are reference counted and objects are freed when the reference count reaches 0. Therefore, you will see +`C.g_XXX_ref_sink` or `C.g_XXX_ref` (increases reference count) and `C.g_XXX_unref()` (decrease reference count) in some +places and `C.free()` in others. A good rule of thumb is that if you see a `g_` prefix you are dealing with a reference +counted object and should not call `C.free()`. See [glib](https://docs.gtk.org/glib/index.html) for more information. +See [gobject](https://docs.gtk.org/gobject/index.html) for more information. diff --git a/internal/plugins/ostree/pkg/libostree/errors.go b/internal/plugins/ostree/pkg/libostree/errors.go new file mode 100644 index 0000000..43b7f9e --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/errors.go @@ -0,0 +1,11 @@ +package ostree + +type Err string + +func (e Err) Error() string { + return string(e) +} + +const ( + ErrInvalidPath = Err("invalid path") +) diff --git a/internal/plugins/ostree/pkg/libostree/generate-testdata.sh b/internal/plugins/ostree/pkg/libostree/generate-testdata.sh new file mode 100755 index 0000000..4e9d001 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/generate-testdata.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +# This script generates test data for the ostree plugin. + +# Clean up any existing test data +rm -rf testdata + +mkdir -p testdata/{repo,tree} + +# Create a simple ostree repo with two branches and a series of commits. +ostree --repo=testdata/repo init --mode=archive + +echo "Test file in a simple ostree repo - branch test1" > ./testdata/tree/testfile.txt +ostree --repo=testdata/repo commit --branch=test1 ./testdata/tree/ + +echo "Test file in a simple ostree repo - branch test2" > ./testdata/tree/testfile.txt +ostree --repo=testdata/repo commit --branch=test2 ./testdata/tree/ + +echo "Another test file" > ./testdata/tree/another_testfile.txt +ostree --repo=testdata/repo commit --branch=test2 ./testdata/tree/ + +ostree --repo=testdata/repo summary --update + +# We don't actually need the tree directory, just the repo. +rm -rf testdata/tree \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go b/internal/plugins/ostree/pkg/libostree/glib_helpers.go new file mode 100644 index 0000000..12c1a29 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go @@ -0,0 +1,21 @@ +package ostree + +// #cgo pkg-config: glib-2.0 gobject-2.0 +// #include +// #include +// #include +// #include +// #include "glib_helpers.go.h" +import "C" +import "errors" + +// GoError converts a C glib error to a Go error. +// The C error is freed after conversion. +func GoError(e *C.GError) error { + defer C.g_error_free(e) + + if e == nil { + return nil + } + return errors.New(C.GoString((*C.char)(C._g_error_get_message(e)))) +} diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go.h b/internal/plugins/ostree/pkg/libostree/glib_helpers.go.h new file mode 100644 index 0000000..4aceb8f --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go.h @@ -0,0 +1,11 @@ +#include +#include +#include +#include + +static char * +_g_error_get_message (GError *error) +{ + g_assert (error != NULL); + return error->message; +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go new file mode 100644 index 0000000..cc9eb6d --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -0,0 +1,66 @@ +package ostree + +// #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 +// #include +// #include +// #include +// #include +// #include +// #include "options.go.h" +import "C" +import "unsafe" + +// Option defines an option for pulling ostree repos. +// It is used to build a *C.GVariant via a *C.GVariantBuilder. +// free is an optional function that frees the memory allocated by the option. free may be called more than once. +type Option func(builder *C.GVariantBuilder, free freeFunc) +type freeFunc func(...unsafe.Pointer) + +// ToGVariant converts the given Options to a GVariant using a GVaraintBuilder. +func toGVariant(opts ...Option) *C.GVariant { + + typeStr := (*C.gchar)(C.CString("a{sv}")) + defer C.free(unsafe.Pointer(typeStr)) + + variantType := C.g_variant_type_new(typeStr) + + // The builder is freed by g_variant_builder_end below. + // See https://docs.gtk.org/glib/method.VariantBuilder.init.html + var builder C.GVariantBuilder + C.g_variant_builder_init(&builder, variantType) + + // Collect pointers to free later + var toFree []unsafe.Pointer + freeFn := func(ptrs ...unsafe.Pointer) { + toFree = append(toFree, ptrs...) + } + + for _, opt := range opts { + opt(&builder, freeFn) + } + defer func() { + for i := 0; i < len(toFree); i++ { + C.free(toFree[i]) + } + }() + + variant := C.g_variant_builder_end(&builder) + return C.g_variant_ref_sink(variant) +} + +func gVariantBuilderAddVariant(builder *C.GVariantBuilder, key *C.gchar, variant *C.GVariant) { + C.g_variant_builder_add_variant(builder, key, variant) +} + +// NoGPGVerify sets the gpg-verify option to false in the pull options. +func NoGPGVerify() Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("gpg-verify") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(0))), + ) + } +} diff --git a/internal/plugins/ostree/pkg/libostree/options.go.h b/internal/plugins/ostree/pkg/libostree/options.go.h new file mode 100644 index 0000000..20b89d9 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/options.go.h @@ -0,0 +1,14 @@ +#include +#include +#include +#include + +// This exists because cGo doesn't support variadic functions +void +g_variant_builder_add_variant( + GVariantBuilder *builder, + const gchar *key, + GVariant *value +) { + g_variant_builder_add(builder, "{s@v}", key, value); +} diff --git a/internal/plugins/ostree/pkg/libostree/ostree.go b/internal/plugins/ostree/pkg/libostree/ostree.go new file mode 100644 index 0000000..6f66afd --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/ostree.go @@ -0,0 +1 @@ +package ostree diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go new file mode 100644 index 0000000..0bf6801 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -0,0 +1,200 @@ +package ostree + +// #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 +// #include +// #include +// #include +// #include "pull.go.h" +import "C" +import ( + "context" + "unsafe" +) + +// Pull pulls refs from the named remote. +// Returns an error if the refs could not be fetched. +func (r *Repo) Pull(_ context.Context, remote string, opts ...Option) error { + cremote := C.CString(remote) + defer C.free(unsafe.Pointer(cremote)) + + options := toGVariant(opts...) + defer C.g_variant_unref(options) + + var cErr *C.GError + + // Pull refs from remote + // TODO: Implement cancellable so that we can cancel the pull if needed. + if C.ostree_repo_pull_with_options( + r.native, + cremote, + options, + nil, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +type FlagSet int + +const ( + // Mirror - Write out refs suitable for mirrors and fetch all refs if none requested + Mirror = 1 << iota + + // CommitOnly - Fetch only the commit metadata + CommitOnly + + // Untrusted - Do verify checksums of local (filesystem-accessible) repositories (defaults on for HTTP) + Untrusted + + // BaseUserOnlyFiles - Since 2017.7. Reject writes of content objects with modes outside of 0775. + BaseUserOnlyFiles + + // TrustedHttp - Don't verify checksums of objects HTTP repositories (Since: 2017.12) + TrustedHttp + + // None - No special options for pull + None = 0 +) + +// Flags adds the given flags to the pull options. +func Flags(flags FlagSet) Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("flags") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_int32(C.gint32(flags))), + ) + } +} + +// Refs adds the given refs to the pull options. +// When pulling refs from a remote, only the specified refs will be pulled. +func Refs(refs ...string) Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + cRefs := C.MakeRefArray(C.int(len(refs))) + free(unsafe.Pointer(cRefs)) + for i := 0; i < len(refs); i++ { + cRef := C.CString(refs[i]) + free(unsafe.Pointer(cRef)) + C.AppendRef(cRefs, C.int(i), cRef) + } + C.g_variant_builder_add_refs( + builder, + cRefs, + ) + } +} + +// NoGPGVerifySummary sets the gpg-verify-summary option to false in the pull options. +func NoGPGVerifySummary() Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("gpg-verify-summary") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(0))), + ) + } +} + +// Depth sets the depth option to the given value in the pull options. +// How far in the history to traverse; default is 0, -1 means infinite +func Depth(depth int) Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + // 0 is the default depth so there is no need to add it to the builder. + if depth != 0 { + return + } + key := C.CString("depth") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_int32(C.gint32(depth))), + ) + } +} + +// DisableStaticDelta sets the disable-static-deltas option to true in the pull options. +// Do not use static deltas. +func DisableStaticDelta() Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("disable-static-deltas") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(1))), + ) + } +} + +// RequireStaticDelta sets the require-static-deltas option to true in the pull options. +// Require static deltas. +func RequireStaticDelta() Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("require-static-deltas") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(1))), + ) + } +} + +// DryRun sets the dry-run option to true in the pull options. +// Only print information on what will be downloaded (requires static deltas). +func DryRun() Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("dry-run") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(1))), + ) + } +} + +// AppendUserAgent sets the append-user-agent option to the given value in the pull options. +// Additional string to append to the user agent. +func AppendUserAgent(appendUserAgent string) Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + // "" is the default so there is no need to add it to the builder. + if appendUserAgent == "" { + return + } + + key := C.CString("append-user-agent") + free(unsafe.Pointer(key)) + cAppendUserAgent := C.CString(appendUserAgent) + free(unsafe.Pointer(cAppendUserAgent)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_string(cAppendUserAgent)), + ) + } +} + +// NetworkRetries sets the n-network-retries option to the given value in the pull options. +// Number of times to retry each download on receiving. +func NetworkRetries(n int) Option { + return func(builder *C.GVariantBuilder, free freeFunc) { + key := C.CString("n-network-retries") + free(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_int32(C.gint32(n))), + ) + } +} diff --git a/internal/plugins/ostree/pkg/libostree/pull.go.h b/internal/plugins/ostree/pkg/libostree/pull.go.h new file mode 100644 index 0000000..a78004f --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/pull.go.h @@ -0,0 +1,34 @@ +#include +#include +#include +#include + +// The following is a mechanism for converting a Go slice of strings to a char**. This could have been done in Go, but +// it's easier and less error prone to do it here. +char** MakeRefArray(int size) { + return calloc(sizeof(char*), size); +} + +void AppendRef(char** refs, int index, char* ref) { + refs[index] = ref; +} + +void FreeRefArray(char** refs) { + int i; + for (i = 0; refs[i] != NULL; i++) { + free(refs[i]); + } + free(refs); +} + +// This exists because cGo doesn't provide a way to cast char** to const char *const *. +void g_variant_builder_add_refs(GVariantBuilder *builder, char** refs) { + g_variant_builder_add( + builder, + "{s@v}", + "refs", + g_variant_new_variant( + g_variant_new_strv((const char *const *) refs, -1) + ) + ); +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go new file mode 100644 index 0000000..0bb56e8 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -0,0 +1,164 @@ +package ostree + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "log" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestMain(m *testing.M) { + _, err := os.Stat("testdata/repo/summary") + if os.IsNotExist(err) { + log.Fatalln("testdata/repo does not exist: please run ./generate-testdata.sh") + } + + os.Exit(m.Run()) +} + +func TestRepo_Pull(t *testing.T) { + + fmt.Println(os.Getwd()) + svr := httptest.NewServer(http.FileServer(http.Dir("testdata/repo"))) + defer svr.Close() + + remoteName := "local" + remoteURL := svr.URL + //refs := []string{ + // "test1", + // "test2", + //} + + modes := []RepoMode{ + RepoModeArchive, + RepoModeArchiveZ2, + RepoModeBare, + RepoModeBareUser, + RepoModeBareUserOnly, + //RepoModeBareSplitXAttrs, + } + + // Test pull for each mode + for _, mode := range modes { + mode := mode + repoName := fmt.Sprintf("repo-%s", mode) + repoPath := fmt.Sprintf("testdata/%s", repoName) + + t.Run(repoName, func(t *testing.T) { + t.Cleanup(func() { + _ = os.RemoveAll(repoPath) + }) + + t.Run(fmt.Sprintf("should create repo in %s mode", mode), func(t *testing.T) { + repo, err := Init(repoPath, mode) + assert.NotNil(t, repo) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to initialize repo", "err: %s", err.Error()) + } + + t.Run("should not fail to init twice", func(t *testing.T) { + repo, err := Init(repoPath, mode) + assert.NotNil(t, repo) + assert.NoError(t, err) + }) + }) + + var repo *Repo + t.Run("should open repo", func(t *testing.T) { + var err error + repo, err = Open(repoPath) + assert.NotNil(t, repo) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to open repo", "err: %s", err.Error()) + } + }) + + t.Run("should create remote", func(t *testing.T) { + err := repo.AddRemote(remoteName, remoteURL, NoGPGVerify()) + assert.NoError(t, err) + + // Manually check the config file to ensure the remote was added + configData, err := os.ReadFile(fmt.Sprintf("%s/config", repoPath)) + if err != nil { + t.Errorf("failed to read config file: %s", err.Error()) + } + assert.Contains(t, string(configData), fmt.Sprintf(`[remote "%s"]`, remoteName)) + assert.Contains(t, string(configData), fmt.Sprintf(`url=%s`, remoteURL)) + }) + + t.Run("should error - remote already exists", func(t *testing.T) { + err := repo.AddRemote(remoteName, remoteURL) + assert.Error(t, err) + }) + + t.Run("should list remotes", func(t *testing.T) { + remotes := repo.ListRemotes() + assert.Equal(t, remotes, []string{remoteName}) + }) + + //TODO: Repeat the following tests for only a specific ref + t.Run("should pull entire repo", func(t *testing.T) { + err := repo.Pull( + context.TODO(), + remoteName, + Flags(Mirror|TrustedHttp), + ) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to pull repo", "err: %s", err.Error()) + } + }) + + t.Run("should list refs from original repo", func(t *testing.T) { + expectedChecksums := map[string]bool{} + test1Data, err := os.ReadFile("testdata/repo/refs/heads/test1") + test2Data, err := os.ReadFile("testdata/repo/refs/heads/test2") + if err != nil { + t.Errorf("failed to read refs file: %s", err.Error()) + } + + // Update in case of changes to testdata + expectedChecksums[strings.TrimRight(string(test1Data), "\n")] = false + expectedChecksums[strings.TrimRight(string(test2Data), "\n")] = false + + refs, err := repo.ListRefsExt(ListRefsExtFlags_None) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to list refs", "err: %s", err.Error()) + } + assert.NotEmpty(t, refs) + + for _, ref := range refs { + checksum := ref.Checksum + assert.NotEmpty(t, checksum) + for sum, _ := range expectedChecksums { + if sum == checksum { + expectedChecksums[sum] = true + } + } + } + + for sum, exists := range expectedChecksums { + assert.True(t, exists, "checksum %s not found", sum) + } + }) + + t.Run("should generate summary file", func(t *testing.T) { + err := repo.RegenerateSummary() + assert.NoError(t, err) + _, err = os.Stat(fmt.Sprintf("%s/summary", repoPath)) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to stat summary file", "err: %s", err.Error()) + } + }) + }) + } +} diff --git a/internal/plugins/ostree/pkg/libostree/repo.go b/internal/plugins/ostree/pkg/libostree/repo.go new file mode 100644 index 0000000..a71fedc --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/repo.go @@ -0,0 +1,318 @@ +package ostree + +// #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 +// #include +// #include +// #include +// #include +// #include +import "C" +import ( + "runtime" + "unsafe" +) + +// RepoMode - The mode to use when creating a new repo. +// If an unknown mode is passed, RepoModeBare will be used silently. +// +// See https://ostreedev.github.io/ostree/formats/#the-archive-format +// See https://ostreedev.github.io/ostree/formats/#aside-bare-formats +type RepoMode string + +func (r RepoMode) toC() C.OstreeRepoMode { + switch r { + case RepoModeBare: + return C.OSTREE_REPO_MODE_BARE + case RepoModeArchive: + return C.OSTREE_REPO_MODE_ARCHIVE + case RepoModeArchiveZ2: + return C.OSTREE_REPO_MODE_ARCHIVE_Z2 + case RepoModeBareUser: + return C.OSTREE_REPO_MODE_BARE_USER + case RepoModeBareUserOnly: + return C.OSTREE_REPO_MODE_BARE_USER_ONLY + case RepoModeBareSplitXAttrs: + return C.OSTREE_REPO_MODE_BARE_SPLIT_XATTRS + default: + return C.OSTREE_REPO_MODE_BARE + } +} + +const ( + // RepoModeBare - The default mode. Keeps all file metadata. May require elevated privileges. + // The bare repository format is the simplest one. In this mode regular files are directly stored to disk, and all + // metadata (e.g. uid/gid and xattrs) is reflected to the filesystem. It allows further direct access to content and + // metadata, but it may require elevated privileges when writing objects to the repository. + RepoModeBare RepoMode = "bare" + + // RepoModeArchive - The archive format. Best for small storage footprint. Mostly used for server-side repositories. + // The archive format simply gzip-compresses each content object. Metadata objects are stored uncompressed. This + // means that it’s easy to serve via static HTTP. Note: the repo config file still uses the historical term + // archive-z2 as mode. But this essentially indicates the modern archive format. + // + // When you commit new content, you will see new .filez files appearing in `objects/`. + RepoModeArchive RepoMode = "archive" + + // RepoModeArchiveZ2 - Functionally equivalent to RepoModeArchive. Only useful for backwards compatibility. + RepoModeArchiveZ2 RepoMode = "archive-z2" + + // RepoModeBareUser - Like RepoModeBare but ignore incoming uid/gid and xattrs. + // The bare-user format is a bit special in that the uid/gid and xattrs from the content are ignored. This is + // primarily useful if you want to have the same OSTree-managed content that can be run on a host system or an + // unprivileged container. + RepoModeBareUser RepoMode = "bare-user" + + // RepoModeBareUserOnly - Like RepoModeBareUser. No metadata stored. Only useful for checkouts. Does not need xattrs. + // Same as BARE_USER, but all metadata is not stored, so it can only be used for user checkouts. Does not need xattrs. + RepoModeBareUserOnly RepoMode = "bare-user-only" + + // RepoModeBareSplitXAttrs - Like RepoModeBare but store xattrs in a separate file. + // Similarly, the bare-split-xattrs format is a special mode where xattrs are stored as separate repository objects, + // and not directly reflected to the filesystem. This is primarily useful when transporting xattrs through lossy + // environments (e.g. tar streams and containerized environments). It also allows carrying security-sensitive xattrs + // (e.g. SELinux labels) out-of-band without involving OS filesystem logic. + RepoModeBareSplitXAttrs RepoMode = "bare-split-xattrs" +) + +type Repo struct { + native *C.OstreeRepo +} + +func fromNative(cRepo *C.OstreeRepo) *Repo { + repo := &Repo{ + native: cRepo, + } + + // Let the GB trigger free the cRepo for us when repo is freed. + runtime.SetFinalizer(repo, func(r *Repo) { + C.free(unsafe.Pointer(cRepo)) + }) + + return repo +} + +// Init initializes & opens a new ostree repository at the given path. +// +// Create the underlying structure on disk for the repository, and call +// ostree_repo_open() on the result, preparing it for use. +// +// Since version 2016.8, this function will succeed on an existing +// repository, and finish creating any necessary files in a partially +// created repository. However, this function cannot change the mode +// of an existing repository, and will silently ignore an attempt to +// do so. +// +// Since 2017.9, "existing repository" is defined by the existence of an +// `objects` subdirectory. +// +// This function predates ostree_repo_create_at(). It is an error to call +// this function on a repository initialized via ostree_repo_open_at(). +func Init(path string, mode RepoMode) (repo *Repo, err error) { + if path == "" { + return nil, ErrInvalidPath + } + + cPathStr := C.CString(path) + defer C.free(unsafe.Pointer(cPathStr)) + cPath := C.g_file_new_for_path(cPathStr) + defer C.g_object_unref(C.gpointer(cPath)) + + // Create a *C.OstreeRepo from the path + cRepo := C.ostree_repo_new(cPath) + defer func() { + if err != nil { + C.free(unsafe.Pointer(cRepo)) + } + }() + + var cErr *C.GError + + if r := C.ostree_repo_create(cRepo, mode.toC(), nil, &cErr); r == C.gboolean(0) { + return nil, GoError(cErr) + } + return fromNative(cRepo), nil +} + +// Open opens an ostree repository at the given path. +func Open(path string) (*Repo, error) { + if path == "" { + return nil, ErrInvalidPath + } + + cPathStr := C.CString(path) + defer C.free(unsafe.Pointer(cPathStr)) + cPath := C.g_file_new_for_path(cPathStr) + defer C.g_object_unref(C.gpointer(cPath)) + + // Create a *C.OstreeRepo from the path + cRepo := C.ostree_repo_new(cPath) + + var cErr *C.GError + + if r := C.ostree_repo_open(cRepo, nil, &cErr); r == C.gboolean(0) { + return nil, GoError(cErr) + } + + return fromNative(cRepo), nil +} + +// AddRemote adds a remote to the repository. +func (r *Repo) AddRemote(name, url string, opts ...Option) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + cURL := C.CString(url) + defer C.free(unsafe.Pointer(cURL)) + + options := toGVariant(opts...) + defer C.g_variant_unref(options) + + var cErr *C.GError + + /* + gboolean + ostree_repo_remote_add(OstreeRepo *self, + const char *name, + const char *url, + GVariant *options, + GCancellable *cancellable, + GError **error) + */ + if C.ostree_repo_remote_add( + r.native, + cName, + cURL, + options, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +// DeleteRemote deletes a remote from the repository. +func (r *Repo) DeleteRemote(name string) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + var cErr *C.GError + if C.ostree_repo_remote_delete( + r.native, + cName, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +// ReloadRemoteConfig reloads the remote configuration. +func (r *Repo) ReloadRemoteConfig() error { + var cErr *C.GError + + if C.ostree_repo_reload_config( + r.native, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +// ListRemotes lists the remotes in the repository. +func (r *Repo) ListRemotes() []string { + var n C.guint + remotes := C.ostree_repo_remote_list( + r.native, + &n, + ) + + var ret []string + for { + if *remotes == nil { + break + } + ret = append(ret, C.GoString(*remotes)) + remotes = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(remotes)) + unsafe.Sizeof(uintptr(0)))) + } + + return ret +} + +type ListRefsExtFlags int + +const ( + ListRefsExtFlags_Aliases = 1 << iota + ListRefsExtFlags_ExcludeRemotes + ListRefsExtFlags_ExcludeMirrors + ListRefsExtFlags_None ListRefsExtFlags = 0 +) + +type Ref struct { + Name string + Checksum string +} + +func (r *Repo) ListRefsExt(flags ListRefsExtFlags, prefix ...string) ([]Ref, error) { + var cPrefix *C.char + if len(prefix) > 0 { + cPrefix = C.CString(prefix[0]) + defer C.free(unsafe.Pointer(cPrefix)) + } + + cFlags := (C.OstreeRepoListRefsExtFlags)(C.int(flags)) + + var cErr *C.GError + var outAllRefs *C.GHashTable + if C.ostree_repo_list_refs_ext( + r.native, + cPrefix, + &outAllRefs, + cFlags, + nil, + &cErr, + ) == C.gboolean(0) { + return nil, GoError(cErr) + } + + // iter is freed when g_hash_table_iter_next returns false + var iter C.GHashTableIter + C.g_hash_table_iter_init(&iter, outAllRefs) + + var cRef, cChecksum C.gpointer + var ret []Ref + for C.g_hash_table_iter_next(&iter, &cRef, &cChecksum) == C.gboolean(1) { + if cRef == nil { + break + } + + ref := (*C.OstreeCollectionRef)(unsafe.Pointer(&cRef)) + + ret = append(ret, Ref{ + Name: C.GoString((*C.char)((*C.gchar)(ref.ref_name))), + Checksum: C.GoString((*C.char)(cChecksum)), + }) + } + + return ret, nil +} + +func (r *Repo) RegenerateSummary() error { + var cErr *C.GError + if C.ostree_repo_regenerate_summary( + r.native, + nil, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go new file mode 100644 index 0000000..4b3c8bd --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -0,0 +1,215 @@ +package ostreerepository + +import ( + "context" + "errors" + "fmt" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" + "go.ciq.dev/beskar/pkg/orasostree" + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" + "go.ciq.dev/beskar/pkg/utils" + "golang.org/x/sync/errgroup" + "os" + "path/filepath" +) + +var ( + errHandlerNotStarted = errors.New("handler not started") + errProvisionInProgress = errors.New("provision in progress") + errSyncInProgress = errors.New("sync in progress") + errDeleteInProgress = errors.New("delete in progress") +) + +func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTreeRepositoryProperties) (err error) { + h.logger.Debug("creating repository", "repository", h.Repository) + // Validate request + if len(properties.Remotes) == 0 { + return ctl.Errf("at least one remote is required") + } + + // Transition to provisioning state + if err := h.setState(StateProvisioning); err != nil { + return err + } + defer h.clearState() + + // Check if repo already exists + if h.checkRepoExists(ctx) { + return ctl.Errf("repository %s already exists", h.Repository) + } + + if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + + // Add user provided remotes + // We do not need to add beskar remote here + for _, remote := range properties.Remotes { + var opts []ostree.Option + if remote.NoGPGVerify { + opts = append(opts, ostree.NoGPGVerify()) + } + if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { + return ctl.Errf("while adding remote to ostree repository %s: %s", remote.Name, err) + } + } + return nil + + }); err != nil { + return err + } + + return nil +} + +// DeleteRepository deletes the repository from beskar and the local filesystem. +// +// This could lead to an invalid _state if the repository fails to completely deleting from beskar. +func (h *Handler) DeleteRepository(ctx context.Context) (err error) { + // Transition to deleting state + if err := h.setState(StateDeleting); err != nil { + return err + } + + // Check if repo already exists + if h.checkRepoExists(ctx) { + return ctl.Errf("repository %s already exists", h.Repository) + } + + go func() { + defer func() { + h.clearState() + if err == nil { + // stop the repo handler and trigger cleanup + h.Stop() + } + }() + h.logger.Debug("deleting repository", "repository", h.Repository) + + if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + + // Create a worker pool to deleting each file in the repository concurrently. + // ctx will be cancelled on error, and the error will be returned. + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(100) + + // Walk the directory tree, skipping directories and deleting each file. + if err := filepath.WalkDir(h.repoDir, func(path string, d os.DirEntry, err error) error { + // If there was an error with the file, return it. + if err != nil { + return fmt.Errorf("while walking %s: %w", path, err) + } + // Skip directories. + if d.IsDir() { + return nil + } + // Skip the rest of the files if the context has been cancelled. + if ctx.Err() != nil { + // Skip remaining files because our context has been cancelled. + // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). + // This is because we would never be able to handle an error returned from the last job. + return filepath.SkipAll + } + // Schedule deletion to run in a worker. + eg.Go(func() error { + // Delete the file from the repository + filename := filepath.Base(path) + digest := orasostree.MakeTag(filename) + digestRef := filepath.Join(h.Repository, "file:"+digest) + if err := h.DeleteManifest(digestRef); err != nil { + h.logger.Error("deleting file from beskar", "repository", h.Repository, "error", err.Error()) + } + + return nil + }) + + return nil + }); err != nil { + return nil + } + + return eg.Wait() + + }); err != nil { + h.logger.Error("while deleting repository", "repository", h.Repository, "error", err.Error()) + } + }() + + return nil +} + +func (h *Handler) AddRemote(ctx context.Context, remote *apiv1.OSTreeRemoteProperties) (err error) { + // Transition to provisioning state + if err := h.setState(StateProvisioning); err != nil { + return err + } + defer h.clearState() + + if h.checkRepoExists(ctx) { + return ctl.Errf("repository %s does not exist", h.Repository) + } + + if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + // Add user provided remote + var opts []ostree.Option + if remote.NoGPGVerify { + opts = append(opts, ostree.NoGPGVerify()) + } + if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { + return ctl.Errf("while adding remote to ostree repository %s: %s", remote.Name, err) + } + + return nil + + }); err != nil { + return err + } + + return nil +} + +func (h *Handler) SyncRepository(ctx context.Context, request *apiv1.OSTreeRepositorySyncRequest) (err error) { + // Transition to syncing state + if err := h.setState(StateSyncing); err != nil { + return err + } + + // Spin up pull worker + go func() { + defer func() { + h.clearState() + h.logger.Debug("repository sync complete", "repository", h.Repository, "request", request) + }() + + if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + // Pull the latest changes from the remote. + opts := []ostree.Option{ + ostree.Depth(request.Depth), + } + if len(request.Refs) > 0 { + opts = append(opts, ostree.Refs(request.Refs...)) + } + + // pull remote content into local repo + if err := repo.Pull(ctx, request.Remote, opts...); err != nil { + return ctl.Errf("while pulling ostree repository from %s: %s", request.Remote, err) + } + + return nil + + }); err != nil { + h.logger.Error("repository sync", "repository", h.Repository, "request", request, "error", err.Error()) + } + }() + + return nil +} + +func (h *Handler) GetRepositorySyncStatus(_ context.Context) (syncStatus *apiv1.SyncStatus, err error) { + repoSync := h.getRepoSync() + return &apiv1.SyncStatus{ + Syncing: repoSync.Syncing, + StartTime: utils.TimeToString(repoSync.StartTime), + EndTime: utils.TimeToString(repoSync.EndTime), + SyncError: repoSync.SyncError, + }, nil +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/local.go b/internal/plugins/ostree/pkg/ostreerepository/local.go new file mode 100644 index 0000000..bf1fc8d --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/local.go @@ -0,0 +1,86 @@ +package ostreerepository + +import ( + "context" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" + "go.ciq.dev/beskar/pkg/orasostree" + "os" + "path/filepath" +) + +// checkRepoExists checks if the ostree repository exists in beskar. +func (h *Handler) checkRepoExists(_ context.Context) bool { + // Check if repo already exists + configTag := orasostree.MakeTag(orasostree.FileConfig) + configRef := filepath.Join(h.Repository, "file:"+configTag) + _, err := h.GetManifestDigest(configRef) + return err == nil +} + +func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, transactor func(ctx context.Context, repo *ostree.Repo) error) error { + // We control the local repo lifecycle here, so we need to lock it. + h.repoLock.Lock() + defer h.repoLock.Unlock() + + // Open the local repo + // Create the repository directory + if err := os.MkdirAll(h.repoDir, 0o700); err != nil { + // If the directory already exists, we can continue + if !os.IsExist(err) { + return ctl.Errf("create repository dir: %s", err) + } + } + + // We will always use archive mode here + repo, err := ostree.Init(h.repoDir, ostree.RepoModeArchive) + if err != nil { + return ctl.Errf("while opening ostree repository %s: %s", h.repoDir, err) + } + + // Ad beskar as a remote so that we can pull from it + beskarServiceURL := h.Params.GetBeskarServiceHostPort() + if err := repo.AddRemote(beskarRemoteName, beskarServiceURL, ostree.NoGPGVerify()); err != nil { + return ctl.Errf("while adding remote to ostree repository %s: %s", beskarRemoteName, err) + } + + // pull remote content into local repo from beskar + if h.checkRepoExists(ctx) { + if err := repo.Pull(ctx, beskarRemoteName, ostree.NoGPGVerify()); err != nil { + return ctl.Errf("while pulling ostree repository from %s: %s", beskarRemoteName, err) + } + } + + // Execute the transaction + if err := transactor(ctx, repo); err != nil { + return ctl.Errf("while executing transaction: %s", err) + } + + if err := repo.RegenerateSummary(); err != nil { + return ctl.Errf("while regenerating summary for ostree repository %s: %s", h.repoDir, err) + } + + // Remove the internal beskar remote so that external clients can't pull from it, not that it would work. + if err := repo.DeleteRemote(beskarRemoteName); err != nil { + return ctl.Errf("while deleting remote %s: %s", beskarRemoteName, err) + } + + // Close the local + // Push local repo to beskar using OSTreePusher + if err := orasostree.PushOSTreeRepository( + ctx, + h.repoDir, + h.Repository, + 100, + h.Params.NameOptions..., + ); err != nil { + return err + } + + // Clean up the disk + if err := os.RemoveAll(h.repoDir); err != nil { + return ctl.Errf("while removing local repo %s: %s", h.repoDir, err) + } + + return nil +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go b/internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go new file mode 100644 index 0000000..db68b5f --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go @@ -0,0 +1,120 @@ +package ostreerepository + +import ( + "context" + "fmt" + "github.com/RussellLuo/kun/pkg/werror" + "github.com/RussellLuo/kun/pkg/werror/gcode" + "go.ciq.dev/beskar/internal/pkg/repository" + eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" + "log/slog" + "os" + "path/filepath" + "sync" + "sync/atomic" +) + +const ( + beskarRemoteName = "_beskar_" +) + +type State int32 + +const ( + // StateStopped - The repository _state is unknown. + StateStopped State = iota + // StateReady - The repository is ready. + StateReady + // StateProvisioning - The repository is being provisioned. + StateProvisioning + // StateSyncing - The repository is being synced. + StateSyncing + // StateDeleting - The repository is being deleted. + StateDeleting +) + +func (s State) String() string { + switch s { + case StateStopped: + return "stopped" + case StateReady: + return "ready" + case StateProvisioning: + return "provisioning" + case StateSyncing: + return "syncing" + case StateDeleting: + return "deleting" + default: + return "unknown" + } +} + +type Handler struct { + *repository.RepoHandler + logger *slog.Logger + repoDir string + repoLock sync.RWMutex + repoSync atomic.Pointer[RepoSync] + + _state atomic.Int32 +} + +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { + return &Handler{ + RepoHandler: repoHandler, + repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), + logger: logger, + } +} + +func (h *Handler) setState(state State) error { + current := h.getState() + if current != StateReady { + return werror.Wrap(gcode.ErrUnavailable, fmt.Errorf("repository is not ready: %s", current)) + } + h._state.Swap(int32(state)) + if state == StateSyncing || current == StateSyncing { + h.updateSyncing(state == StateSyncing) + } + return nil +} + +func (h *Handler) clearState() { + h._state.Swap(int32(StateReady)) +} + +func (h *Handler) getState() State { + if !h.Started() { + return StateStopped + } + return State(h._state.Load()) +} + +func (h *Handler) cleanup() { + h.logger.Debug("repository cleanup", "repository", h.Repository) + h.repoLock.Lock() + defer h.repoLock.Unlock() + + close(h.Queued) + h.Params.Remove(h.Repository) + _ = os.RemoveAll(h.repoDir) +} + +func (h *Handler) QueueEvent(_ *eventv1.EventPayload, _ bool) error { + return nil +} + +func (h *Handler) Start(ctx context.Context) { + h.logger.Debug("starting repository", "repository", h.Repository) + h.clearState() + go func() { + for !h.Stopped.Load() { + select { + case <-ctx.Done(): + h.Stopped.Store(true) + } + } + h.cleanup() + }() +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/sync.go b/internal/plugins/ostree/pkg/ostreerepository/sync.go new file mode 100644 index 0000000..9b87f59 --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/sync.go @@ -0,0 +1,35 @@ +package ostreerepository + +import ( + "time" +) + +type RepoSync struct { + Syncing bool `db:"syncing"` + StartTime int64 `db:"start_time"` + EndTime int64 `db:"end_time"` + SyncError string `db:"sync_error"` +} + +func (h *Handler) getRepoSync() *RepoSync { + return h.repoSync.Load() +} + +func (h *Handler) setRepoSync(repoSync *RepoSync) { + rs := *repoSync + h.repoSync.Store(&rs) +} + +func (h *Handler) updateSyncing(syncing bool) *RepoSync { + repoSync := *h.getRepoSync() + previousSyncing := repoSync.Syncing + repoSync.Syncing = syncing + if syncing && !previousSyncing { + repoSync.StartTime = time.Now().UTC().Unix() + repoSync.SyncError = "" + } else if !syncing && previousSyncing { + repoSync.EndTime = time.Now().UTC().Unix() + } + h.repoSync.Store(&repoSync) + return h.repoSync.Load() +} diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go index 1af8d3c..2c52696 100644 --- a/internal/plugins/ostree/plugin.go +++ b/internal/plugins/ostree/plugin.go @@ -6,20 +6,25 @@ package ostree import ( "context" _ "embed" - "net/http" - "net/http/pprof" - "github.com/RussellLuo/kun/pkg/httpcodec" "github.com/go-chi/chi" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" "go.ciq.dev/beskar/internal/pkg/gossip" "go.ciq.dev/beskar/internal/pkg/log" "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/pkg/repository" "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/ostreerepository" pluginv1 "go.ciq.dev/beskar/pkg/api/plugin/v1" "go.ciq.dev/beskar/pkg/mtls" apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" "go.ciq.dev/beskar/pkg/version" + "net" + "net/http" + "net/http/pprof" + "path/filepath" + "strconv" ) const ( @@ -36,6 +41,9 @@ var routerData []byte type Plugin struct { ctx context.Context config pluginsrv.Config + + repositoryManager *repository.Manager[*ostreerepository.Handler] + handlerParams *repository.HandlerParams } func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*Plugin, error) { @@ -45,8 +53,23 @@ func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*P } ctx = log.SetContextLogger(ctx, logger) - apiSrv := newAPIService() - router := makeRouter(apiSrv, beskarOSTreeConfig.Profiling) + router := chi.NewRouter() + + // for kubernetes probes + router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + if beskarOSTreeConfig.Profiling { + router.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + router.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + router.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + router.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + router.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + router.Handle("/debug/pprof/{cmd}", http.HandlerFunc(pprof.Index)) // special handling for Gorilla mux + } + + params := &repository.HandlerParams{ + Dir: filepath.Join(beskarOSTreeConfig.DataDir, "_repohandlers_"), + } return &Plugin{ ctx: ctx, @@ -56,7 +79,6 @@ func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*P Info: &pluginv1.Info{ Name: PluginName, // Not registering media types so that Beskar doesn't send events. - // This plugin as no internal state so events are not needed. Mediatypes: []string{}, Version: version.Semver, Router: &pluginv1.Router{ @@ -65,53 +87,48 @@ func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*P }, }, }, + handlerParams: params, + repositoryManager: repository.NewManager[*ostreerepository.Handler]( + params, + ostreerepository.NewHandler, + ), }, nil } -func (p *Plugin) Start(_ http.RoundTripper, _ *mtls.CAPEM, _ *gossip.BeskarMeta) error { - // Nothing to do here as this plugin has no internal state - // and router is already configured. - return nil -} - -func (p *Plugin) Context() context.Context { - return p.ctx -} - -func (p *Plugin) Config() pluginsrv.Config { - return p.config -} - -func (p *Plugin) RepositoryManager() *repository.Manager { - // this plugin has no internal state so no need for a repository manager - return nil -} - -func makeRouter(apiSrv *apiService, profilingEnabled bool) *chi.Mux { - router := chi.NewRouter() - - // for kubernetes probes - router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) +func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *gossip.BeskarMeta) error { + // Collection beskar http service endpoint for later pulls + p.handlerParams.BeskarMeta = beskarMeta - if profilingEnabled { - router.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) - router.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) - router.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) - router.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) - router.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) - router.Handle("/debug/pprof/{cmd}", http.HandlerFunc(pprof.Index)) // special handling for Gorilla mux + hostport := net.JoinHostPort(beskarMeta.Hostname, strconv.Itoa(int(beskarMeta.RegistryPort))) + p.handlerParams.NameOptions = []name.Option{ + name.WithDefaultRegistry(hostport), + } + p.handlerParams.RemoteOptions = []remote.Option{ + remote.WithTransport(transport), } - router.Route( + p.config.Router.Route( PluginAPIPathPattern, func(r chi.Router) { r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( - apiSrv, + p, httpcodec.NewDefaultCodecs(nil), )) }, ) - return router + return nil +} + +func (p *Plugin) Context() context.Context { + return p.ctx +} + +func (p *Plugin) Config() pluginsrv.Config { + return p.config +} + +func (p *Plugin) RepositoryManager() *repository.Manager[*ostreerepository.Handler] { + return p.repositoryManager } diff --git a/internal/plugins/static/api.go b/internal/plugins/static/api.go index 3249b6f..ce0cb7a 100644 --- a/internal/plugins/static/api.go +++ b/internal/plugins/static/api.go @@ -6,8 +6,6 @@ package static import ( "context" - "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" - "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" apiv1 "go.ciq.dev/beskar/pkg/plugins/static/api/v1" @@ -20,83 +18,44 @@ func checkRepository(repository string) error { return nil } -func (p *Plugin) getHandlerForRepository(ctx context.Context, repository string) (*staticrepository.Handler, error) { - h, ok := p.repositoryManager.Get(ctx, repository).(*staticrepository.Handler) - if !ok { - return nil, werror.Wrapf(gcode.ErrNotFound, "repository %q does not exist in the required form", repository) - } - - return h, nil -} - func (p *Plugin) DeleteRepository(ctx context.Context, repository string, deleteFiles bool) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.DeleteRepository(ctx, deleteFiles) + return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx, deleteFiles) } func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.ListRepositoryLogs(ctx, page) + return p.repositoryManager.Get(ctx, repository).ListRepositoryLogs(ctx, page) } func (p *Plugin) RemoveRepositoryFile(ctx context.Context, repository string, tag string) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.RemoveRepositoryFile(ctx, tag) + return p.repositoryManager.Get(ctx, repository).RemoveRepositoryFile(ctx, tag) } func (p *Plugin) GetRepositoryFileByTag(ctx context.Context, repository string, tag string) (repositoryFile *apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.GetRepositoryFileByTag(ctx, tag) + return p.repositoryManager.Get(ctx, repository).GetRepositoryFileByTag(ctx, tag) } func (p *Plugin) GetRepositoryFileByName(ctx context.Context, repository string, name string) (repositoryFile *apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.GetRepositoryFileByName(ctx, name) + return p.repositoryManager.Get(ctx, repository).GetRepositoryFileByName(ctx, name) } func (p *Plugin) ListRepositoryFiles(ctx context.Context, repository string, page *apiv1.Page) (repositoryFiles []*apiv1.RepositoryFile, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.ListRepositoryFiles(ctx, page) + return p.repositoryManager.Get(ctx, repository).ListRepositoryFiles(ctx, page) } diff --git a/internal/plugins/static/pkg/staticrepository/api.go b/internal/plugins/static/pkg/staticrepository/api.go index 184ebf2..ea51025 100644 --- a/internal/plugins/static/pkg/staticrepository/api.go +++ b/internal/plugins/static/pkg/staticrepository/api.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "go.ciq.dev/beskar/pkg/utils" "path/filepath" "time" @@ -90,7 +91,7 @@ func (h *Handler) ListRepositoryLogs(ctx context.Context, _ *apiv1.Page) (logs [ logs = append(logs, apiv1.RepositoryLog{ Level: log.Level, Message: log.Message, - Date: timeToString(log.Date), + Date: utils.TimeToString(log.Date), }) return nil }) @@ -221,16 +222,7 @@ func toRepositoryFileAPI(pkg *staticdb.RepositoryFile) *apiv1.RepositoryFile { Tag: pkg.Tag, ID: pkg.ID, Name: pkg.Name, - UploadTime: timeToString(pkg.UploadTime), + UploadTime: utils.TimeToString(pkg.UploadTime), Size: pkg.Size, } } - -const timeFormat = time.DateTime + " MST" - -func timeToString(t int64) string { - if t == 0 { - return "" - } - return time.Unix(t, 0).Format(timeFormat) -} diff --git a/internal/plugins/static/pkg/staticrepository/handler.go b/internal/plugins/static/pkg/staticrepository/handler.go index 1f09859..2e04eab 100644 --- a/internal/plugins/static/pkg/staticrepository/handler.go +++ b/internal/plugins/static/pkg/staticrepository/handler.go @@ -35,7 +35,7 @@ type Handler struct { delete atomic.Bool } -func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) repository.Handler { +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { return &Handler{ RepoHandler: repoHandler, repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), diff --git a/internal/plugins/static/plugin.go b/internal/plugins/static/plugin.go index ded40d8..4aab58a 100644 --- a/internal/plugins/static/plugin.go +++ b/internal/plugins/static/plugin.go @@ -41,11 +41,11 @@ type Plugin struct { ctx context.Context config pluginsrv.Config - repositoryManager *repository.Manager + repositoryManager *repository.Manager[*staticrepository.Handler] handlerParams *repository.HandlerParams } -var _ pluginsrv.Service = &Plugin{} +var _ pluginsrv.Service[*staticrepository.Handler] = &Plugin{} func New(ctx context.Context, beskarStaticConfig *config.BeskarStaticConfig) (*Plugin, error) { logger, err := beskarStaticConfig.Log.Logger(log.ContextHandler) @@ -65,7 +65,7 @@ func New(ctx context.Context, beskarStaticConfig *config.BeskarStaticConfig) (*P Dir: filepath.Join(beskarStaticConfig.DataDir, "_repohandlers_"), }, } - plugin.repositoryManager = repository.NewManager( + plugin.repositoryManager = repository.NewManager[*staticrepository.Handler]( plugin.handlerParams, staticrepository.NewHandler, ) @@ -124,7 +124,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/static/api/v1", func(r chi.Router) { - r.Use(pluginsrv.IsTLSMiddleware) + r.Use(p.apiMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -143,6 +143,15 @@ func (p *Plugin) Context() context.Context { return p.ctx } -func (p *Plugin) RepositoryManager() *repository.Manager { +func (p *Plugin) RepositoryManager() *repository.Manager[*staticrepository.Handler] { return p.repositoryManager } + +func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !pluginsrv.IsTLS(w, r) { + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/internal/plugins/yum/api.go b/internal/plugins/yum/api.go index eefd60f..665f561 100644 --- a/internal/plugins/yum/api.go +++ b/internal/plugins/yum/api.go @@ -6,8 +6,6 @@ package yum import ( "context" - "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" - "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" apiv1 "go.ciq.dev/beskar/pkg/plugins/yum/api/v1" @@ -20,157 +18,86 @@ func checkRepository(repository string) error { return nil } -func (p *Plugin) getHandlerForRepository(ctx context.Context, repository string) (*yumrepository.Handler, error) { - h, ok := p.repositoryManager.Get(ctx, repository).(*yumrepository.Handler) - if !ok { - return nil, werror.Wrapf(gcode.ErrNotFound, "repository %q does not exist in the required form", repository) - } - - return h, nil -} - func (p *Plugin) CreateRepository(ctx context.Context, repository string, properties *apiv1.RepositoryProperties) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.CreateRepository(ctx, properties) + return p.repositoryManager.Get(ctx, repository).CreateRepository(ctx, properties) } func (p *Plugin) DeleteRepository(ctx context.Context, repository string, deletePackages bool) (err error) { if err := checkRepository(repository); err != nil { return err } - - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.DeleteRepository(ctx, deletePackages) + return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx, deletePackages) } func (p *Plugin) UpdateRepository(ctx context.Context, repository string, properties *apiv1.RepositoryProperties) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.UpdateRepository(ctx, properties) + return p.repositoryManager.Get(ctx, repository).UpdateRepository(ctx, properties) } func (p *Plugin) GetRepository(ctx context.Context, repository string) (properties *apiv1.RepositoryProperties, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.GetRepository(ctx) + return p.repositoryManager.Get(ctx, repository).GetRepository(ctx) } func (p *Plugin) SyncRepository(ctx context.Context, repository string, wait bool) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h..SyncRepository(ctx, wait) - + return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, wait) } func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.GetRepositorySyncStatus(ctx) + return p.repositoryManager.Get(ctx, repository).GetRepositorySyncStatus(ctx) } func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.ListRepositoryLogs(ctx, page) + return p.repositoryManager.Get(ctx, repository).ListRepositoryLogs(ctx, page) } func (p *Plugin) RemoveRepositoryPackage(ctx context.Context, repository string, id string) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.RemoveRepositoryPackage(ctx, id) + return p.repositoryManager.Get(ctx, repository).RemoveRepositoryPackage(ctx, id) } func (p *Plugin) RemoveRepositoryPackageByTag(ctx context.Context, repository string, tag string) (err error) { if err := checkRepository(repository); err != nil { return err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return err - } - - return h.RemoveRepositoryPackageByTag(ctx, tag) + return p.repositoryManager.Get(ctx, repository).RemoveRepositoryPackageByTag(ctx, tag) } func (p *Plugin) GetRepositoryPackage(ctx context.Context, repository string, id string) (repositoryPackage *apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.GetRepositoryPackage(ctx, id) + return p.repositoryManager.Get(ctx, repository).GetRepositoryPackage(ctx, id) } func (p *Plugin) GetRepositoryPackageByTag(ctx context.Context, repository string, tag string) (repositoryPackage *apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.GetRepositoryPackageByTag(ctx, tag) + return p.repositoryManager.Get(ctx, repository).GetRepositoryPackageByTag(ctx, tag) } func (p *Plugin) ListRepositoryPackages(ctx context.Context, repository string, page *apiv1.Page) (repositoryPackages []*apiv1.RepositoryPackage, err error) { if err := checkRepository(repository); err != nil { return nil, err } - h, err := p.getHandlerForRepository(ctx, repository) - if err != nil { - return nil, err - } - - return h.ListRepositoryPackages(ctx, page) + return p.repositoryManager.Get(ctx, repository).ListRepositoryPackages(ctx, page) } diff --git a/internal/plugins/yum/pkg/yumrepository/api.go b/internal/plugins/yum/pkg/yumrepository/api.go index 23ffa9a..633349d 100644 --- a/internal/plugins/yum/pkg/yumrepository/api.go +++ b/internal/plugins/yum/pkg/yumrepository/api.go @@ -9,6 +9,7 @@ import ( "encoding/gob" "errors" "fmt" + "go.ciq.dev/beskar/pkg/utils" "path/filepath" "time" @@ -24,8 +25,6 @@ import ( var dbCtx = context.Background() -const timeFormat = time.DateTime + " MST" - func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.RepositoryProperties) (err error) { if !h.Started() { return werror.Wrap(gcode.ErrUnavailable, err) @@ -305,8 +304,8 @@ func (h *Handler) GetRepositorySyncStatus(context.Context) (syncStatus *apiv1.Sy reposync := h.getReposync() return &apiv1.SyncStatus{ Syncing: reposync.Syncing, - StartTime: timeToString(reposync.StartTime), - EndTime: timeToString(reposync.EndTime), + StartTime: utils.TimeToString(reposync.StartTime), + EndTime: utils.TimeToString(reposync.EndTime), TotalPackages: reposync.TotalPackages, SyncedPackages: reposync.SyncedPackages, SyncError: reposync.SyncError, @@ -328,7 +327,7 @@ func (h *Handler) ListRepositoryLogs(ctx context.Context, _ *apiv1.Page) (logs [ logs = append(logs, apiv1.RepositoryLog{ Level: log.Level, Message: log.Message, - Date: timeToString(log.Date), + Date: utils.TimeToString(log.Date), }) return nil }) @@ -530,8 +529,8 @@ func toRepositoryPackageAPI(pkg *yumdb.RepositoryPackage) *apiv1.RepositoryPacka Tag: pkg.Tag, ID: pkg.ID, Name: pkg.Name, - UploadTime: timeToString(pkg.UploadTime), - BuildTime: timeToString(pkg.BuildTime), + UploadTime: utils.TimeToString(pkg.UploadTime), + BuildTime: utils.TimeToString(pkg.BuildTime), Size: pkg.Size, Architecture: pkg.Architecture, SourceRPM: pkg.SourceRPM, @@ -546,10 +545,3 @@ func toRepositoryPackageAPI(pkg *yumdb.RepositoryPackage) *apiv1.RepositoryPacka GPGSignature: pkg.GPGSignature, } } - -func timeToString(t int64) string { - if t == 0 { - return "" - } - return time.Unix(t, 0).Format(timeFormat) -} diff --git a/internal/plugins/yum/pkg/yumrepository/handler.go b/internal/plugins/yum/pkg/yumrepository/handler.go index 8c980ce..10fad30 100644 --- a/internal/plugins/yum/pkg/yumrepository/handler.go +++ b/internal/plugins/yum/pkg/yumrepository/handler.go @@ -52,7 +52,7 @@ type Handler struct { delete atomic.Bool } -func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) repository.Handler { +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { return &Handler{ RepoHandler: repoHandler, repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), diff --git a/internal/plugins/yum/plugin.go b/internal/plugins/yum/plugin.go index cb205a8..2688fd4 100644 --- a/internal/plugins/yum/plugin.go +++ b/internal/plugins/yum/plugin.go @@ -41,11 +41,11 @@ type Plugin struct { ctx context.Context config pluginsrv.Config - repositoryManager *repository.Manager + repositoryManager *repository.Manager[*yumrepository.Handler] handlerParams *repository.HandlerParams } -var _ pluginsrv.Service = &Plugin{} +var _ pluginsrv.Service[*yumrepository.Handler] = &Plugin{} func New(ctx context.Context, beskarYumConfig *config.BeskarYumConfig) (*Plugin, error) { logger, err := beskarYumConfig.Log.Logger(log.ContextHandler) @@ -65,7 +65,7 @@ func New(ctx context.Context, beskarYumConfig *config.BeskarYumConfig) (*Plugin, Dir: filepath.Join(beskarYumConfig.DataDir, "_repohandlers_"), }, } - plugin.repositoryManager = repository.NewManager( + plugin.repositoryManager = repository.NewManager[*yumrepository.Handler]( plugin.handlerParams, yumrepository.NewHandler, ) @@ -146,6 +146,6 @@ func (p *Plugin) Context() context.Context { return p.ctx } -func (p *Plugin) RepositoryManager() *repository.Manager { +func (p *Plugin) RepositoryManager() *repository.Manager[*yumrepository.Handler] { return p.repositoryManager } diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go index 0a86011..1b04bb7 100644 --- a/pkg/orasostree/ostree.go +++ b/pkg/orasostree/ostree.go @@ -20,9 +20,9 @@ const ( OSTreeConfigType = "application/vnd.ciq.ostree.file.v1.config+json" OSTreeLayerType = "application/vnd.ciq.ostree.v1.file" - KnownFileSummary = "summary" - KnownFileSummarySig = "summary.sig" - KnownFileConfig = "config" + FileSummary = "summary" + FileSummarySig = "summary.sig" + FileConfig = "config" ) func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras.Pusher, error) { @@ -39,7 +39,7 @@ func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras. path = strings.TrimPrefix(path, repoRootDir) path = strings.TrimPrefix(path, "/") - fileTag := makeTag(path) + fileTag := MakeTag(path) rawRef := filepath.Join(repo, "file:"+fileTag) ref, err := name.ParseReference(rawRef, opts...) if err != nil { @@ -51,7 +51,7 @@ func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras. return oras.NewGenericPusher( ref, oras.NewManifestConfig(OSTreeConfigType, nil), - oras.NewLocalFileLayer(absolutePath, oras.WithLocalFileLayerMediaType(OSTreeLayerType)), + oras.NewLocalFileLayer(absolutePath, oras.WithLayerMediaType(OSTreeLayerType)), ), nil } @@ -59,15 +59,15 @@ func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras. // These files are meant to stand out in the registry. // Note: Values are not limited to the repo's root directory, but at the moment on the following have been identified. var specialTags = []string{ - KnownFileSummary, - KnownFileSummarySig, - KnownFileConfig, + FileSummary, + FileSummarySig, + FileConfig, } -// makeTag creates a tag for a file. +// MakeTag creates a tag for a file. // If the filename starts with a special tag, the tag is returned as-is. // Otherwise, the tag is the md5 hash of the filename. -func makeTag(filename string) string { +func MakeTag(filename string) string { for _, tag := range specialTags { if filename == tag { return tag diff --git a/pkg/orasostree/push.go b/pkg/orasostree/push.go new file mode 100644 index 0000000..b69fcc4 --- /dev/null +++ b/pkg/orasostree/push.go @@ -0,0 +1,85 @@ +package orasostree + +import ( + "context" + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "go.ciq.dev/beskar/pkg/oras" + "golang.org/x/sync/errgroup" + "os" + "path/filepath" + "strings" +) + +// PushOSTreeRepository walks a local ostree repository and pushes each file to the given registry. +// dir is the root directory of the ostree repository, i.e., the directory containing the summary file. +// repo is the name of the ostree repository. +// registry is the registry to push to. +func PushOSTreeRepository(ctx context.Context, dir, repo string, jobCount int, opts ...name.Option) error { + // Prove that we were given the root directory of an ostree repository + // by checking for the existence of the config file. + // Typically, libostree will check for the "objects" directory, but this will do just the same. + fileInfo, err := os.Stat(filepath.Join(dir, FileConfig)) + if os.IsNotExist(err) || fileInfo.IsDir() { + return fmt.Errorf("%s file not found in %s: you may need to call ostree init", FileConfig, dir) + } else if err != nil { + return fmt.Errorf("error accessing %s in %s: %w", FileConfig, dir, err) + } + + // Create a worker pool to push each file in the repository concurrently. + // ctx will be cancelled on error, and the error will be returned. + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(jobCount) + + // Walk the directory tree, skipping directories and pushing each file. + if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + // If there was an error with the file, return it. + if err != nil { + return fmt.Errorf("while walking %s: %w", path, err) + } + + // Skip directories. + if d.IsDir() { + return nil + } + + if ctx.Err() != nil { + // Skip remaining files because our context has been cancelled. + // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). + // This is because we would never be able to handle an error returned from the last job. + return filepath.SkipAll + } + + eg.Go(func() error { + if err := push(dir, path, repo, opts...); err != nil { + return fmt.Errorf("while pushing %s: %w", path, err) + } + return nil + }) + + return nil + }); err != nil { + // We should only receive here if filepath.WalkDir() returns an error. + // Push errors are handled below. + return fmt.Errorf("while walking %s: %w", dir, err) + } + + // Wait for all workers to finish. + // If any worker returns an error, eg.Wait() will return that error. + return eg.Wait() +} + +func push(repoRootDir, path, repo string, opts ...name.Option) error { + pusher, err := NewOSTreePusher(repoRootDir, path, repo, opts...) + if err != nil { + return fmt.Errorf("while creating OSTree pusher: %w", err) + } + + path = strings.TrimPrefix(path, repoRootDir) + path = strings.TrimPrefix(path, "/") + fmt.Printf("Pushing %s to %s\n", path, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 7f9d224..f11be93 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -25,10 +25,43 @@ type Page struct { } type OSTreeRepositoryProperties struct { + // Remotes - The remote repositories to mirror. + Remotes []OSTreeRemoteProperties `json:"remotes"` +} + +type OSTreeRemoteProperties struct { + // Name - The name of the remote repository. + Name string `json:"name"` + + // RemoteURL - The http url of the remote repository. RemoteURL string `json:"remote_url"` - Branch string `json:"branch"` - Depth int `json:"depth"` - Mirror bool `json:"mirror"` + + // GPGVerify - Whether to verify the GPG signature of the repository. + NoGPGVerify bool `json:"no_gpg_verify"` +} + +type OSTreeRepositorySyncRequest struct { + // Remote - The name of the remote to sync. + Remote string `json:"remote"` + + // Refs - The branches/refs to mirror. Leave empty to mirror all branches/refs. + Refs []string `json:"branch"` + + // Depth - The depth of the mirror. Defaults is 0, -1 means infinite. + Depth int `json:"depth"` +} + +// Mirror sync status. +type SyncStatus struct { + Syncing bool `json:"syncing"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time"` + SyncError string `json:"sync_error"` + + //TODO: Implement these + // The data for these is present when performing a pull via the ostree cli, so it is in the libostree code base. + //SyncedMetadata int `json:"synced_metadata"` + //SyncedObjects int `json:"synced_objects"` } // OSTree is used for managing ostree repositories. @@ -40,8 +73,28 @@ type OSTreeRepositoryProperties struct { //kun:oas docsPath=/doc/swagger.yaml //kun:oas tags=ostree type OSTree interface { + // Create an OSTree repository. + //kun:op POST /repository + //kun:success statusCode=200 + CreateRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) + + // Delete a OSTree repository. + //kun:op DELETE /repository + //kun:success statusCode=200 + DeleteRepository(ctx context.Context, repository string) (err error) + + // Add a new remote to the OSTree repository. + //kun:op POST /repository/remote:add + //kun:success statusCode=200 + AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) + // Mirror an ostree repository. - //kun:op POST /repository/mirror + //kun:op POST /repository/sync + //kun:success statusCode=200 + SyncRepository(ctx context.Context, repository string, request *OSTreeRepositorySyncRequest) (err error) + + // Get YUM repository sync status. + //kun:op GET /repository/sync:status //kun:success statusCode=200 - MirrorRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) + GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *SyncStatus, err error) } diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go index 99c6dd7..dd7fd1c 100644 --- a/pkg/plugins/ostree/api/v1/endpoint.go +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -11,38 +11,184 @@ import ( "github.com/go-kit/kit/endpoint" ) -type MirrorRepositoryRequest struct { +type AddRemoteRequest struct { + Repository string `json:"repository"` + Properties *OSTreeRemoteProperties `json:"properties"` +} + +// ValidateAddRemoteRequest creates a validator for AddRemoteRequest. +func ValidateAddRemoteRequest(newSchema func(*AddRemoteRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*AddRemoteRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type AddRemoteResponse struct { + Err error `json:"-"` +} + +func (r *AddRemoteResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *AddRemoteResponse) Failed() error { return r.Err } + +// MakeEndpointOfAddRemote creates the endpoint for s.AddRemote. +func MakeEndpointOfAddRemote(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*AddRemoteRequest) + err := s.AddRemote( + ctx, + req.Repository, + req.Properties, + ) + return &AddRemoteResponse{ + Err: err, + }, nil + } +} + +type CreateRepositoryRequest struct { Repository string `json:"repository"` Properties *OSTreeRepositoryProperties `json:"properties"` } -// ValidateMirrorRepositoryRequest creates a validator for MirrorRepositoryRequest. -func ValidateMirrorRepositoryRequest(newSchema func(*MirrorRepositoryRequest) validating.Schema) httpoption.Validator { +// ValidateCreateRepositoryRequest creates a validator for CreateRepositoryRequest. +func ValidateCreateRepositoryRequest(newSchema func(*CreateRepositoryRequest) validating.Schema) httpoption.Validator { return httpoption.FuncValidator(func(value interface{}) error { - req := value.(*MirrorRepositoryRequest) + req := value.(*CreateRepositoryRequest) return httpoption.Validate(newSchema(req)) }) } -type MirrorRepositoryResponse struct { +type CreateRepositoryResponse struct { Err error `json:"-"` } -func (r *MirrorRepositoryResponse) Body() interface{} { return r } +func (r *CreateRepositoryResponse) Body() interface{} { return r } // Failed implements endpoint.Failer. -func (r *MirrorRepositoryResponse) Failed() error { return r.Err } +func (r *CreateRepositoryResponse) Failed() error { return r.Err } -// MakeEndpointOfMirrorRepository creates the endpoint for s.MirrorRepository. -func MakeEndpointOfMirrorRepository(s OSTree) endpoint.Endpoint { +// MakeEndpointOfCreateRepository creates the endpoint for s.CreateRepository. +func MakeEndpointOfCreateRepository(s OSTree) endpoint.Endpoint { return func(ctx context.Context, request interface{}) (interface{}, error) { - req := request.(*MirrorRepositoryRequest) - err := s.MirrorRepository( + req := request.(*CreateRepositoryRequest) + err := s.CreateRepository( ctx, req.Repository, req.Properties, ) - return &MirrorRepositoryResponse{ + return &CreateRepositoryResponse{ + Err: err, + }, nil + } +} + +type DeleteRepositoryRequest struct { + Repository string `json:"repository"` +} + +// ValidateDeleteRepositoryRequest creates a validator for DeleteRepositoryRequest. +func ValidateDeleteRepositoryRequest(newSchema func(*DeleteRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*DeleteRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type DeleteRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *DeleteRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *DeleteRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfDeleteRepository creates the endpoint for s.DeleteRepository. +func MakeEndpointOfDeleteRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*DeleteRepositoryRequest) + err := s.DeleteRepository( + ctx, + req.Repository, + ) + return &DeleteRepositoryResponse{ + Err: err, + }, nil + } +} + +type GetRepositorySyncStatusRequest struct { + Repository string `json:"repository"` +} + +// ValidateGetRepositorySyncStatusRequest creates a validator for GetRepositorySyncStatusRequest. +func ValidateGetRepositorySyncStatusRequest(newSchema func(*GetRepositorySyncStatusRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*GetRepositorySyncStatusRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type GetRepositorySyncStatusResponse struct { + SyncStatus *SyncStatus `json:"sync_status"` + Err error `json:"-"` +} + +func (r *GetRepositorySyncStatusResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *GetRepositorySyncStatusResponse) Failed() error { return r.Err } + +// MakeEndpointOfGetRepositorySyncStatus creates the endpoint for s.GetRepositorySyncStatus. +func MakeEndpointOfGetRepositorySyncStatus(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*GetRepositorySyncStatusRequest) + syncStatus, err := s.GetRepositorySyncStatus( + ctx, + req.Repository, + ) + return &GetRepositorySyncStatusResponse{ + SyncStatus: syncStatus, + Err: err, + }, nil + } +} + +type SyncRepositoryRequest struct { + Repository string `json:"repository"` + Request *OSTreeRepositorySyncRequest `json:"request"` +} + +// ValidateSyncRepositoryRequest creates a validator for SyncRepositoryRequest. +func ValidateSyncRepositoryRequest(newSchema func(*SyncRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*SyncRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type SyncRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *SyncRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *SyncRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfSyncRepository creates the endpoint for s.SyncRepository. +func MakeEndpointOfSyncRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*SyncRepositoryRequest) + err := s.SyncRepository( + ctx, + req.Repository, + req.Request, + ) + return &SyncRepositoryResponse{ Err: err, }, nil } diff --git a/pkg/plugins/ostree/api/v1/http.go b/pkg/plugins/ostree/api/v1/http.go index 08d5f91..7feed56 100644 --- a/pkg/plugins/ostree/api/v1/http.go +++ b/pkg/plugins/ostree/api/v1/http.go @@ -24,13 +24,69 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio var validator httpoption.Validator var kitOptions []kithttp.ServerOption - codec = codecs.EncodeDecoder("MirrorRepository") - validator = options.RequestValidator("MirrorRepository") + codec = codecs.EncodeDecoder("AddRemote") + validator = options.RequestValidator("AddRemote") r.Method( - "POST", "/repository/mirror", + "POST", "/repository/remote:add", kithttp.NewServer( - MakeEndpointOfMirrorRepository(svc), - decodeMirrorRepositoryRequest(codec, validator), + MakeEndpointOfAddRemote(svc), + decodeAddRemoteRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("CreateRepository") + validator = options.RequestValidator("CreateRepository") + r.Method( + "POST", "/repository", + kithttp.NewServer( + MakeEndpointOfCreateRepository(svc), + decodeCreateRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("DeleteRepository") + validator = options.RequestValidator("DeleteRepository") + r.Method( + "DELETE", "/repository", + kithttp.NewServer( + MakeEndpointOfDeleteRepository(svc), + decodeDeleteRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("GetRepositorySyncStatus") + validator = options.RequestValidator("GetRepositorySyncStatus") + r.Method( + "GET", "/repository/sync:status", + kithttp.NewServer( + MakeEndpointOfGetRepositorySyncStatus(svc), + decodeGetRepositorySyncStatusRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("SyncRepository") + validator = options.RequestValidator("SyncRepository") + r.Method( + "POST", "/repository/sync", + kithttp.NewServer( + MakeEndpointOfSyncRepository(svc), + decodeSyncRepositoryRequest(codec, validator), httpcodec.MakeResponseEncoder(codec, 200), append(kitOptions, kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), @@ -41,9 +97,73 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio return r } -func decodeMirrorRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { +func decodeAddRemoteRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req AddRemoteRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeCreateRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req CreateRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeDeleteRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req DeleteRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeGetRepositorySyncStatusRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req GetRepositorySyncStatusRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeSyncRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { return func(_ context.Context, r *http.Request) (interface{}, error) { - var _req MirrorRepositoryRequest + var _req SyncRepositoryRequest if err := codec.DecodeRequestBody(r, &_req); err != nil { return nil, err diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go index 2d50a80..7ba8884 100644 --- a/pkg/plugins/ostree/api/v1/http_client.go +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -34,10 +34,59 @@ func NewHTTPClient(codecs httpcodec.Codecs, httpClient *http.Client, baseURL str }, nil } -func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) { - codec := c.codecs.EncodeDecoder("MirrorRepository") +func (c *HTTPClient) AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) { + codec := c.codecs.EncodeDecoder("AddRemote") - path := "/repository/mirror" + path := "/repository/remote:add" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Properties *OSTreeRemoteProperties `json:"properties"` + }{ + Repository: repository, + Properties: properties, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + +func (c *HTTPClient) CreateRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) { + codec := c.codecs.EncodeDecoder("CreateRepository") + + path := "/repository" u := &url.URL{ Scheme: c.scheme, Host: c.host, @@ -82,3 +131,151 @@ func (c *HTTPClient) MirrorRepository(ctx context.Context, repository string, pr return nil } + +func (c *HTTPClient) DeleteRepository(ctx context.Context, repository string) (err error) { + codec := c.codecs.EncodeDecoder("DeleteRepository") + + path := "/repository" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + }{ + Repository: repository, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "DELETE", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + +func (c *HTTPClient) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *SyncStatus, err error) { + codec := c.codecs.EncodeDecoder("GetRepositorySyncStatus") + + path := "/repository/sync:status" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + }{ + Repository: repository, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return nil, err + } + + _req, err := http.NewRequestWithContext(ctx, "GET", u.String(), reqBodyReader) + if err != nil { + return nil, err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return nil, err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return nil, err + } + + respBody := &GetRepositorySyncStatusResponse{} + err = codec.DecodeSuccessResponse(_resp.Body, respBody.Body()) + if err != nil { + return nil, err + } + return respBody.SyncStatus, nil +} + +func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, request *OSTreeRepositorySyncRequest) (err error) { + codec := c.codecs.EncodeDecoder("SyncRepository") + + path := "/repository/sync" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Request *OSTreeRepositorySyncRequest `json:"request"` + }{ + Repository: repository, + Request: request, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go index 369a082..222f048 100644 --- a/pkg/plugins/ostree/api/v1/oas2.go +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -29,35 +29,108 @@ produces: paths = ` paths: - /repository/mirror: + /repository/remote:add: + post: + description: "Add a new remote to the OSTree repository." + operationId: "AddRemote" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/AddRemoteRequestBody" + %s + /repository: + post: + description: "Create an OSTree repository." + operationId: "CreateRepository" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/CreateRepositoryRequestBody" + %s + delete: + description: "Delete a OSTree repository." + operationId: "DeleteRepository" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/DeleteRepositoryRequestBody" + %s + /repository/sync:status: + get: + description: "Get YUM repository sync status." + operationId: "GetRepositorySyncStatus" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/GetRepositorySyncStatusRequestBody" + %s + /repository/sync: post: description: "Mirror an ostree repository." - operationId: "MirrorRepository" + operationId: "SyncRepository" tags: - ostree parameters: - name: body in: body schema: - $ref: "#/definitions/MirrorRepositoryRequestBody" + $ref: "#/definitions/SyncRepositoryRequestBody" %s ` ) func getResponses(schema oas2.Schema) []oas2.OASResponses { return []oas2.OASResponses{ - oas2.GetOASResponses(schema, "MirrorRepository", 200, &MirrorRepositoryResponse{}), + oas2.GetOASResponses(schema, "AddRemote", 200, &AddRemoteResponse{}), + oas2.GetOASResponses(schema, "CreateRepository", 200, &CreateRepositoryResponse{}), + oas2.GetOASResponses(schema, "DeleteRepository", 200, &DeleteRepositoryResponse{}), + oas2.GetOASResponses(schema, "GetRepositorySyncStatus", 200, &GetRepositorySyncStatusResponse{}), + oas2.GetOASResponses(schema, "SyncRepository", 200, &SyncRepositoryResponse{}), } } func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { defs := make(map[string]oas2.Definition) - oas2.AddDefinition(defs, "MirrorRepositoryRequestBody", reflect.ValueOf(&struct { + oas2.AddDefinition(defs, "AddRemoteRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Properties *OSTreeRemoteProperties `json:"properties"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "AddRemote", 200, (&AddRemoteResponse{}).Body()) + + oas2.AddDefinition(defs, "CreateRepositoryRequestBody", reflect.ValueOf(&struct { Repository string `json:"repository"` Properties *OSTreeRepositoryProperties `json:"properties"` }{})) - oas2.AddResponseDefinitions(defs, schema, "MirrorRepository", 200, (&MirrorRepositoryResponse{}).Body()) + oas2.AddResponseDefinitions(defs, schema, "CreateRepository", 200, (&CreateRepositoryResponse{}).Body()) + + oas2.AddDefinition(defs, "DeleteRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "DeleteRepository", 200, (&DeleteRepositoryResponse{}).Body()) + + oas2.AddDefinition(defs, "GetRepositorySyncStatusRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "GetRepositorySyncStatus", 200, (&GetRepositorySyncStatusResponse{}).Body()) + + oas2.AddDefinition(defs, "SyncRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Request *OSTreeRepositorySyncRequest `json:"request"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "SyncRepository", 200, (&SyncRepositoryResponse{}).Body()) return defs } diff --git a/pkg/utils/time.go b/pkg/utils/time.go new file mode 100644 index 0000000..3e29d4f --- /dev/null +++ b/pkg/utils/time.go @@ -0,0 +1,12 @@ +package utils + +import "time" + +const timeFormat = time.DateTime + " MST" + +func TimeToString(t int64) string { + if t == 0 { + return "" + } + return time.Unix(t, 0).Format(timeFormat) +} From 9478fb430c9628eb20e88d3019787bfc1e4b27bf Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 11 Jan 2024 22:09:48 -0500 Subject: [PATCH 19/30] fixes various bugs discovered after manual testing renames ostree to libostree to better represent what it does implements GCancellable to allow c functions to respect go context.Context --- cmd/beskarctl/ostree/{root.go => file.go} | 0 cmd/beskarctl/ostree/{push.go => repo.go} | 7 +- internal/pkg/repository/handler.go | 8 ++ internal/plugins/ostree/api.go | 4 +- .../plugins/ostree/pkg/libostree/README.md | 2 +- .../plugins/ostree/pkg/libostree/errors.go | 2 +- .../ostree/pkg/libostree/glib_helpers.go | 6 +- .../plugins/ostree/pkg/libostree/options.go | 2 +- .../plugins/ostree/pkg/libostree/ostree.go | 2 +- internal/plugins/ostree/pkg/libostree/pull.go | 18 ++- .../plugins/ostree/pkg/libostree/pull_test.go | 17 ++- internal/plugins/ostree/pkg/libostree/repo.go | 4 +- .../ostree/pkg/ostreerepository/api.go | 133 ++++++++++-------- .../{ostreerepository.go => handler.go} | 36 ++++- .../ostree/pkg/ostreerepository/local.go | 122 +++++++++++----- .../ostree/pkg/ostreerepository/sync.go | 18 +-- pkg/orasostree/ostree.go | 2 +- pkg/orasostree/push.go | 71 +++++++--- pkg/plugins/ostree/api/v1/api.go | 16 +-- pkg/plugins/ostree/api/v1/endpoint.go | 4 +- pkg/plugins/ostree/api/v1/http.go | 8 +- pkg/plugins/ostree/api/v1/http_client.go | 10 +- pkg/plugins/ostree/api/v1/oas2.go | 19 ++- 23 files changed, 344 insertions(+), 167 deletions(-) rename cmd/beskarctl/ostree/{root.go => file.go} (100%) rename cmd/beskarctl/ostree/{push.go => repo.go} (66%) rename internal/plugins/ostree/pkg/ostreerepository/{ostreerepository.go => handler.go} (74%) diff --git a/cmd/beskarctl/ostree/root.go b/cmd/beskarctl/ostree/file.go similarity index 100% rename from cmd/beskarctl/ostree/root.go rename to cmd/beskarctl/ostree/file.go diff --git a/cmd/beskarctl/ostree/push.go b/cmd/beskarctl/ostree/repo.go similarity index 66% rename from cmd/beskarctl/ostree/push.go rename to cmd/beskarctl/ostree/repo.go index 2006703..fb339ac 100644 --- a/cmd/beskarctl/ostree/push.go +++ b/cmd/beskarctl/ostree/repo.go @@ -5,7 +5,9 @@ package ostree import ( "context" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/cobra" "go.ciq.dev/beskar/cmd/beskarctl/ctl" "go.ciq.dev/beskar/pkg/orasostree" @@ -22,7 +24,10 @@ var ( return ctl.Err("a directory must be specified") } - if err := orasostree.PushOSTreeRepository(context.Background(), dir, ctl.Repo(), jobCount, name.WithDefaultRegistry(ctl.Registry())); err != nil { + repoPusher := orasostree.NewOSTreeRepositoryPusher(context.Background(), dir, ctl.Repo(), jobCount) + repoPusher = repoPusher.WithNameOptions(name.WithDefaultRegistry(ctl.Registry())) + repoPusher = repoPusher.WithRemoteOptions(remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err := repoPusher.Push(); err != nil { return ctl.Errf("while pushing ostree repository: %s", err) } return nil diff --git a/internal/pkg/repository/handler.go b/internal/pkg/repository/handler.go index df076f4..10b9eca 100644 --- a/internal/pkg/repository/handler.go +++ b/internal/pkg/repository/handler.go @@ -194,6 +194,14 @@ func (rh *RepoHandler) DeleteManifest(ref string) (errFn error) { return remote.Delete(namedRef, rh.Params.RemoteOptions...) } +func (rh *RepoHandler) PullManifest(ref string) (errFn error) { + namedRef, err := name.ParseReference(ref, rh.Params.NameOptions...) + if err != nil { + return err + } + return remote.Delete(namedRef, rh.Params.RemoteOptions...) +} + func (rh *RepoHandler) SyncArtifact(ctx context.Context, name string, timeout time.Duration) (chan error, func() error) { errCh := make(chan error, 1) diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index adecfa2..c605019 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -42,12 +42,12 @@ func (p *Plugin) AddRemote(ctx context.Context, repository string, properties *a return p.repositoryManager.Get(ctx, repository).AddRemote(ctx, properties) } -func (p *Plugin) SyncRepository(ctx context.Context, repository string, request *apiv1.OSTreeRepositorySyncRequest) (err error) { +func (p *Plugin) SyncRepository(ctx context.Context, repository string, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { if err := checkRepository(repository); err != nil { return err } - return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, request) + return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, properties) } func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { diff --git a/internal/plugins/ostree/pkg/libostree/README.md b/internal/plugins/ostree/pkg/libostree/README.md index 14186b8..7c2295e 100644 --- a/internal/plugins/ostree/pkg/libostree/README.md +++ b/internal/plugins/ostree/pkg/libostree/README.md @@ -7,7 +7,7 @@ ostree is a wrapper around [libostree](https://github.com/ostreedev/ostree) that 1. A minimal glib implementation exists within the ostree pkg. This is to avoid a dependency on glib for the time being. - This implementation is not complete and will be expanded as needed. - The glib implementation is not intended to be used outside of the ostree pkg. - - `GCancellable` is not implemented. Just send nil. + - `GCancellable` is not implemented on some functions. If the func accepts a context.Context it most likely implements a GCancellable. 2. Not all of libostree is wrapped. Only the parts that are needed for beskar are wrapped. Which is basically everything need to perform pull operations. - `OstreeAsyncProgress` is not implemented. Just send nil. diff --git a/internal/plugins/ostree/pkg/libostree/errors.go b/internal/plugins/ostree/pkg/libostree/errors.go index 43b7f9e..0151579 100644 --- a/internal/plugins/ostree/pkg/libostree/errors.go +++ b/internal/plugins/ostree/pkg/libostree/errors.go @@ -1,4 +1,4 @@ -package ostree +package libostree type Err string diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go b/internal/plugins/ostree/pkg/libostree/glib_helpers.go index 12c1a29..8c1fb39 100644 --- a/internal/plugins/ostree/pkg/libostree/glib_helpers.go +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go @@ -1,4 +1,4 @@ -package ostree +package libostree // #cgo pkg-config: glib-2.0 gobject-2.0 // #include @@ -7,7 +7,9 @@ package ostree // #include // #include "glib_helpers.go.h" import "C" -import "errors" +import ( + "errors" +) // GoError converts a C glib error to a Go error. // The C error is freed after conversion. diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go index cc9eb6d..8df0a81 100644 --- a/internal/plugins/ostree/pkg/libostree/options.go +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -1,4 +1,4 @@ -package ostree +package libostree // #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 // #include diff --git a/internal/plugins/ostree/pkg/libostree/ostree.go b/internal/plugins/ostree/pkg/libostree/ostree.go index 6f66afd..67201a1 100644 --- a/internal/plugins/ostree/pkg/libostree/ostree.go +++ b/internal/plugins/ostree/pkg/libostree/ostree.go @@ -1 +1 @@ -package ostree +package libostree diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go index 0bf6801..c7157ad 100644 --- a/internal/plugins/ostree/pkg/libostree/pull.go +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -1,4 +1,4 @@ -package ostree +package libostree // #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 // #include @@ -13,7 +13,7 @@ import ( // Pull pulls refs from the named remote. // Returns an error if the refs could not be fetched. -func (r *Repo) Pull(_ context.Context, remote string, opts ...Option) error { +func (r *Repo) Pull(ctx context.Context, remote string, opts ...Option) error { cremote := C.CString(remote) defer C.free(unsafe.Pointer(cremote)) @@ -22,14 +22,24 @@ func (r *Repo) Pull(_ context.Context, remote string, opts ...Option) error { var cErr *C.GError + cCancel := C.g_cancellable_new() + go func() { + for { + select { + case <-ctx.Done(): + C.g_cancellable_cancel(cCancel) + return + } + } + }() + // Pull refs from remote - // TODO: Implement cancellable so that we can cancel the pull if needed. if C.ostree_repo_pull_with_options( r.native, cremote, options, nil, - nil, + cCancel, &cErr, ) == C.gboolean(0) { return GoError(cErr) diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index 0bb56e8..26f4262 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -1,4 +1,4 @@ -package ostree +package libostree import ( "context" @@ -103,6 +103,21 @@ func TestRepo_Pull(t *testing.T) { assert.Equal(t, remotes, []string{remoteName}) }) + t.Run("should cancel pull", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := repo.Pull( + ctx, + remoteName, + Flags(Mirror|TrustedHttp), + ) + assert.Error(t, err) + if err == nil { + assert.Failf(t, "failed to cancel pull", "err: %s", err.Error()) + } + }) + //TODO: Repeat the following tests for only a specific ref t.Run("should pull entire repo", func(t *testing.T) { err := repo.Pull( diff --git a/internal/plugins/ostree/pkg/libostree/repo.go b/internal/plugins/ostree/pkg/libostree/repo.go index a71fedc..8498f81 100644 --- a/internal/plugins/ostree/pkg/libostree/repo.go +++ b/internal/plugins/ostree/pkg/libostree/repo.go @@ -1,4 +1,4 @@ -package ostree +package libostree // #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 // #include @@ -85,7 +85,7 @@ func fromNative(cRepo *C.OstreeRepo) *Repo { // Let the GB trigger free the cRepo for us when repo is freed. runtime.SetFinalizer(repo, func(r *Repo) { - C.free(unsafe.Pointer(cRepo)) + C.free(unsafe.Pointer(r.native)) }) return repo diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go index 4b3c8bd..52458fd 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/api.go +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -2,7 +2,6 @@ package ostreerepository import ( "context" - "errors" "fmt" "go.ciq.dev/beskar/cmd/beskarctl/ctl" "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" @@ -12,13 +11,7 @@ import ( "golang.org/x/sync/errgroup" "os" "path/filepath" -) - -var ( - errHandlerNotStarted = errors.New("handler not started") - errProvisionInProgress = errors.New("provision in progress") - errSyncInProgress = errors.New("sync in progress") - errDeleteInProgress = errors.New("delete in progress") + "time" ) func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTreeRepositoryProperties) (err error) { @@ -28,37 +21,38 @@ func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTree return ctl.Errf("at least one remote is required") } + // Check if repo already exists + if h.checkRepoExists(ctx) { + return ctl.Err("repository already exists") + } + // Transition to provisioning state if err := h.setState(StateProvisioning); err != nil { return err } defer h.clearState() - // Check if repo already exists - if h.checkRepoExists(ctx) { - return ctl.Errf("repository %s already exists", h.Repository) - } - - if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { // Add user provided remotes // We do not need to add beskar remote here for _, remote := range properties.Remotes { - var opts []ostree.Option + var opts []libostree.Option if remote.NoGPGVerify { - opts = append(opts, ostree.NoGPGVerify()) + opts = append(opts, libostree.NoGPGVerify()) } if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { - return ctl.Errf("while adding remote to ostree repository %s: %s", remote.Name, err) + return false, ctl.Errf("adding remote to ostree repository %s: %s", remote.Name, err) } } - return nil - }); err != nil { - return err - } + if err := repo.RegenerateSummary(); err != nil { + return false, ctl.Errf("regenerating summary for ostree repository %s: %s", h.repoDir, err) + } - return nil + return true, nil + + }, SkipPull()) } // DeleteRepository deletes the repository from beskar and the local filesystem. @@ -71,21 +65,22 @@ func (h *Handler) DeleteRepository(ctx context.Context) (err error) { } // Check if repo already exists - if h.checkRepoExists(ctx) { - return ctl.Errf("repository %s already exists", h.Repository) + if !h.checkRepoExists(ctx) { + defer h.clearState() + return ctl.Err("repository does not exist") } go func() { defer func() { - h.clearState() if err == nil { // stop the repo handler and trigger cleanup h.Stop() } + h.clearState() }() - h.logger.Debug("deleting repository", "repository", h.Repository) + h.logger.Debug("deleting repository") - if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + err := h.BeginLocalRepoTransaction(context.Background(), func(ctx context.Context, repo *libostree.Repo) (bool, error) { // Create a worker pool to deleting each file in the repository concurrently. // ctx will be cancelled on error, and the error will be returned. @@ -96,7 +91,7 @@ func (h *Handler) DeleteRepository(ctx context.Context) (err error) { if err := filepath.WalkDir(h.repoDir, func(path string, d os.DirEntry, err error) error { // If there was an error with the file, return it. if err != nil { - return fmt.Errorf("while walking %s: %w", path, err) + return fmt.Errorf("walking %s: %w", path, err) } // Skip directories. if d.IsDir() { @@ -113,10 +108,11 @@ func (h *Handler) DeleteRepository(ctx context.Context) (err error) { eg.Go(func() error { // Delete the file from the repository filename := filepath.Base(path) + h.logger.Debug("deleting file from beskar", "file", filename) digest := orasostree.MakeTag(filename) digestRef := filepath.Join(h.Repository, "file:"+digest) if err := h.DeleteManifest(digestRef); err != nil { - h.logger.Error("deleting file from beskar", "repository", h.Repository, "error", err.Error()) + h.logger.Error("deleting file from beskar", "error", err.Error()) } return nil @@ -124,13 +120,16 @@ func (h *Handler) DeleteRepository(ctx context.Context) (err error) { return nil }); err != nil { - return nil + return false, err } - return eg.Wait() + // We don't want to push any changes to beskar. + return false, eg.Wait() + + }) - }); err != nil { - h.logger.Error("while deleting repository", "repository", h.Repository, "error", err.Error()) + if err != nil { + h.logger.Error("deleting repository", "error", err.Error()) } }() @@ -144,30 +143,27 @@ func (h *Handler) AddRemote(ctx context.Context, remote *apiv1.OSTreeRemotePrope } defer h.clearState() - if h.checkRepoExists(ctx) { - return ctl.Errf("repository %s does not exist", h.Repository) + if !h.checkRepoExists(ctx) { + return ctl.Errf("repository does not exist") } - if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { // Add user provided remote - var opts []ostree.Option + var opts []libostree.Option if remote.NoGPGVerify { - opts = append(opts, ostree.NoGPGVerify()) + opts = append(opts, libostree.NoGPGVerify()) } if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { - return ctl.Errf("while adding remote to ostree repository %s: %s", remote.Name, err) + // No need to make error pretty, it is already pretty + return false, err } - return nil - - }); err != nil { - return err - } + return true, nil - return nil + }, SkipPull()) } -func (h *Handler) SyncRepository(ctx context.Context, request *apiv1.OSTreeRepositorySyncRequest) (err error) { +func (h *Handler) SyncRepository(ctx context.Context, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { // Transition to syncing state if err := h.setState(StateSyncing); err != nil { return err @@ -175,37 +171,56 @@ func (h *Handler) SyncRepository(ctx context.Context, request *apiv1.OSTreeRepos // Spin up pull worker go func() { + h.logger.Debug("syncing repository") + + var err error defer func() { + if err != nil { + h.logger.Error("repository sync failed", "properties", properties, "error", err.Error()) + repoSync := *h.repoSync.Load() + repoSync.SyncError = err.Error() + h.setRepoSync(&repoSync) + } else { + h.logger.Debug("repository sync complete", "properties", properties) + } h.clearState() - h.logger.Debug("repository sync complete", "repository", h.Repository, "request", request) }() - if err := h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *ostree.Repo) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + err = h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { // Pull the latest changes from the remote. - opts := []ostree.Option{ - ostree.Depth(request.Depth), + opts := []libostree.Option{ + libostree.Depth(properties.Depth), + libostree.Flags(libostree.Mirror | libostree.TrustedHttp), } - if len(request.Refs) > 0 { - opts = append(opts, ostree.Refs(request.Refs...)) + if len(properties.Refs) > 0 { + opts = append(opts, libostree.Refs(properties.Refs...)) } // pull remote content into local repo - if err := repo.Pull(ctx, request.Remote, opts...); err != nil { - return ctl.Errf("while pulling ostree repository from %s: %s", request.Remote, err) + if err := repo.Pull(ctx, properties.Remote, opts...); err != nil { + return false, ctl.Errf("pulling ostree repository: %s", err) } - return nil + if err := repo.RegenerateSummary(); err != nil { + return false, ctl.Errf("regenerating summary for ostree repository %s: %s", h.repoDir, err) + } - }); err != nil { - h.logger.Error("repository sync", "repository", h.Repository, "request", request, "error", err.Error()) - } + return true, nil + + }) }() return nil } func (h *Handler) GetRepositorySyncStatus(_ context.Context) (syncStatus *apiv1.SyncStatus, err error) { - repoSync := h.getRepoSync() + repoSync := h.repoSync.Load() + if repoSync == nil { + return nil, ctl.Errf("repository sync status not available") + } return &apiv1.SyncStatus{ Syncing: repoSync.Syncing, StartTime: utils.TimeToString(repoSync.StartTime), diff --git a/internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go b/internal/plugins/ostree/pkg/ostreerepository/handler.go similarity index 74% rename from internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go rename to internal/plugins/ostree/pkg/ostreerepository/handler.go index db68b5f..d836a77 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/ostreerepository.go +++ b/internal/plugins/ostree/pkg/ostreerepository/handler.go @@ -5,10 +5,14 @@ import ( "fmt" "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" "go.ciq.dev/beskar/internal/pkg/repository" eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" + "io" "log/slog" + "net/http" "os" + "path" "path/filepath" "sync" "sync/atomic" @@ -70,8 +74,8 @@ func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handl func (h *Handler) setState(state State) error { current := h.getState() - if current != StateReady { - return werror.Wrap(gcode.ErrUnavailable, fmt.Errorf("repository is not ready: %s", current)) + if current != StateReady && state != StateReady { + return werror.Wrap(gcode.ErrUnavailable, fmt.Errorf("repository is busy: %s", current)) } h._state.Swap(int32(state)) if state == StateSyncing || current == StateSyncing { @@ -82,6 +86,7 @@ func (h *Handler) setState(state State) error { func (h *Handler) clearState() { h._state.Swap(int32(StateReady)) + h.updateSyncing(false) } func (h *Handler) getState() State { @@ -108,6 +113,7 @@ func (h *Handler) QueueEvent(_ *eventv1.EventPayload, _ bool) error { func (h *Handler) Start(ctx context.Context) { h.logger.Debug("starting repository", "repository", h.Repository) h.clearState() + go func() { for !h.Stopped.Load() { select { @@ -118,3 +124,29 @@ func (h *Handler) Start(ctx context.Context) { h.cleanup() }() } + +// pullConfig pulls the config file from beskar. +func (h *Handler) pullFile(_ context.Context, filename string) error { + //TODO: Replace with appropriate puller mechanism + url := "http://" + h.Params.GetBeskarRegistryHostPort() + path.Join("/", h.Repository, "repo", filename) + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check Content-Length + if resp.ContentLength <= 0 { + return ctl.Errf("content-length is 0") + } + + // Create the file + out, err := os.Create(path.Join(h.repoDir, filename)) + if err != nil { + return err + } + + // Write the body to file + _, err = io.Copy(out, resp.Body) + return err +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/local.go b/internal/plugins/ostree/pkg/ostreerepository/local.go index bf1fc8d..8a0267a 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/local.go +++ b/internal/plugins/ostree/pkg/ostreerepository/local.go @@ -6,6 +6,7 @@ import ( "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" "go.ciq.dev/beskar/pkg/orasostree" "os" + "path" "path/filepath" ) @@ -18,7 +19,34 @@ func (h *Handler) checkRepoExists(_ context.Context) bool { return err == nil } -func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, transactor func(ctx context.Context, repo *ostree.Repo) error) error { +type TransactionFn func(ctx context.Context, repo *libostree.Repo) (commit bool, err error) +type TransactionOptions struct { + skipPull bool +} + +type TransactionOption func(*TransactionOptions) + +func SkipPull() TransactionOption { + return func(opts *TransactionOptions) { + opts.skipPull = true + } +} + +// BeginLocalRepoTransaction executes a transaction against the local ostree repository. +// The transaction is executed in a temporary directory in which the following steps are performed: +// 1. The local ostree repository is opened. +// 2. The beskar remote is added to the local ostree repository. +// 3. The beskar version of the repo is pulled into the local ostree repository. +// 4. The transactorFn is executed. +// 5. If the transactorFn returns true, the local ostree repository is pushed to beskar. If false, all local changes are discarded. +// 6. The temporary directory is removed. +func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, tFn TransactionFn, opts ...TransactionOption) error { + + options := TransactionOptions{} + for _, opt := range opts { + opt(&options) + } + // We control the local repo lifecycle here, so we need to lock it. h.repoLock.Lock() defer h.repoLock.Unlock() @@ -32,54 +60,80 @@ func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, transactor func } } + // Clean up the disk when we are done + defer func() { + if err := os.RemoveAll(h.repoDir); err != nil { + h.logger.Error("removing local repo", "repo", h.repoDir, "error", err) + } + }() + // We will always use archive mode here - repo, err := ostree.Init(h.repoDir, ostree.RepoModeArchive) + // Note that we are not using the returned repo pointer here. We will re-open the repo later. + _, err := libostree.Init(h.repoDir, libostree.RepoModeArchive) if err != nil { - return ctl.Errf("while opening ostree repository %s: %s", h.repoDir, err) + return ctl.Errf("initializing ostree repository %s: %s", h.repoDir, err) } - // Ad beskar as a remote so that we can pull from it - beskarServiceURL := h.Params.GetBeskarServiceHostPort() - if err := repo.AddRemote(beskarRemoteName, beskarServiceURL, ostree.NoGPGVerify()); err != nil { - return ctl.Errf("while adding remote to ostree repository %s: %s", beskarRemoteName, err) + // It is necessary to pull the config from beskar before we can add the beskar remote. This is because config files + // are unique the instance of a repo you are interacting with. Meaning, remotes are not pulled with the repo's data. + if err := h.pullFile(ctx, orasostree.FileConfig); err != nil { + h.logger.Debug("no config found in beskar", "error", err) } - // pull remote content into local repo from beskar - if h.checkRepoExists(ctx) { - if err := repo.Pull(ctx, beskarRemoteName, ostree.NoGPGVerify()); err != nil { - return ctl.Errf("while pulling ostree repository from %s: %s", beskarRemoteName, err) - } + // Re-open the local repo + // We need to re-open the repo here because we just pulled the config from beskar. If we don't re-open the repo, the + // config we just manually pulled down will not be loaded into memory. + repo, err := libostree.Open(h.repoDir) + if err != nil { + return ctl.Errf("opening ostree repository %s: %s", h.repoDir, err) } - // Execute the transaction - if err := transactor(ctx, repo); err != nil { - return ctl.Errf("while executing transaction: %s", err) + // Add beskar as a remote so that we can pull from it + beskarServiceURL := "http://" + h.Params.GetBeskarRegistryHostPort() + path.Join("/", h.Repository, "repo") + if err := repo.AddRemote(beskarRemoteName, beskarServiceURL, libostree.NoGPGVerify()); err != nil { + return ctl.Errf("adding remote to ostree repository %s: %s", beskarRemoteName, err) } - if err := repo.RegenerateSummary(); err != nil { - return ctl.Errf("while regenerating summary for ostree repository %s: %s", h.repoDir, err) + // pull remote content into local repo from beskar + if !options.skipPull && h.checkRepoExists(ctx) { + if err := repo.Pull( + ctx, + beskarRemoteName, + libostree.NoGPGVerify(), + libostree.Flags(libostree.Mirror|libostree.TrustedHttp), + ); err != nil { + return ctl.Errf("pulling ostree repository from %s: %s", beskarRemoteName, err) + } } - // Remove the internal beskar remote so that external clients can't pull from it, not that it would work. - if err := repo.DeleteRemote(beskarRemoteName); err != nil { - return ctl.Errf("while deleting remote %s: %s", beskarRemoteName, err) + // Execute the transaction + commit, err := tFn(ctx, repo) + if err != nil { + return ctl.Errf("executing transaction: %s", err) } - // Close the local - // Push local repo to beskar using OSTreePusher - if err := orasostree.PushOSTreeRepository( - ctx, - h.repoDir, - h.Repository, - 100, - h.Params.NameOptions..., - ); err != nil { - return err - } + // Commit the changes to beskar if the transaction deems it necessary + if commit { - // Clean up the disk - if err := os.RemoveAll(h.repoDir); err != nil { - return ctl.Errf("while removing local repo %s: %s", h.repoDir, err) + // Remove the internal beskar remote so that external clients can't pull from it, not that it would work. + if err := repo.DeleteRemote(beskarRemoteName); err != nil { + return ctl.Errf("deleting remote %s: %s", beskarRemoteName, err) + } + + // Close the local + // Push local repo to beskar using OSTreePusher + repoPusher := orasostree.NewOSTreeRepositoryPusher( + ctx, + h.repoDir, + h.Repository, + 100, + ) + repoPusher = repoPusher.WithLogger(h.logger) + repoPusher = repoPusher.WithNameOptions(h.Params.NameOptions...) + repoPusher = repoPusher.WithRemoteOptions(h.Params.RemoteOptions...) + if err := repoPusher.Push(); err != nil { + return ctl.Errf("pushing ostree repository: %s", err) + } } return nil diff --git a/internal/plugins/ostree/pkg/ostreerepository/sync.go b/internal/plugins/ostree/pkg/ostreerepository/sync.go index 9b87f59..8abecc4 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/sync.go +++ b/internal/plugins/ostree/pkg/ostreerepository/sync.go @@ -5,14 +5,10 @@ import ( ) type RepoSync struct { - Syncing bool `db:"syncing"` - StartTime int64 `db:"start_time"` - EndTime int64 `db:"end_time"` - SyncError string `db:"sync_error"` -} - -func (h *Handler) getRepoSync() *RepoSync { - return h.repoSync.Load() + Syncing bool + StartTime int64 + EndTime int64 + SyncError string } func (h *Handler) setRepoSync(repoSync *RepoSync) { @@ -21,7 +17,11 @@ func (h *Handler) setRepoSync(repoSync *RepoSync) { } func (h *Handler) updateSyncing(syncing bool) *RepoSync { - repoSync := *h.getRepoSync() + if h.repoSync.Load() == nil { + h.repoSync.Store(&RepoSync{}) + } + + repoSync := *h.repoSync.Load() previousSyncing := repoSync.Syncing repoSync.Syncing = syncing if syncing && !previousSyncing { diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go index 1b04bb7..0d433aa 100644 --- a/pkg/orasostree/ostree.go +++ b/pkg/orasostree/ostree.go @@ -25,7 +25,7 @@ const ( FileConfig = "config" ) -func NewOSTreePusher(repoRootDir, path, repo string, opts ...name.Option) (oras.Pusher, error) { +func NewOSTreeFilePusher(repoRootDir, path, repo string, opts ...name.Option) (oras.Pusher, error) { if !strings.HasPrefix(repo, ArtifactsPathPrefix+"/") { if !strings.HasPrefix(repo, OSTreePathPrefix+"/") { repo = filepath.Join(OSTreePathPrefix, repo) diff --git a/pkg/orasostree/push.go b/pkg/orasostree/push.go index b69fcc4..110b9e8 100644 --- a/pkg/orasostree/push.go +++ b/pkg/orasostree/push.go @@ -3,38 +3,72 @@ package orasostree import ( "context" "fmt" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" "go.ciq.dev/beskar/pkg/oras" "golang.org/x/sync/errgroup" + "log/slog" "os" "path/filepath" "strings" ) -// PushOSTreeRepository walks a local ostree repository and pushes each file to the given registry. +type OSTreeRepositoryPusher struct { + ctx context.Context + dir string + repo string + jobCount int + nameOpts []name.Option + remoteOpts []remote.Option + logger *slog.Logger +} + +func NewOSTreeRepositoryPusher(ctx context.Context, dir, repo string, jobCount int) *OSTreeRepositoryPusher { + return &OSTreeRepositoryPusher{ + ctx: ctx, + dir: dir, + repo: repo, + jobCount: jobCount, + } +} + +func (p *OSTreeRepositoryPusher) WithNameOptions(opts ...name.Option) *OSTreeRepositoryPusher { + p.nameOpts = opts + return p +} + +func (p *OSTreeRepositoryPusher) WithRemoteOptions(opts ...remote.Option) *OSTreeRepositoryPusher { + p.remoteOpts = opts + return p +} + +func (p *OSTreeRepositoryPusher) WithLogger(logger *slog.Logger) *OSTreeRepositoryPusher { + p.logger = logger + return p +} + +// Push walks a local ostree repository and pushes each file to the given registry. // dir is the root directory of the ostree repository, i.e., the directory containing the summary file. // repo is the name of the ostree repository. // registry is the registry to push to. -func PushOSTreeRepository(ctx context.Context, dir, repo string, jobCount int, opts ...name.Option) error { +func (p *OSTreeRepositoryPusher) Push() error { // Prove that we were given the root directory of an ostree repository // by checking for the existence of the config file. // Typically, libostree will check for the "objects" directory, but this will do just the same. - fileInfo, err := os.Stat(filepath.Join(dir, FileConfig)) + fileInfo, err := os.Stat(filepath.Join(p.dir, FileConfig)) if os.IsNotExist(err) || fileInfo.IsDir() { - return fmt.Errorf("%s file not found in %s: you may need to call ostree init", FileConfig, dir) + return fmt.Errorf("%s file not found in %s: you may need to call ostree init", FileConfig, p.dir) } else if err != nil { - return fmt.Errorf("error accessing %s in %s: %w", FileConfig, dir, err) + return fmt.Errorf("error accessing %s in %s: %w", FileConfig, p.dir, err) } // Create a worker pool to push each file in the repository concurrently. // ctx will be cancelled on error, and the error will be returned. - eg, ctx := errgroup.WithContext(ctx) - eg.SetLimit(jobCount) + eg, ctx := errgroup.WithContext(p.ctx) + eg.SetLimit(p.jobCount) // Walk the directory tree, skipping directories and pushing each file. - if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err := filepath.WalkDir(p.dir, func(path string, d os.DirEntry, err error) error { // If there was an error with the file, return it. if err != nil { return fmt.Errorf("while walking %s: %w", path, err) @@ -53,7 +87,7 @@ func PushOSTreeRepository(ctx context.Context, dir, repo string, jobCount int, o } eg.Go(func() error { - if err := push(dir, path, repo, opts...); err != nil { + if err := p.push(path); err != nil { return fmt.Errorf("while pushing %s: %w", path, err) } return nil @@ -63,7 +97,7 @@ func PushOSTreeRepository(ctx context.Context, dir, repo string, jobCount int, o }); err != nil { // We should only receive here if filepath.WalkDir() returns an error. // Push errors are handled below. - return fmt.Errorf("while walking %s: %w", dir, err) + return fmt.Errorf("while walking %s: %w", p.dir, err) } // Wait for all workers to finish. @@ -71,15 +105,18 @@ func PushOSTreeRepository(ctx context.Context, dir, repo string, jobCount int, o return eg.Wait() } -func push(repoRootDir, path, repo string, opts ...name.Option) error { - pusher, err := NewOSTreePusher(repoRootDir, path, repo, opts...) +func (p *OSTreeRepositoryPusher) push(path string) error { + pusher, err := NewOSTreeFilePusher(p.dir, path, p.repo, p.nameOpts...) if err != nil { + return fmt.Errorf("while creating OSTree pusher: %w", err) } - path = strings.TrimPrefix(path, repoRootDir) - path = strings.TrimPrefix(path, "/") - fmt.Printf("Pushing %s to %s\n", path, pusher.Reference()) + if p.logger != nil { + path = strings.TrimPrefix(path, p.dir) + path = strings.TrimPrefix(path, "/") + p.logger.Debug("pushing file to beskar", "file", path, "reference", pusher.Reference()) + } - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + return oras.Push(pusher, p.remoteOpts...) } diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index f11be93..40797eb 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -45,7 +45,7 @@ type OSTreeRepositorySyncRequest struct { Remote string `json:"remote"` // Refs - The branches/refs to mirror. Leave empty to mirror all branches/refs. - Refs []string `json:"branch"` + Refs []string `json:"refs"` // Depth - The depth of the mirror. Defaults is 0, -1 means infinite. Depth int `json:"depth"` @@ -80,21 +80,21 @@ type OSTree interface { // Delete a OSTree repository. //kun:op DELETE /repository - //kun:success statusCode=200 + //kun:success statusCode=202 DeleteRepository(ctx context.Context, repository string) (err error) // Add a new remote to the OSTree repository. - //kun:op POST /repository/remote:add + //kun:op POST /repository/remote //kun:success statusCode=200 AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) - // Mirror an ostree repository. + // Sync an ostree repository with one of the configured remotes. //kun:op POST /repository/sync - //kun:success statusCode=200 - SyncRepository(ctx context.Context, repository string, request *OSTreeRepositorySyncRequest) (err error) + //kun:success statusCode=202 + SyncRepository(ctx context.Context, repository string, properties *OSTreeRepositorySyncRequest) (err error) - // Get YUM repository sync status. - //kun:op GET /repository/sync:status + // Get OSTree repository sync status. + //kun:op GET /repository/sync //kun:success statusCode=200 GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *SyncStatus, err error) } diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go index dd7fd1c..cabf265 100644 --- a/pkg/plugins/ostree/api/v1/endpoint.go +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -159,7 +159,7 @@ func MakeEndpointOfGetRepositorySyncStatus(s OSTree) endpoint.Endpoint { type SyncRepositoryRequest struct { Repository string `json:"repository"` - Request *OSTreeRepositorySyncRequest `json:"request"` + Properties *OSTreeRepositorySyncRequest `json:"properties"` } // ValidateSyncRepositoryRequest creates a validator for SyncRepositoryRequest. @@ -186,7 +186,7 @@ func MakeEndpointOfSyncRepository(s OSTree) endpoint.Endpoint { err := s.SyncRepository( ctx, req.Repository, - req.Request, + req.Properties, ) return &SyncRepositoryResponse{ Err: err, diff --git a/pkg/plugins/ostree/api/v1/http.go b/pkg/plugins/ostree/api/v1/http.go index 7feed56..fb8eace 100644 --- a/pkg/plugins/ostree/api/v1/http.go +++ b/pkg/plugins/ostree/api/v1/http.go @@ -27,7 +27,7 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio codec = codecs.EncodeDecoder("AddRemote") validator = options.RequestValidator("AddRemote") r.Method( - "POST", "/repository/remote:add", + "POST", "/repository/remote", kithttp.NewServer( MakeEndpointOfAddRemote(svc), decodeAddRemoteRequest(codec, validator), @@ -59,7 +59,7 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio kithttp.NewServer( MakeEndpointOfDeleteRepository(svc), decodeDeleteRepositoryRequest(codec, validator), - httpcodec.MakeResponseEncoder(codec, 200), + httpcodec.MakeResponseEncoder(codec, 202), append(kitOptions, kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), )..., @@ -69,7 +69,7 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio codec = codecs.EncodeDecoder("GetRepositorySyncStatus") validator = options.RequestValidator("GetRepositorySyncStatus") r.Method( - "GET", "/repository/sync:status", + "GET", "/repository/sync", kithttp.NewServer( MakeEndpointOfGetRepositorySyncStatus(svc), decodeGetRepositorySyncStatusRequest(codec, validator), @@ -87,7 +87,7 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio kithttp.NewServer( MakeEndpointOfSyncRepository(svc), decodeSyncRepositoryRequest(codec, validator), - httpcodec.MakeResponseEncoder(codec, 200), + httpcodec.MakeResponseEncoder(codec, 202), append(kitOptions, kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), )..., diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go index 7ba8884..989ec36 100644 --- a/pkg/plugins/ostree/api/v1/http_client.go +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -37,7 +37,7 @@ func NewHTTPClient(codecs httpcodec.Codecs, httpClient *http.Client, baseURL str func (c *HTTPClient) AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) { codec := c.codecs.EncodeDecoder("AddRemote") - path := "/repository/remote:add" + path := "/repository/remote" u := &url.URL{ Scheme: c.scheme, Host: c.host, @@ -182,7 +182,7 @@ func (c *HTTPClient) DeleteRepository(ctx context.Context, repository string) (e func (c *HTTPClient) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *SyncStatus, err error) { codec := c.codecs.EncodeDecoder("GetRepositorySyncStatus") - path := "/repository/sync:status" + path := "/repository/sync" u := &url.URL{ Scheme: c.scheme, Host: c.host, @@ -231,7 +231,7 @@ func (c *HTTPClient) GetRepositorySyncStatus(ctx context.Context, repository str return respBody.SyncStatus, nil } -func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, request *OSTreeRepositorySyncRequest) (err error) { +func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, properties *OSTreeRepositorySyncRequest) (err error) { codec := c.codecs.EncodeDecoder("SyncRepository") path := "/repository/sync" @@ -243,10 +243,10 @@ func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, requ reqBody := struct { Repository string `json:"repository"` - Request *OSTreeRepositorySyncRequest `json:"request"` + Properties *OSTreeRepositorySyncRequest `json:"properties"` }{ Repository: repository, - Request: request, + Properties: properties, } reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) if err != nil { diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go index 222f048..ba7f80b 100644 --- a/pkg/plugins/ostree/api/v1/oas2.go +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -29,7 +29,7 @@ produces: paths = ` paths: - /repository/remote:add: + /repository/remote: post: description: "Add a new remote to the OSTree repository." operationId: "AddRemote" @@ -64,9 +64,9 @@ paths: schema: $ref: "#/definitions/DeleteRepositoryRequestBody" %s - /repository/sync:status: + /repository/sync: get: - description: "Get YUM repository sync status." + description: "Get OSTree repository sync status." operationId: "GetRepositorySyncStatus" tags: - ostree @@ -76,9 +76,8 @@ paths: schema: $ref: "#/definitions/GetRepositorySyncStatusRequestBody" %s - /repository/sync: post: - description: "Mirror an ostree repository." + description: "Sync an ostree repository with one of the configured remotes." operationId: "SyncRepository" tags: - ostree @@ -95,9 +94,9 @@ func getResponses(schema oas2.Schema) []oas2.OASResponses { return []oas2.OASResponses{ oas2.GetOASResponses(schema, "AddRemote", 200, &AddRemoteResponse{}), oas2.GetOASResponses(schema, "CreateRepository", 200, &CreateRepositoryResponse{}), - oas2.GetOASResponses(schema, "DeleteRepository", 200, &DeleteRepositoryResponse{}), + oas2.GetOASResponses(schema, "DeleteRepository", 202, &DeleteRepositoryResponse{}), oas2.GetOASResponses(schema, "GetRepositorySyncStatus", 200, &GetRepositorySyncStatusResponse{}), - oas2.GetOASResponses(schema, "SyncRepository", 200, &SyncRepositoryResponse{}), + oas2.GetOASResponses(schema, "SyncRepository", 202, &SyncRepositoryResponse{}), } } @@ -119,7 +118,7 @@ func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { oas2.AddDefinition(defs, "DeleteRepositoryRequestBody", reflect.ValueOf(&struct { Repository string `json:"repository"` }{})) - oas2.AddResponseDefinitions(defs, schema, "DeleteRepository", 200, (&DeleteRepositoryResponse{}).Body()) + oas2.AddResponseDefinitions(defs, schema, "DeleteRepository", 202, (&DeleteRepositoryResponse{}).Body()) oas2.AddDefinition(defs, "GetRepositorySyncStatusRequestBody", reflect.ValueOf(&struct { Repository string `json:"repository"` @@ -128,9 +127,9 @@ func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { oas2.AddDefinition(defs, "SyncRepositoryRequestBody", reflect.ValueOf(&struct { Repository string `json:"repository"` - Request *OSTreeRepositorySyncRequest `json:"request"` + Properties *OSTreeRepositorySyncRequest `json:"properties"` }{})) - oas2.AddResponseDefinitions(defs, schema, "SyncRepository", 200, (&SyncRepositoryResponse{}).Body()) + oas2.AddResponseDefinitions(defs, schema, "SyncRepository", 202, (&SyncRepositoryResponse{}).Body()) return defs } From 7876f5541054d394aaee44f483a5c1e64e94da63 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 11 Jan 2024 22:16:26 -0500 Subject: [PATCH 20/30] gofumpt linter fixes --- build/mage/build.go | 2 +- cmd/beskar-ostree/main.go | 3 ++- cmd/beskar-static/main.go | 3 ++- cmd/beskar-yum/main.go | 3 ++- cmd/beskarctl/ostree/repo.go | 1 + internal/pkg/repository/handler.go | 3 ++- internal/plugins/ostree/api.go | 1 + .../plugins/ostree/pkg/libostree/errors.go | 3 +++ .../ostree/pkg/libostree/glib_helpers.go | 4 ++++ .../plugins/ostree/pkg/libostree/options.go | 10 ++++++--- .../plugins/ostree/pkg/libostree/ostree.go | 3 +++ internal/plugins/ostree/pkg/libostree/pull.go | 4 ++++ .../plugins/ostree/pkg/libostree/pull_test.go | 13 +++++++----- internal/plugins/ostree/pkg/libostree/repo.go | 4 ++++ .../ostree/pkg/ostreerepository/api.go | 17 +++++++-------- .../ostree/pkg/ostreerepository/handler.go | 16 ++++++++------ .../ostree/pkg/ostreerepository/local.go | 21 ++++++++++++------- .../ostree/pkg/ostreerepository/sync.go | 3 +++ internal/plugins/ostree/plugin.go | 11 +++++----- .../static/pkg/staticrepository/api.go | 3 ++- internal/plugins/yum/pkg/yumrepository/api.go | 3 ++- pkg/orasostree/push.go | 13 +++++++----- pkg/plugins/ostree/api/v1/api.go | 6 +++--- pkg/utils/time.go | 3 +++ 24 files changed, 101 insertions(+), 52 deletions(-) diff --git a/build/mage/build.go b/build/mage/build.go index 6f94e4e..189801d 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -104,7 +104,7 @@ var binaries = map[string]binaryConfig{ }, useProto: true, // NOTE: restore in case alpine createrepo_c package is broken again - //baseImage: "debian:bullseye-slim", + // baseImage: "debian:bullseye-slim", integrationTest: &integrationTest{ isPlugin: true, envs: map[string]string{ diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go index 4ae53d4..ad93634 100644 --- a/cmd/beskar-ostree/main.go +++ b/cmd/beskar-ostree/main.go @@ -6,12 +6,13 @@ package main import ( "flag" "fmt" - "go.ciq.dev/beskar/internal/plugins/ostree/pkg/ostreerepository" "log" "net" "os" "syscall" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/ostreerepository" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/ostree" "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" diff --git a/cmd/beskar-static/main.go b/cmd/beskar-static/main.go index 9f8e7f4..c870ef2 100644 --- a/cmd/beskar-static/main.go +++ b/cmd/beskar-static/main.go @@ -6,12 +6,13 @@ package main import ( "flag" "fmt" - "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "log" "net" "os" "syscall" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/static" "go.ciq.dev/beskar/internal/plugins/static/pkg/config" diff --git a/cmd/beskar-yum/main.go b/cmd/beskar-yum/main.go index f4b5a6c..76cc659 100644 --- a/cmd/beskar-yum/main.go +++ b/cmd/beskar-yum/main.go @@ -6,12 +6,13 @@ package main import ( "flag" "fmt" - "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "log" "net" "os" "syscall" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/yum" "go.ciq.dev/beskar/internal/plugins/yum/pkg/config" diff --git a/cmd/beskarctl/ostree/repo.go b/cmd/beskarctl/ostree/repo.go index fb339ac..0dcce7d 100644 --- a/cmd/beskarctl/ostree/repo.go +++ b/cmd/beskarctl/ostree/repo.go @@ -5,6 +5,7 @@ package ostree import ( "context" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" diff --git a/internal/pkg/repository/handler.go b/internal/pkg/repository/handler.go index 10b9eca..e5c7331 100644 --- a/internal/pkg/repository/handler.go +++ b/internal/pkg/repository/handler.go @@ -6,7 +6,6 @@ package repository import ( "context" "errors" - "go.ciq.dev/beskar/internal/pkg/gossip" "io" "net" "os" @@ -16,6 +15,8 @@ import ( "sync/atomic" "time" + "go.ciq.dev/beskar/internal/pkg/gossip" + "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index c605019..6a343d9 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -5,6 +5,7 @@ package ostree import ( "context" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" diff --git a/internal/plugins/ostree/pkg/libostree/errors.go b/internal/plugins/ostree/pkg/libostree/errors.go index 0151579..d7e1f18 100644 --- a/internal/plugins/ostree/pkg/libostree/errors.go +++ b/internal/plugins/ostree/pkg/libostree/errors.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree type Err string diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go b/internal/plugins/ostree/pkg/libostree/glib_helpers.go index 8c1fb39..a388095 100644 --- a/internal/plugins/ostree/pkg/libostree/glib_helpers.go +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree // #cgo pkg-config: glib-2.0 gobject-2.0 @@ -7,6 +10,7 @@ package libostree // #include // #include "glib_helpers.go.h" import "C" + import ( "errors" ) diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go index 8df0a81..cc476ed 100644 --- a/internal/plugins/ostree/pkg/libostree/options.go +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree // #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 @@ -13,12 +16,13 @@ import "unsafe" // Option defines an option for pulling ostree repos. // It is used to build a *C.GVariant via a *C.GVariantBuilder. // free is an optional function that frees the memory allocated by the option. free may be called more than once. -type Option func(builder *C.GVariantBuilder, free freeFunc) -type freeFunc func(...unsafe.Pointer) +type ( + Option func(builder *C.GVariantBuilder, free freeFunc) + freeFunc func(...unsafe.Pointer) +) // ToGVariant converts the given Options to a GVariant using a GVaraintBuilder. func toGVariant(opts ...Option) *C.GVariant { - typeStr := (*C.gchar)(C.CString("a{sv}")) defer C.free(unsafe.Pointer(typeStr)) diff --git a/internal/plugins/ostree/pkg/libostree/ostree.go b/internal/plugins/ostree/pkg/libostree/ostree.go index 67201a1..c21ff1c 100644 --- a/internal/plugins/ostree/pkg/libostree/ostree.go +++ b/internal/plugins/ostree/pkg/libostree/ostree.go @@ -1 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go index c7157ad..820f26d 100644 --- a/internal/plugins/ostree/pkg/libostree/pull.go +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree // #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 @@ -6,6 +9,7 @@ package libostree // #include // #include "pull.go.h" import "C" + import ( "context" "unsafe" diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index 26f4262..71235ec 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -1,15 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree import ( "context" "fmt" - "github.com/stretchr/testify/assert" "log" "net/http" "net/http/httptest" "os" "strings" "testing" + + "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { @@ -22,7 +26,6 @@ func TestMain(m *testing.M) { } func TestRepo_Pull(t *testing.T) { - fmt.Println(os.Getwd()) svr := httptest.NewServer(http.FileServer(http.Dir("testdata/repo"))) defer svr.Close() @@ -40,7 +43,7 @@ func TestRepo_Pull(t *testing.T) { RepoModeBare, RepoModeBareUser, RepoModeBareUserOnly, - //RepoModeBareSplitXAttrs, + // RepoModeBareSplitXAttrs, } // Test pull for each mode @@ -118,7 +121,7 @@ func TestRepo_Pull(t *testing.T) { } }) - //TODO: Repeat the following tests for only a specific ref + // TODO: Repeat the following tests for only a specific ref t.Run("should pull entire repo", func(t *testing.T) { err := repo.Pull( context.TODO(), @@ -153,7 +156,7 @@ func TestRepo_Pull(t *testing.T) { for _, ref := range refs { checksum := ref.Checksum assert.NotEmpty(t, checksum) - for sum, _ := range expectedChecksums { + for sum := range expectedChecksums { if sum == checksum { expectedChecksums[sum] = true } diff --git a/internal/plugins/ostree/pkg/libostree/repo.go b/internal/plugins/ostree/pkg/libostree/repo.go index 8498f81..7471970 100644 --- a/internal/plugins/ostree/pkg/libostree/repo.go +++ b/internal/plugins/ostree/pkg/libostree/repo.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package libostree // #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 @@ -7,6 +10,7 @@ package libostree // #include // #include import "C" + import ( "runtime" "unsafe" diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go index 52458fd..72264c2 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/api.go +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -1,17 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostreerepository import ( "context" "fmt" + "os" + "path/filepath" + "time" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" "go.ciq.dev/beskar/pkg/orasostree" apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" "go.ciq.dev/beskar/pkg/utils" "golang.org/x/sync/errgroup" - "os" - "path/filepath" - "time" ) func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTreeRepositoryProperties) (err error) { @@ -33,7 +37,6 @@ func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTree defer h.clearState() return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { - // Add user provided remotes // We do not need to add beskar remote here for _, remote := range properties.Remotes { @@ -51,7 +54,6 @@ func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTree } return true, nil - }, SkipPull()) } @@ -81,7 +83,6 @@ func (h *Handler) DeleteRepository(ctx context.Context) (err error) { h.logger.Debug("deleting repository") err := h.BeginLocalRepoTransaction(context.Background(), func(ctx context.Context, repo *libostree.Repo) (bool, error) { - // Create a worker pool to deleting each file in the repository concurrently. // ctx will be cancelled on error, and the error will be returned. eg, ctx := errgroup.WithContext(ctx) @@ -125,9 +126,7 @@ func (h *Handler) DeleteRepository(ctx context.Context) (err error) { // We don't want to push any changes to beskar. return false, eg.Wait() - }) - if err != nil { h.logger.Error("deleting repository", "error", err.Error()) } @@ -159,7 +158,6 @@ func (h *Handler) AddRemote(ctx context.Context, remote *apiv1.OSTreeRemotePrope } return true, nil - }, SkipPull()) } @@ -209,7 +207,6 @@ func (h *Handler) SyncRepository(ctx context.Context, properties *apiv1.OSTreeRe } return true, nil - }) }() diff --git a/internal/plugins/ostree/pkg/ostreerepository/handler.go b/internal/plugins/ostree/pkg/ostreerepository/handler.go index d836a77..549f001 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/handler.go +++ b/internal/plugins/ostree/pkg/ostreerepository/handler.go @@ -1,13 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostreerepository import ( "context" "fmt" - "github.com/RussellLuo/kun/pkg/werror" - "github.com/RussellLuo/kun/pkg/werror/gcode" - "go.ciq.dev/beskar/cmd/beskarctl/ctl" - "go.ciq.dev/beskar/internal/pkg/repository" - eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" "io" "log/slog" "net/http" @@ -16,6 +14,12 @@ import ( "path/filepath" "sync" "sync/atomic" + + "github.com/RussellLuo/kun/pkg/werror" + "github.com/RussellLuo/kun/pkg/werror/gcode" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/pkg/repository" + eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" ) const ( @@ -127,7 +131,7 @@ func (h *Handler) Start(ctx context.Context) { // pullConfig pulls the config file from beskar. func (h *Handler) pullFile(_ context.Context, filename string) error { - //TODO: Replace with appropriate puller mechanism + // TODO: Replace with appropriate puller mechanism url := "http://" + h.Params.GetBeskarRegistryHostPort() + path.Join("/", h.Repository, "repo", filename) resp, err := http.Get(url) if err != nil { diff --git a/internal/plugins/ostree/pkg/ostreerepository/local.go b/internal/plugins/ostree/pkg/ostreerepository/local.go index 8a0267a..7d7d9ee 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/local.go +++ b/internal/plugins/ostree/pkg/ostreerepository/local.go @@ -1,13 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostreerepository import ( "context" - "go.ciq.dev/beskar/cmd/beskarctl/ctl" - "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" - "go.ciq.dev/beskar/pkg/orasostree" "os" "path" "path/filepath" + + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" + "go.ciq.dev/beskar/pkg/orasostree" ) // checkRepoExists checks if the ostree repository exists in beskar. @@ -19,10 +23,12 @@ func (h *Handler) checkRepoExists(_ context.Context) bool { return err == nil } -type TransactionFn func(ctx context.Context, repo *libostree.Repo) (commit bool, err error) -type TransactionOptions struct { - skipPull bool -} +type ( + TransactionFn func(ctx context.Context, repo *libostree.Repo) (commit bool, err error) + TransactionOptions struct { + skipPull bool + } +) type TransactionOption func(*TransactionOptions) @@ -41,7 +47,6 @@ func SkipPull() TransactionOption { // 5. If the transactorFn returns true, the local ostree repository is pushed to beskar. If false, all local changes are discarded. // 6. The temporary directory is removed. func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, tFn TransactionFn, opts ...TransactionOption) error { - options := TransactionOptions{} for _, opt := range opts { opt(&options) diff --git a/internal/plugins/ostree/pkg/ostreerepository/sync.go b/internal/plugins/ostree/pkg/ostreerepository/sync.go index 8abecc4..7ed8973 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/sync.go +++ b/internal/plugins/ostree/pkg/ostreerepository/sync.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package ostreerepository import ( diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go index 2c52696..e6fdc67 100644 --- a/internal/plugins/ostree/plugin.go +++ b/internal/plugins/ostree/plugin.go @@ -6,6 +6,12 @@ package ostree import ( "context" _ "embed" + "net" + "net/http" + "net/http/pprof" + "path/filepath" + "strconv" + "github.com/RussellLuo/kun/pkg/httpcodec" "github.com/go-chi/chi" "github.com/google/go-containerregistry/pkg/name" @@ -20,11 +26,6 @@ import ( "go.ciq.dev/beskar/pkg/mtls" apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" "go.ciq.dev/beskar/pkg/version" - "net" - "net/http" - "net/http/pprof" - "path/filepath" - "strconv" ) const ( diff --git a/internal/plugins/static/pkg/staticrepository/api.go b/internal/plugins/static/pkg/staticrepository/api.go index ea51025..82b1e0e 100644 --- a/internal/plugins/static/pkg/staticrepository/api.go +++ b/internal/plugins/static/pkg/staticrepository/api.go @@ -7,10 +7,11 @@ import ( "context" "errors" "fmt" - "go.ciq.dev/beskar/pkg/utils" "path/filepath" "time" + "go.ciq.dev/beskar/pkg/utils" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" "github.com/hashicorp/go-multierror" diff --git a/internal/plugins/yum/pkg/yumrepository/api.go b/internal/plugins/yum/pkg/yumrepository/api.go index 633349d..75a2cb4 100644 --- a/internal/plugins/yum/pkg/yumrepository/api.go +++ b/internal/plugins/yum/pkg/yumrepository/api.go @@ -9,10 +9,11 @@ import ( "encoding/gob" "errors" "fmt" - "go.ciq.dev/beskar/pkg/utils" "path/filepath" "time" + "go.ciq.dev/beskar/pkg/utils" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" "github.com/google/go-containerregistry/pkg/v1/remote/transport" diff --git a/pkg/orasostree/push.go b/pkg/orasostree/push.go index 110b9e8..6bc6a22 100644 --- a/pkg/orasostree/push.go +++ b/pkg/orasostree/push.go @@ -1,16 +1,20 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package orasostree import ( "context" "fmt" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "go.ciq.dev/beskar/pkg/oras" - "golang.org/x/sync/errgroup" "log/slog" "os" "path/filepath" "strings" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "go.ciq.dev/beskar/pkg/oras" + "golang.org/x/sync/errgroup" ) type OSTreeRepositoryPusher struct { @@ -108,7 +112,6 @@ func (p *OSTreeRepositoryPusher) Push() error { func (p *OSTreeRepositoryPusher) push(path string) error { pusher, err := NewOSTreeFilePusher(p.dir, path, p.repo, p.nameOpts...) if err != nil { - return fmt.Errorf("while creating OSTree pusher: %w", err) } diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 40797eb..0ea979d 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -58,10 +58,10 @@ type SyncStatus struct { EndTime string `json:"end_time"` SyncError string `json:"sync_error"` - //TODO: Implement these + // TODO: Implement these // The data for these is present when performing a pull via the ostree cli, so it is in the libostree code base. - //SyncedMetadata int `json:"synced_metadata"` - //SyncedObjects int `json:"synced_objects"` + // SyncedMetadata int `json:"synced_metadata"` + // SyncedObjects int `json:"synced_objects"` } // OSTree is used for managing ostree repositories. diff --git a/pkg/utils/time.go b/pkg/utils/time.go index 3e29d4f..c215c3c 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + package utils import "time" From 4320e9c90e7fbe9e7798c6dc525420f3c513039b Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Thu, 11 Jan 2024 22:17:20 -0500 Subject: [PATCH 21/30] removes unused dockerfile --- build/mage/dockerfiles/ostree.dockerfile | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 build/mage/dockerfiles/ostree.dockerfile diff --git a/build/mage/dockerfiles/ostree.dockerfile b/build/mage/dockerfiles/ostree.dockerfile deleted file mode 100644 index 2ceeb5f..0000000 --- a/build/mage/dockerfiles/ostree.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM rockylinux:8-minimal as Builder - -RUN microdnf update && \ - microdnf -y install ostree ostree-devel - From 31586512f4e4ff9718c311fe8ada13bcc703f06c Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 12 Jan 2024 17:52:18 -0500 Subject: [PATCH 22/30] fixes unit test build failure by introducing buildEnv and buildExecStmts from binaryConfigs attempts to ignore ALL intellij files this time renames free freeFunc to deferFree deferredFreeFn for a more clear api bumps beskar-ostree go version to match others --- .github/workflows/release.yml | 4 +- .gitignore | 3 +- .idea/.gitignore | 8 ++++ .idea/go.imports.xml | 12 ++++++ .idea/modules.xml | 8 ++++ .idea/vcs.xml | 6 +++ build/mage/test.go | 16 ++++++-- internal/pkg/pluginsrv/webhandler.go | 2 +- .../plugins/ostree/pkg/libostree/options.go | 14 +++---- internal/plugins/ostree/pkg/libostree/pull.go | 40 +++++++++---------- internal/plugins/static/plugin.go | 11 +---- 11 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/go.imports.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7c60b7..5e3b9de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,12 +73,12 @@ jobs: release-beskar-ostree: name: release beskar-ostree - needs: lint + needs: [lint, tests] runs-on: ubuntu-22.04 steps: - uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/checkout@v3 - name: Release beskar-ostree image run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-ostree:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index faae1fa..fa8baec 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ build/output vendor go.work.sum -.idea + +*/.idea internal/plugins/ostree/pkg/libostree/testdata \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml new file mode 100644 index 0000000..d5ed47a --- /dev/null +++ b/.idea/go.imports.xml @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..50cc4b9 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/build/mage/test.go b/build/mage/test.go index 53f87c0..0bfe3e5 100644 --- a/build/mage/test.go +++ b/build/mage/test.go @@ -1,12 +1,12 @@ package mage +import "C" import ( "context" - "fmt" - "strings" - "dagger.io/dagger" + "fmt" "github.com/magefile/mage/mg" + "strings" ) type Test mg.Namespace @@ -29,6 +29,16 @@ func (Test) Unit(ctx context.Context) error { WithWorkdir("/src"). With(goCache(client)) + for _, config := range binaries { + for key, value := range config.buildEnv { + unitTest = unitTest.WithEnvVariable(key, value) + } + + for _, execStmt := range config.buildExecStmts { + unitTest = unitTest.WithExec(execStmt) + } + } + unitTest = unitTest.WithExec([]string{ "go", "test", "-v", "-count=1", "./...", }) diff --git a/internal/pkg/pluginsrv/webhandler.go b/internal/pkg/pluginsrv/webhandler.go index b1f3ef0..e57d2f0 100644 --- a/internal/pkg/pluginsrv/webhandler.go +++ b/internal/pkg/pluginsrv/webhandler.go @@ -41,7 +41,7 @@ func IsTLSMiddleware(next http.Handler) http.Handler { func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { if wh.manager == nil { - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(http.StatusNotImplemented) return } diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go index cc476ed..b5386a0 100644 --- a/internal/plugins/ostree/pkg/libostree/options.go +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -15,10 +15,10 @@ import "unsafe" // Option defines an option for pulling ostree repos. // It is used to build a *C.GVariant via a *C.GVariantBuilder. -// free is an optional function that frees the memory allocated by the option. free may be called more than once. +// deferFree is an optional function that frees the memory allocated by the option. deferFree may be called more than once. type ( - Option func(builder *C.GVariantBuilder, free freeFunc) - freeFunc func(...unsafe.Pointer) + Option func(builder *C.GVariantBuilder, deferFree deferredFreeFn) + deferredFreeFn func(...unsafe.Pointer) ) // ToGVariant converts the given Options to a GVariant using a GVaraintBuilder. @@ -35,12 +35,12 @@ func toGVariant(opts ...Option) *C.GVariant { // Collect pointers to free later var toFree []unsafe.Pointer - freeFn := func(ptrs ...unsafe.Pointer) { + deferFreeFn := func(ptrs ...unsafe.Pointer) { toFree = append(toFree, ptrs...) } for _, opt := range opts { - opt(&builder, freeFn) + opt(&builder, deferFreeFn) } defer func() { for i := 0; i < len(toFree); i++ { @@ -58,9 +58,9 @@ func gVariantBuilderAddVariant(builder *C.GVariantBuilder, key *C.gchar, variant // NoGPGVerify sets the gpg-verify option to false in the pull options. func NoGPGVerify() Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("gpg-verify") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go index 820f26d..b29d1fe 100644 --- a/internal/plugins/ostree/pkg/libostree/pull.go +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -76,9 +76,9 @@ const ( // Flags adds the given flags to the pull options. func Flags(flags FlagSet) Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("flags") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, @@ -90,12 +90,12 @@ func Flags(flags FlagSet) Option { // Refs adds the given refs to the pull options. // When pulling refs from a remote, only the specified refs will be pulled. func Refs(refs ...string) Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { cRefs := C.MakeRefArray(C.int(len(refs))) - free(unsafe.Pointer(cRefs)) + deferFree(unsafe.Pointer(cRefs)) for i := 0; i < len(refs); i++ { cRef := C.CString(refs[i]) - free(unsafe.Pointer(cRef)) + deferFree(unsafe.Pointer(cRef)) C.AppendRef(cRefs, C.int(i), cRef) } C.g_variant_builder_add_refs( @@ -107,9 +107,9 @@ func Refs(refs ...string) Option { // NoGPGVerifySummary sets the gpg-verify-summary option to false in the pull options. func NoGPGVerifySummary() Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("gpg-verify-summary") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, @@ -121,13 +121,13 @@ func NoGPGVerifySummary() Option { // Depth sets the depth option to the given value in the pull options. // How far in the history to traverse; default is 0, -1 means infinite func Depth(depth int) Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { // 0 is the default depth so there is no need to add it to the builder. if depth != 0 { return } key := C.CString("depth") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, @@ -139,9 +139,9 @@ func Depth(depth int) Option { // DisableStaticDelta sets the disable-static-deltas option to true in the pull options. // Do not use static deltas. func DisableStaticDelta() Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("disable-static-deltas") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, @@ -153,9 +153,9 @@ func DisableStaticDelta() Option { // RequireStaticDelta sets the require-static-deltas option to true in the pull options. // Require static deltas. func RequireStaticDelta() Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("require-static-deltas") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, @@ -167,9 +167,9 @@ func RequireStaticDelta() Option { // DryRun sets the dry-run option to true in the pull options. // Only print information on what will be downloaded (requires static deltas). func DryRun() Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("dry-run") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, @@ -181,16 +181,16 @@ func DryRun() Option { // AppendUserAgent sets the append-user-agent option to the given value in the pull options. // Additional string to append to the user agent. func AppendUserAgent(appendUserAgent string) Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { // "" is the default so there is no need to add it to the builder. if appendUserAgent == "" { return } key := C.CString("append-user-agent") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) cAppendUserAgent := C.CString(appendUserAgent) - free(unsafe.Pointer(cAppendUserAgent)) + deferFree(unsafe.Pointer(cAppendUserAgent)) gVariantBuilderAddVariant( builder, key, @@ -202,9 +202,9 @@ func AppendUserAgent(appendUserAgent string) Option { // NetworkRetries sets the n-network-retries option to the given value in the pull options. // Number of times to retry each download on receiving. func NetworkRetries(n int) Option { - return func(builder *C.GVariantBuilder, free freeFunc) { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { key := C.CString("n-network-retries") - free(unsafe.Pointer(key)) + deferFree(unsafe.Pointer(key)) gVariantBuilderAddVariant( builder, key, diff --git a/internal/plugins/static/plugin.go b/internal/plugins/static/plugin.go index 4aab58a..773279c 100644 --- a/internal/plugins/static/plugin.go +++ b/internal/plugins/static/plugin.go @@ -124,7 +124,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/static/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -146,12 +146,3 @@ func (p *Plugin) Context() context.Context { func (p *Plugin) RepositoryManager() *repository.Manager[*staticrepository.Handler] { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} From feae074a015cff1ee708e99f6d4f202aaed9c6dc Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 12 Jan 2024 18:06:35 -0500 Subject: [PATCH 23/30] removes mistaken C import causing mage to no longer see test targets. removes intellij files fixes timeFormat constant per coderabbit suggestion --- .gitignore | 3 ++- .idea/go.imports.xml | 12 ------------ .idea/vcs.xml | 6 ------ build/mage/test.go | 1 - pkg/utils/time.go | 2 +- 5 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 .idea/go.imports.xml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index fa8baec..8598532 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ go.work.sum */.idea -internal/plugins/ostree/pkg/libostree/testdata \ No newline at end of file +internal/plugins/ostree/pkg/libostree/testdata +/.idea/ diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml deleted file mode 100644 index d5ed47a..0000000 --- a/.idea/go.imports.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/build/mage/test.go b/build/mage/test.go index 0bfe3e5..180f646 100644 --- a/build/mage/test.go +++ b/build/mage/test.go @@ -1,6 +1,5 @@ package mage -import "C" import ( "context" "dagger.io/dagger" diff --git a/pkg/utils/time.go b/pkg/utils/time.go index c215c3c..0c8dcc2 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -5,7 +5,7 @@ package utils import "time" -const timeFormat = time.DateTime + " MST" +const timeFormat = "2006-01-02 15:04:05 MST" func TimeToString(t int64) string { if t == 0 { From 793ac413eb81323dc70b70eaae4d711874cc0b49 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 12 Jan 2024 18:09:52 -0500 Subject: [PATCH 24/30] fixes logic error in Depth() libostree option --- internal/plugins/ostree/pkg/libostree/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go index b29d1fe..80cae32 100644 --- a/internal/plugins/ostree/pkg/libostree/pull.go +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -123,7 +123,7 @@ func NoGPGVerifySummary() Option { func Depth(depth int) Option { return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { // 0 is the default depth so there is no need to add it to the builder. - if depth != 0 { + if depth == 0 { return } key := C.CString("depth") From 9d3909349e40218a827908525173020d842167f2 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 12 Jan 2024 18:12:33 -0500 Subject: [PATCH 25/30] rebase upstream main fixes copyright headers --- cmd/beskar-ostree/main.go | 2 +- cmd/beskarctl/ctl/error.go | 2 +- cmd/beskarctl/ctl/helpers.go | 2 +- cmd/beskarctl/ctl/root.go | 2 +- cmd/beskarctl/ostree/file.go | 2 +- cmd/beskarctl/ostree/repo.go | 2 +- cmd/beskarctl/static/push.go | 2 +- cmd/beskarctl/static/root.go | 2 +- cmd/beskarctl/yum/push.go | 2 +- cmd/beskarctl/yum/pushmetadata.go | 2 +- cmd/beskarctl/yum/root.go | 2 +- internal/plugins/ostree/api.go | 2 +- internal/plugins/ostree/pkg/config/beskar-ostree.go | 2 +- internal/plugins/ostree/pkg/libostree/errors.go | 2 +- internal/plugins/ostree/pkg/libostree/glib_helpers.go | 2 +- internal/plugins/ostree/pkg/libostree/options.go | 2 +- internal/plugins/ostree/pkg/libostree/ostree.go | 2 +- internal/plugins/ostree/pkg/libostree/pull.go | 2 +- internal/plugins/ostree/pkg/libostree/pull_test.go | 2 +- internal/plugins/ostree/pkg/libostree/repo.go | 2 +- internal/plugins/ostree/pkg/ostreerepository/api.go | 2 +- internal/plugins/ostree/pkg/ostreerepository/handler.go | 2 +- internal/plugins/ostree/pkg/ostreerepository/local.go | 2 +- internal/plugins/ostree/pkg/ostreerepository/sync.go | 2 +- internal/plugins/ostree/plugin.go | 2 +- pkg/orasostree/ostree.go | 2 +- pkg/orasostree/push.go | 2 +- pkg/plugins/ostree/api/v1/api.go | 2 +- pkg/utils/time.go | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go index ad93634..6fd4399 100644 --- a/cmd/beskar-ostree/main.go +++ b/cmd/beskar-ostree/main.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package main diff --git a/cmd/beskarctl/ctl/error.go b/cmd/beskarctl/ctl/error.go index cd98fd2..706dc5f 100644 --- a/cmd/beskarctl/ctl/error.go +++ b/cmd/beskarctl/ctl/error.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ctl diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go index eb86300..445a5e8 100644 --- a/cmd/beskarctl/ctl/helpers.go +++ b/cmd/beskarctl/ctl/helpers.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ctl diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go index d0afdf1..fcb6efe 100644 --- a/cmd/beskarctl/ctl/root.go +++ b/cmd/beskarctl/ctl/root.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ctl diff --git a/cmd/beskarctl/ostree/file.go b/cmd/beskarctl/ostree/file.go index f6934d1..8de9bca 100644 --- a/cmd/beskarctl/ostree/file.go +++ b/cmd/beskarctl/ostree/file.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostree diff --git a/cmd/beskarctl/ostree/repo.go b/cmd/beskarctl/ostree/repo.go index 0dcce7d..412c85e 100644 --- a/cmd/beskarctl/ostree/repo.go +++ b/cmd/beskarctl/ostree/repo.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostree diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go index 616a61b..56e7d88 100644 --- a/cmd/beskarctl/static/push.go +++ b/cmd/beskarctl/static/push.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package static diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go index 8bcae1c..f917157 100644 --- a/cmd/beskarctl/static/root.go +++ b/cmd/beskarctl/static/root.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package static diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go index a3cb9c0..a8b6379 100644 --- a/cmd/beskarctl/yum/push.go +++ b/cmd/beskarctl/yum/push.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package yum diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go index 09d1023..6f1015c 100644 --- a/cmd/beskarctl/yum/pushmetadata.go +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package yum diff --git a/cmd/beskarctl/yum/root.go b/cmd/beskarctl/yum/root.go index 29fe668..3c2ea4b 100644 --- a/cmd/beskarctl/yum/root.go +++ b/cmd/beskarctl/yum/root.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package yum diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index 6a343d9..9bbaa2f 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostree diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go index 7663530..10df529 100644 --- a/internal/plugins/ostree/pkg/config/beskar-ostree.go +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package config diff --git a/internal/plugins/ostree/pkg/libostree/errors.go b/internal/plugins/ostree/pkg/libostree/errors.go index d7e1f18..f942f8f 100644 --- a/internal/plugins/ostree/pkg/libostree/errors.go +++ b/internal/plugins/ostree/pkg/libostree/errors.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go b/internal/plugins/ostree/pkg/libostree/glib_helpers.go index a388095..d4153a7 100644 --- a/internal/plugins/ostree/pkg/libostree/glib_helpers.go +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go index b5386a0..579ed1e 100644 --- a/internal/plugins/ostree/pkg/libostree/options.go +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/libostree/ostree.go b/internal/plugins/ostree/pkg/libostree/ostree.go index c21ff1c..4a20c4a 100644 --- a/internal/plugins/ostree/pkg/libostree/ostree.go +++ b/internal/plugins/ostree/pkg/libostree/ostree.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go index 80cae32..4d2976a 100644 --- a/internal/plugins/ostree/pkg/libostree/pull.go +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index 71235ec..f9db57d 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/libostree/repo.go b/internal/plugins/ostree/pkg/libostree/repo.go index 7471970..ced8142 100644 --- a/internal/plugins/ostree/pkg/libostree/repo.go +++ b/internal/plugins/ostree/pkg/libostree/repo.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go index 72264c2..c399785 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/api.go +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostreerepository diff --git a/internal/plugins/ostree/pkg/ostreerepository/handler.go b/internal/plugins/ostree/pkg/ostreerepository/handler.go index 549f001..7a9b121 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/handler.go +++ b/internal/plugins/ostree/pkg/ostreerepository/handler.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostreerepository diff --git a/internal/plugins/ostree/pkg/ostreerepository/local.go b/internal/plugins/ostree/pkg/ostreerepository/local.go index 7d7d9ee..5a87784 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/local.go +++ b/internal/plugins/ostree/pkg/ostreerepository/local.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostreerepository diff --git a/internal/plugins/ostree/pkg/ostreerepository/sync.go b/internal/plugins/ostree/pkg/ostreerepository/sync.go index 7ed8973..6947760 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/sync.go +++ b/internal/plugins/ostree/pkg/ostreerepository/sync.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostreerepository diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go index e6fdc67..398730d 100644 --- a/internal/plugins/ostree/plugin.go +++ b/internal/plugins/ostree/plugin.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package ostree diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go index 0d433aa..a41f5c3 100644 --- a/pkg/orasostree/ostree.go +++ b/pkg/orasostree/ostree.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package orasostree diff --git a/pkg/orasostree/push.go b/pkg/orasostree/push.go index 6bc6a22..e06c21b 100644 --- a/pkg/orasostree/push.go +++ b/pkg/orasostree/push.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package orasostree diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 0ea979d..46ab6c8 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package apiv1 diff --git a/pkg/utils/time.go b/pkg/utils/time.go index 0c8dcc2..16981bd 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package utils From fb8337fca34f5feefc37f3ff8a96d532027a38c3 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Fri, 12 Jan 2024 19:55:09 -0500 Subject: [PATCH 26/30] removed comments again... --- internal/pkg/beskar/plugin.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/pkg/beskar/plugin.go b/internal/pkg/beskar/plugin.go index 3002260..b382a85 100644 --- a/internal/pkg/beskar/plugin.go +++ b/internal/pkg/beskar/plugin.go @@ -419,7 +419,3 @@ func loadPlugins(ctx context.Context) (func(), error) { return wg.Wait, nil } - -// Mountain Team - Lead Developer -// Fuzzball Team - -// Innovation Group - Tech Ambassador () From 85c6ea549ca6819763fc0d1d4dc0dcb19832158c Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Mon, 15 Jan 2024 12:43:53 -0500 Subject: [PATCH 27/30] fixes linter issues --- build/mage/build.go | 6 ++++++ build/mage/lint.go | 12 ++++++++++++ .../plugins/ostree/pkg/libostree/glib_helpers.go | 8 ++------ internal/plugins/ostree/pkg/libostree/options.go | 9 ++------- internal/plugins/ostree/pkg/libostree/ostree.go | 6 ++++++ internal/plugins/ostree/pkg/libostree/pull.go | 9 ++++----- .../plugins/ostree/pkg/libostree/pull_test.go | 6 +++--- internal/plugins/ostree/pkg/libostree/repo.go | 15 ++++++--------- .../plugins/ostree/pkg/ostreerepository/api.go | 4 ++-- .../ostree/pkg/ostreerepository/handler.go | 14 ++++++++++---- .../plugins/ostree/pkg/ostreerepository/local.go | 3 +-- .../plugins/ostree/pkg/ostreerepository/sync.go | 1 + 12 files changed, 55 insertions(+), 38 deletions(-) diff --git a/build/mage/build.go b/build/mage/build.go index 189801d..b32bf1d 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -150,9 +150,15 @@ var binaries = map[string]binaryConfig{ }, buildExecStmts: [][]string{ { + // pkg-config is needed to compute CFLAGS + "apk", "add", "pkgconfig", + }, + { + // Install gcc. Could have installed gc directly but this seems to be the recommended way for alpine. "apk", "add", "build-base", }, { + // Install ostree development libraries "apk", "add", "ostree", "ostree-dev", }, }, diff --git a/build/mage/lint.go b/build/mage/lint.go index b6cc1ea..12a59d9 100644 --- a/build/mage/lint.go +++ b/build/mage/lint.go @@ -36,6 +36,18 @@ func (Lint) Go(ctx context.Context) error { WithWorkdir("/src"). With(goCache(client)) + // Set up the environment for the linter per the settings of each binary. + // This could lead to conflicts if the binaries have different settings. + for _, config := range binaries { + for key, value := range config.buildEnv { + golangciLint = golangciLint.WithEnvVariable(key, value) + } + + for _, execStmt := range config.buildExecStmts { + golangciLint = golangciLint.WithExec(execStmt) + } + } + golangciLint = golangciLint.WithExec([]string{ "golangci-lint", "-v", "run", "--modules-download-mode", "readonly", "--timeout", "5m", }) diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go b/internal/plugins/ostree/pkg/libostree/glib_helpers.go index d4153a7..e6fda4a 100644 --- a/internal/plugins/ostree/pkg/libostree/glib_helpers.go +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go @@ -1,13 +1,9 @@ +//nolint:goheader // SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree -// #cgo pkg-config: glib-2.0 gobject-2.0 -// #include -// #include -// #include -// #include // #include "glib_helpers.go.h" import "C" @@ -23,5 +19,5 @@ func GoError(e *C.GError) error { if e == nil { return nil } - return errors.New(C.GoString((*C.char)(C._g_error_get_message(e)))) + return errors.New(C.GoString(C._g_error_get_message(e))) } diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go index 579ed1e..90f0735 100644 --- a/internal/plugins/ostree/pkg/libostree/options.go +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -1,14 +1,9 @@ +//nolint:goheader // SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree -// #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 -// #include -// #include -// #include -// #include -// #include // #include "options.go.h" import "C" import "unsafe" @@ -23,7 +18,7 @@ type ( // ToGVariant converts the given Options to a GVariant using a GVaraintBuilder. func toGVariant(opts ...Option) *C.GVariant { - typeStr := (*C.gchar)(C.CString("a{sv}")) + typeStr := C.CString("a{sv}") defer C.free(unsafe.Pointer(typeStr)) variantType := C.g_variant_type_new(typeStr) diff --git a/internal/plugins/ostree/pkg/libostree/ostree.go b/internal/plugins/ostree/pkg/libostree/ostree.go index 4a20c4a..f59b83c 100644 --- a/internal/plugins/ostree/pkg/libostree/ostree.go +++ b/internal/plugins/ostree/pkg/libostree/ostree.go @@ -1,4 +1,10 @@ +//nolint:goheader // SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree + +// This file is used to generate the cgo pkg-config flags for libostree. + +// #cgo pkg-config: ostree-1 glib-2.0 +import "C" diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go index 4d2976a..bb05bf3 100644 --- a/internal/plugins/ostree/pkg/libostree/pull.go +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -1,11 +1,9 @@ +//nolint:goheader // SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree -// #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 -// #include -// #include // #include // #include "pull.go.h" import "C" @@ -28,6 +26,7 @@ func (r *Repo) Pull(ctx context.Context, remote string, opts ...Option) error { cCancel := C.g_cancellable_new() go func() { + //nolint:gosimple for { select { case <-ctx.Done(): @@ -67,8 +66,8 @@ const ( // BaseUserOnlyFiles - Since 2017.7. Reject writes of content objects with modes outside of 0775. BaseUserOnlyFiles - // TrustedHttp - Don't verify checksums of objects HTTP repositories (Since: 2017.12) - TrustedHttp + // TrustedHTTP - Don't verify checksums of objects HTTP repositories (Since: 2017.12) + TrustedHTTP // None - No special options for pull None = 0 diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index f9db57d..1d1ac97 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -113,7 +113,7 @@ func TestRepo_Pull(t *testing.T) { err := repo.Pull( ctx, remoteName, - Flags(Mirror|TrustedHttp), + Flags(Mirror|TrustedHTTP), ) assert.Error(t, err) if err == nil { @@ -126,7 +126,7 @@ func TestRepo_Pull(t *testing.T) { err := repo.Pull( context.TODO(), remoteName, - Flags(Mirror|TrustedHttp), + Flags(Mirror|TrustedHTTP), ) assert.NoError(t, err) if err != nil { @@ -146,7 +146,7 @@ func TestRepo_Pull(t *testing.T) { expectedChecksums[strings.TrimRight(string(test1Data), "\n")] = false expectedChecksums[strings.TrimRight(string(test2Data), "\n")] = false - refs, err := repo.ListRefsExt(ListRefsExtFlags_None) + refs, err := repo.ListRefsExt(ListRefsExtFlagsNone) assert.NoError(t, err) if err != nil { assert.Failf(t, "failed to list refs", "err: %s", err.Error()) diff --git a/internal/plugins/ostree/pkg/libostree/repo.go b/internal/plugins/ostree/pkg/libostree/repo.go index ced8142..0f4a291 100644 --- a/internal/plugins/ostree/pkg/libostree/repo.go +++ b/internal/plugins/ostree/pkg/libostree/repo.go @@ -1,13 +1,10 @@ +//nolint:goheader // SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved // SPDX-License-Identifier: Apache-2.0 package libostree -// #cgo pkg-config: ostree-1 glib-2.0 gobject-2.0 -// #include // #include -// #include -// #include // #include import "C" @@ -252,10 +249,10 @@ func (r *Repo) ListRemotes() []string { type ListRefsExtFlags int const ( - ListRefsExtFlags_Aliases = 1 << iota - ListRefsExtFlags_ExcludeRemotes - ListRefsExtFlags_ExcludeMirrors - ListRefsExtFlags_None ListRefsExtFlags = 0 + ListRefsExtFlagsAliases = 1 << iota + ListRefsExtFlagsExcludeRemotes + ListRefsExtFlagsExcludeMirrors + ListRefsExtFlagsNone ListRefsExtFlags = 0 ) type Ref struct { @@ -299,7 +296,7 @@ func (r *Repo) ListRefsExt(flags ListRefsExtFlags, prefix ...string) ([]Ref, err ref := (*C.OstreeCollectionRef)(unsafe.Pointer(&cRef)) ret = append(ret, Ref{ - Name: C.GoString((*C.char)((*C.gchar)(ref.ref_name))), + Name: C.GoString(ref.ref_name), Checksum: C.GoString((*C.char)(cChecksum)), }) } diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go index c399785..d25522d 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/api.go +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -161,7 +161,7 @@ func (h *Handler) AddRemote(ctx context.Context, remote *apiv1.OSTreeRemotePrope }, SkipPull()) } -func (h *Handler) SyncRepository(ctx context.Context, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { +func (h *Handler) SyncRepository(_ context.Context, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { // Transition to syncing state if err := h.setState(StateSyncing); err != nil { return err @@ -191,7 +191,7 @@ func (h *Handler) SyncRepository(ctx context.Context, properties *apiv1.OSTreeRe // Pull the latest changes from the remote. opts := []libostree.Option{ libostree.Depth(properties.Depth), - libostree.Flags(libostree.Mirror | libostree.TrustedHttp), + libostree.Flags(libostree.Mirror | libostree.TrustedHTTP), } if len(properties.Refs) > 0 { opts = append(opts, libostree.Refs(properties.Refs...)) diff --git a/internal/plugins/ostree/pkg/ostreerepository/handler.go b/internal/plugins/ostree/pkg/ostreerepository/handler.go index 7a9b121..59d0e47 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/handler.go +++ b/internal/plugins/ostree/pkg/ostreerepository/handler.go @@ -83,14 +83,14 @@ func (h *Handler) setState(state State) error { } h._state.Swap(int32(state)) if state == StateSyncing || current == StateSyncing { - h.updateSyncing(state == StateSyncing) + _ = h.updateSyncing(state == StateSyncing) } return nil } func (h *Handler) clearState() { h._state.Swap(int32(StateReady)) - h.updateSyncing(false) + _ = h.updateSyncing(false) } func (h *Handler) getState() State { @@ -120,6 +120,7 @@ func (h *Handler) Start(ctx context.Context) { go func() { for !h.Stopped.Load() { + //nolint: gosimple select { case <-ctx.Done(): h.Stopped.Store(true) @@ -130,10 +131,15 @@ func (h *Handler) Start(ctx context.Context) { } // pullConfig pulls the config file from beskar. -func (h *Handler) pullFile(_ context.Context, filename string) error { +func (h *Handler) pullFile(ctx context.Context, filename string) error { // TODO: Replace with appropriate puller mechanism url := "http://" + h.Params.GetBeskarRegistryHostPort() + path.Join("/", h.Repository, "repo", filename) - resp, err := http.Get(url) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + resp, err := http.DefaultClient.Do(req) if err != nil { return err } diff --git a/internal/plugins/ostree/pkg/ostreerepository/local.go b/internal/plugins/ostree/pkg/ostreerepository/local.go index 5a87784..c8f4eff 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/local.go +++ b/internal/plugins/ostree/pkg/ostreerepository/local.go @@ -105,7 +105,7 @@ func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, tFn Transaction ctx, beskarRemoteName, libostree.NoGPGVerify(), - libostree.Flags(libostree.Mirror|libostree.TrustedHttp), + libostree.Flags(libostree.Mirror|libostree.TrustedHTTP), ); err != nil { return ctl.Errf("pulling ostree repository from %s: %s", beskarRemoteName, err) } @@ -119,7 +119,6 @@ func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, tFn Transaction // Commit the changes to beskar if the transaction deems it necessary if commit { - // Remove the internal beskar remote so that external clients can't pull from it, not that it would work. if err := repo.DeleteRemote(beskarRemoteName); err != nil { return ctl.Errf("deleting remote %s: %s", beskarRemoteName, err) diff --git a/internal/plugins/ostree/pkg/ostreerepository/sync.go b/internal/plugins/ostree/pkg/ostreerepository/sync.go index 6947760..74f6cf9 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/sync.go +++ b/internal/plugins/ostree/pkg/ostreerepository/sync.go @@ -19,6 +19,7 @@ func (h *Handler) setRepoSync(repoSync *RepoSync) { h.repoSync.Store(&rs) } +//nolint:unparam func (h *Handler) updateSyncing(syncing bool) *RepoSync { if h.repoSync.Load() == nil { h.repoSync.Store(&RepoSync{}) From 7049ce3091273b783373dbcb8bb7eea00b8779e9 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Mon, 15 Jan 2024 13:27:22 -0500 Subject: [PATCH 28/30] adds libostree testdata adds documentation about generating libostree testdata --- .gitignore | 2 -- internal/plugins/ostree/pkg/libostree/README.md | 7 +++++++ .../ostree/pkg/libostree/generate-testdata.sh | 2 +- .../plugins/ostree/pkg/libostree/pull_test.go | 6 ++++++ .../ostree/pkg/libostree/testdata/repo/.lock | 0 .../ostree/pkg/libostree/testdata/repo/config | 4 ++++ ...811918927eb06f8354d88a4b670e0588d10a628b.filez | Bin 0 -> 138 bytes ...1011c5791a6ce28584127d68b4452321be431c.dirtree | Bin 0 -> 48 bytes ...25eb709b260bb8c688c87890f34b13b96565c2.dirtree | Bin 0 -> 103 bytes ...662c4947ea18a96be03b406bc8eb9ccf913ff22.commit | Bin 0 -> 118 bytes ...099c79e795ac1d667aed5b9121a0c75f9b38e41.commit | Bin 0 -> 150 bytes ...38bac61853554c724cbd7c5de2bbfb224e9779.dirmeta | Bin 0 -> 68 bytes ...32d593549b625507b8e2350b24b0e172d90e4000.filez | Bin 0 -> 110 bytes ...10e95391dd4a31217025a8fb0acebf7ea8db88b.commit | Bin 0 -> 118 bytes ...38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree | Bin 0 -> 48 bytes ...e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez | Bin 0 -> 138 bytes .../pkg/libostree/testdata/repo/refs/heads/test1 | 1 + .../pkg/libostree/testdata/repo/refs/heads/test2 | 1 + .../ostree/pkg/libostree/testdata/repo/summary | Bin 0 -> 354 bytes 19 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/.lock create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/config create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/05/b8c96e271b1d932571374a811918927eb06f8354d88a4b670e0588d10a628b.filez create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/07/10a0f5925abdefc9816329711011c5791a6ce28584127d68b4452321be431c.dirtree create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/08/9d8abd9ca632baabbaf908c825eb709b260bb8c688c87890f34b13b96565c2.dirtree create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/0e/1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22.commit create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/43/f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41.commit create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/6b/1f7b40b1be1309032c26d50438bac61853554c724cbd7c5de2bbfb224e9779.dirmeta create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/93/7a454371f7f367ba8d168932d593549b625507b8e2350b24b0e172d90e4000.filez create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/aa/659a25dc52d863e4017c35110e95391dd4a31217025a8fb0acebf7ea8db88b.commit create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/e1/95f272f46c434d26c5c7499e38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/objects/ee/3d4763e0ff1f327e102da7e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 create mode 100644 internal/plugins/ostree/pkg/libostree/testdata/repo/summary diff --git a/.gitignore b/.gitignore index 8598532..9c13cd1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,4 @@ go.work.sum */.idea - -internal/plugins/ostree/pkg/libostree/testdata /.idea/ diff --git a/internal/plugins/ostree/pkg/libostree/README.md b/internal/plugins/ostree/pkg/libostree/README.md index 7c2295e..785c327 100644 --- a/internal/plugins/ostree/pkg/libostree/README.md +++ b/internal/plugins/ostree/pkg/libostree/README.md @@ -20,3 +20,10 @@ glib/gobject are reference counted and objects are freed when the reference coun places and `C.free()` in others. A good rule of thumb is that if you see a `g_` prefix you are dealing with a reference counted object and should not call `C.free()`. See [glib](https://docs.gtk.org/glib/index.html) for more information. See [gobject](https://docs.gtk.org/gobject/index.html) for more information. + + +### Testdata +The testdata directory contains a simple ostree repo that can be used for testing. It was created using the generate-testdata.sh +script. testdata has been committed to this git repo so that it remains static. If you need to regenerate the testdata you can, +however, keep in mind that newer versions of ostree may produce different results and may cause tests to fail. The version +of ostree used to generate the testdata is 2023.7. \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/generate-testdata.sh b/internal/plugins/ostree/pkg/libostree/generate-testdata.sh index 4e9d001..d5dd3d0 100755 --- a/internal/plugins/ostree/pkg/libostree/generate-testdata.sh +++ b/internal/plugins/ostree/pkg/libostree/generate-testdata.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -# This script generates test data for the ostree plugin. +# This script generates test data for libostree. # Clean up any existing test data rm -rf testdata diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index 1d1ac97..bd6c536 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -16,6 +16,12 @@ import ( "github.com/stretchr/testify/assert" ) +/* +The testdata directory contains a simple ostree repo that can be used for testing. It was created using the generate-testdata.sh +script. testdata has been committed to this git repo so that it remains static. If you need to regenerate the testdata you can, +however, keep in mind that newer versions of ostree may produce different results and may cause tests to fail. The version +of ostree used to generate the testdata is 2023.7. +*/ func TestMain(m *testing.M) { _, err := os.Stat("testdata/repo/summary") if os.IsNotExist(err) { diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/.lock b/internal/plugins/ostree/pkg/libostree/testdata/repo/.lock new file mode 100644 index 0000000..e69de29 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/config b/internal/plugins/ostree/pkg/libostree/testdata/repo/config new file mode 100644 index 0000000..d289d74 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/testdata/repo/config @@ -0,0 +1,4 @@ +[core] +repo_version=1 +mode=archive-z2 +indexed-deltas=true diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/05/b8c96e271b1d932571374a811918927eb06f8354d88a4b670e0588d10a628b.filez b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/05/b8c96e271b1d932571374a811918927eb06f8354d88a4b670e0588d10a628b.filez new file mode 100644 index 0000000000000000000000000000000000000000..e95c4e9983b47dcdf1ba102f9b5d48eeac3b7a3b GIT binary patch literal 138 zcmZQzUNjA!VHGr=Aq8t2aFGHm+X?cEg= aEZP;s{YkV$bW)el=SjvQUJYKoI~V{3>?}3_ literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/07/10a0f5925abdefc9816329711011c5791a6ce28584127d68b4452321be431c.dirtree b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/07/10a0f5925abdefc9816329711011c5791a6ce28584127d68b4452321be431c.dirtree new file mode 100644 index 0000000000000000000000000000000000000000..7f67df1ad0a016370e4e47fcb29689fa550bc3bd GIT binary patch literal 48 zcmXR(EiOsR%t_TNsVHG!-ElHcU0QasYN5GTqol;7x()fwAve0b)A?9CE^;Mx^XlmX E0J)hFg#Z8m literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/08/9d8abd9ca632baabbaf908c825eb709b260bb8c688c87890f34b13b96565c2.dirtree b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/08/9d8abd9ca632baabbaf908c825eb709b260bb8c688c87890f34b13b96565c2.dirtree new file mode 100644 index 0000000000000000000000000000000000000000..365ec28331b61d22e904cca74d180c9ce904a90b GIT binary patch literal 103 zcmYey%P+}DEs8HmEiOsR%t_TNsVHHXT;=Lq`2BPGu3oWDqpOocW+#QR?|5X&t+L@^ x(M>)F22m8%tUFHTsY}aFRxLF5YLt|iRJS3&Ipju{cRC+y$3?EBZeFvLGyt_4DJK8` literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/0e/1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22.commit b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/0e/1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22.commit new file mode 100644 index 0000000000000000000000000000000000000000..93b2837a7ed037ee96cc6b295334b490e8178092 GIT binary patch literal 118 zcmc~VE-6Y))hkL((@o0EOUcYjX8?ne)Z!9D1~!JoVqs;72m?dvf>jS6PW@E$CCAxU z?dWmOc^1DuNlO0x`{p(0{*)6*%U1tB<(4g9?XYp5FekH)+Eo^dUB@JXLw$;T_SVEc R+WlL}Z+c~bnyQM53IOsJE(ibs literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/43/f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41.commit b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/43/f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41.commit new file mode 100644 index 0000000000000000000000000000000000000000..6000091151059ec891f610685c21cc8224d11a2f GIT binary patch literal 150 zcmc~VE-6Y))hkL((@o0EOUcYjX8?ne)Z!8&1~!JoVqxV~sk2n?1l>q}!dPP}$T!te z_R3-*ai*yL4QpP1f7QFA8=`}OA$7s3ha7Xe_Rd*mv}^UQpByJtUl+_)lAv5$8DR`Q!(S>WdC;-Ufom5@8f literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/6b/1f7b40b1be1309032c26d50438bac61853554c724cbd7c5de2bbfb224e9779.dirmeta b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/6b/1f7b40b1be1309032c26d50438bac61853554c724cbd7c5de2bbfb224e9779.dirmeta new file mode 100644 index 0000000000000000000000000000000000000000..162d63159ecace0a7ae444862f383db818d74cbc GIT binary patch literal 68 zcmZQzV1B^>#*S}`QykE TQ;Xs=@^e$;ORS0w7zE7$GVB;D literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/93/7a454371f7f367ba8d168932d593549b625507b8e2350b24b0e172d90e4000.filez b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/93/7a454371f7f367ba8d168932d593549b625507b8e2350b24b0e172d90e4000.filez new file mode 100644 index 0000000000000000000000000000000000000000..cdb2348793a79b0922a0f15538aba1f75450d3e7 GIT binary patch literal 110 zcmZQzUjUM1r~gr6t(yL z$;M>OLIJ^}l~Os6T3du_Gq$)YEADfa$(FBn*tk!alUYaYDvQOgV-mrkK1DuzYhoYm Q{;lLUy)r;eRYgSw0M=V5c>n+a literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/e1/95f272f46c434d26c5c7499e38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/e1/95f272f46c434d26c5c7499e38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree new file mode 100644 index 0000000000000000000000000000000000000000..08ef7173a5827da6d7640c997e81db3685b500f3 GIT binary patch literal 48 zcmV-00MGw)Wpi|9X>4UKba-?C?mb6i;Qt>oeh@9E;>sp~LBgxz=RezauBev0bhV~9 G4K6Pm3>L@$ literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/ee/3d4763e0ff1f327e102da7e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/ee/3d4763e0ff1f327e102da7e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez new file mode 100644 index 0000000000000000000000000000000000000000..2443490caa8f8cf3f1ad8bbb1dccb3fe695ac6cf GIT binary patch literal 138 zcmZQzUNjA!VHGr=Aq8t2aFGHm+X?cEg= aEZP;s{YkV$bW)el=SjvQUJYKnI~f23ZY#F{ literal 0 HcmV?d00001 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 new file mode 100644 index 0000000..7fc311c --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 @@ -0,0 +1 @@ +0e1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 new file mode 100644 index 0000000..d4b9ffd --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 @@ -0,0 +1 @@ +43f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/summary b/internal/plugins/ostree/pkg/libostree/testdata/repo/summary new file mode 100644 index 0000000000000000000000000000000000000000..b464f037584151eb7c5a2b777260d399592f4a8d GIT binary patch literal 354 zcmXR(EiN%+U|=W%Vi4dHm3SA=qPW&1BQj!J(vc~33%jQ6W8T8Hr*G$(pThr@@{3D~ zQd9Mk^K)}EOY}-IbAkF3a|^(FQx~jy$WS7otii^>Py!S;0*X!p%P=rFe?QwWtGdE= z>2C4N4Kt6=tBg8$Eq&e9okCJPr9U_KITB}X*gB}X;?msQ#G*>Q-29YOunmbt$r+ht zsk&7_ON&Jq7|_(^Bo>$G0u^VbWu~S;ox}ifuPm60roJRUH>tQJKQC1m;?rW79tMUa iMGzZJeP&)tYDH>_Zc1uSNg_xwBSVs$T5!vn4GaL18Ffnl literal 0 HcmV?d00001 From bca7378e86ff327fcf4bdd5d4d8368f286e72e36 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Mon, 15 Jan 2024 14:54:44 -0500 Subject: [PATCH 29/30] adds libostree testdata changes libostree to write to /tmp instead of ./testdata --- internal/plugins/ostree/pkg/libostree/pull_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index bd6c536..f62965e 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -28,6 +28,10 @@ func TestMain(m *testing.M) { log.Fatalln("testdata/repo does not exist: please run ./generate-testdata.sh") } + if err := os.MkdirAll("/tmp/libostree-pull_test", 0755); err != nil { + log.Fatalf("failed to create test directory: %s", err.Error()) + } + os.Exit(m.Run()) } @@ -56,7 +60,7 @@ func TestRepo_Pull(t *testing.T) { for _, mode := range modes { mode := mode repoName := fmt.Sprintf("repo-%s", mode) - repoPath := fmt.Sprintf("testdata/%s", repoName) + repoPath := fmt.Sprintf("/tmp/libostree-pull_test/%s", repoName) t.Run(repoName, func(t *testing.T) { t.Cleanup(func() { From a95e957016e1f5c79ba0bbc173b8c86cdfe7fde2 Mon Sep 17 00:00:00 2001 From: Kyle Ishie Date: Mon, 15 Jan 2024 15:03:26 -0500 Subject: [PATCH 30/30] adds --priviledged option to test:unit container --- build/mage/test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/mage/test.go b/build/mage/test.go index 180f646..014b778 100644 --- a/build/mage/test.go +++ b/build/mage/test.go @@ -40,6 +40,8 @@ func (Test) Unit(ctx context.Context) error { unitTest = unitTest.WithExec([]string{ "go", "test", "-v", "-count=1", "./...", + }, dagger.ContainerWithExecOpts{ + InsecureRootCapabilities: true, }) return printOutput(ctx, unitTest)