Skip to content

Commit

Permalink
Merge pull request #807 from traPtitech/impr/gitea-integration
Browse files Browse the repository at this point in the history
Resync gitea on repository creation / speedup
  • Loading branch information
motoki317 authored Dec 6, 2023
2 parents 0c6a64e + cb1b03c commit 2be4215
Show file tree
Hide file tree
Showing 17 changed files with 597 additions and 220 deletions.
3 changes: 2 additions & 1 deletion .local-dev/config/ns.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@ components:
giteaIntegration:
url: https://git.trap.jp
token: ""
intervalSeconds: 600
controller:
url: http://ns-controller:10000

ssgen:
artifactsRoot: /artifacts
Expand Down
6 changes: 6 additions & 0 deletions .local-dev/manifest/ns-system/config/ns.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ components:
container_memory_usage_bytes{namespace="ns-apps", pod="nsapp-{{ .App.ID }}-0", container="app"}
+ container_memory_swap{namespace="ns-apps", pod="nsapp-{{ .App.ID }}-0", container="app"}
giteaIntegration:
url: https://git.trap.jp
token: ""
controller:
url: http://ns-controller:10000

ssgen:
artifactsRoot: /artifacts
healthPort: 8081
Expand Down
11 changes: 11 additions & 0 deletions api/proto/neoshowcase/protobuf/controller.proto
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ message SSGenRequest {
Type type = 1;
}

message GiteaIntegrationRequest {
enum Type {
RESYNC = 0;
}
Type type = 1;
}

service ControllerService {
rpc GetSystemInfo(google.protobuf.Empty) returns (SystemInfo);

Expand All @@ -85,3 +92,7 @@ service ControllerBuilderService {
service ControllerSSGenService {
rpc ConnectSSGen(google.protobuf.Empty) returns (stream SSGenRequest);
}

service ControllerGiteaIntegrationService {
rpc Connect(google.protobuf.Empty) returns (stream GiteaIntegrationRequest);
}
14 changes: 8 additions & 6 deletions cmd/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,11 @@ type GatewayConfig struct {
}

type GiteaIntegrationConfig struct {
URL string `mapstructure:"url" yaml:"url"`
Token string `mapstructure:"token" yaml:"token"`
IntervalSeconds int `mapstructure:"intervalSeconds" yaml:"intervalSeconds"`
ListIntervalMs int `mapstructure:"listIntervalMs" yaml:"listIntervalMs"`
URL string `mapstructure:"url" yaml:"url"`
Token string `mapstructure:"token" yaml:"token"`
IntervalSeconds int `mapstructure:"intervalSeconds" yaml:"intervalSeconds"`
Concurrency int `mapstructure:"concurrency" yaml:"concurrency"`
Controller grpc.ControllerServiceClientConfig `mapstructure:"controller" yaml:"controller"`
}

type SSGenConfig struct {
Expand Down Expand Up @@ -237,8 +238,9 @@ func init() {

viper.SetDefault("components.giteaIntegration.url", "https://git.trap.jp")
viper.SetDefault("components.giteaIntegration.token", "")
viper.SetDefault("components.giteaIntegration.intervalSeconds", 600)
viper.SetDefault("components.giteaIntegration.listIntervalMs", 250)
viper.SetDefault("components.giteaIntegration.intervalSeconds", 86400)
viper.SetDefault("components.giteaIntegration.concurrency", 10)
viper.SetDefault("components.giteaIntegration.controller.url", "http://ns-controller:10000")

viper.SetDefault("components.ssgen.artifactsRoot", "/srv/artifacts")
viper.SetDefault("components.ssgen.healthPort", 8081)
Expand Down
10 changes: 7 additions & 3 deletions cmd/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ var providers = wire.NewSet(
grpc.NewControllerService,
grpc.NewControllerServiceClient,
grpc.NewControllerBuilderService,
grpc.NewControllerGiteaIntegrationService,
grpc.NewControllerGiteaIntegrationServiceClient,
provideControllerBuilderServiceClient,
grpc.NewControllerSSGenService,
grpc.NewControllerSSGenServiceClient,
Expand All @@ -92,7 +94,7 @@ var providers = wire.NewSet(
provideStorage,
provideAuthDevServer,
provideBuildpackBackend,
provdeBuildkitClient,
provideBuildkitClient,
provideControllerServer,
provideContainerLogger,
provideMetricsService,
Expand Down Expand Up @@ -150,7 +152,7 @@ func provideBuildpackBackend(c Config) (builder.BuildpackBackend, error) {
}
}

func provdeBuildkitClient(c Config) (*buildkit.Client, error) {
func provideBuildkitClient(c Config) (*buildkit.Client, error) {
cc := c.Components.Builder
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
Expand All @@ -166,13 +168,15 @@ func provideControllerServer(
controllerHandler pbconnect.ControllerServiceHandler,
builderHandler domain.ControllerBuilderService,
ssgenHandler domain.ControllerSSGenService,
giteaIntegrationHandler domain.ControllerGiteaIntegrationService,
) *controller.APIServer {
wc := web.H2CConfig{
Port: c.Components.Controller.Port,
SetupRoute: func(mux *http.ServeMux) {
mux.Handle(pbconnect.NewControllerServiceHandler(controllerHandler))
mux.Handle(pbconnect.NewControllerBuilderServiceHandler(builderHandler))
mux.Handle(pbconnect.NewControllerSSGenServiceHandler(ssgenHandler))
mux.Handle(pbconnect.NewControllerGiteaIntegrationServiceHandler(giteaIntegrationHandler))
},
}
return &controller.APIServer{H2CServer: web.NewH2CServer(wc)}
Expand Down Expand Up @@ -221,7 +225,7 @@ func provideGiteaIntegrationConfig(c Config) giteaintegration.Config {
URL: cc.URL,
Token: cc.Token,
IntervalSeconds: cc.IntervalSeconds,
ListIntervalMs: cc.ListIntervalMs,
Concurrency: cc.Concurrency,
}
}

Expand Down
1 change: 1 addition & 0 deletions cmd/wire.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func NewGateway(c Config) (component, error) {
func NewGiteaIntegration(c Config) (component, error) {
wire.Build(
providers,
wire.FieldsOf(new(GiteaIntegrationConfig), "Controller"),
wire.Bind(new(component), new(*giteaintegration.Server)),
wire.Struct(new(giteaintegration.Server), "*"),
)
Expand Down
18 changes: 12 additions & 6 deletions cmd/wire_gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ services:
# context: .
# target: ns-gitea-integration
# image: ghcr.io/traptitech/ns-gitea-integration:main
# command: --debug --loglevel=trace --config=/config.yaml
# command: --config=/config.yaml
# restart: always
# volumes:
# - ./.local-dev/config/ns.yaml:/config.yaml
Expand Down
9 changes: 9 additions & 0 deletions pkg/domain/component.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,12 @@ type ControllerSSGenService interface {
type ControllerSSGenServiceClient interface {
ConnectSSGen(ctx context.Context, onRequest func(req *pb.SSGenRequest)) error
}

type ControllerGiteaIntegrationService interface {
pbconnect.ControllerGiteaIntegrationServiceHandler
Broadcast(req *pb.GiteaIntegrationRequest)
}

type ControllerGiteaIntegrationServiceClient interface {
Connect(ctx context.Context, onRequest func(req *pb.GiteaIntegrationRequest)) error
}
75 changes: 75 additions & 0 deletions pkg/infrastructure/grpc/controller_gitea_integration_service.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package grpc

import (
"context"
"sync"

"connectrpc.com/connect"
"github.com/samber/lo"
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/types/known/emptypb"

"github.com/traPtitech/neoshowcase/pkg/domain"
"github.com/traPtitech/neoshowcase/pkg/infrastructure/grpc/pb"
)

type giteaIntegrationConnection struct {
reqSender chan<- *pb.GiteaIntegrationRequest
}

type ControllerGiteaIntegrationService struct {
connections []*giteaIntegrationConnection
lock sync.Mutex
}

func NewControllerGiteaIntegrationService() domain.ControllerGiteaIntegrationService {
return &ControllerGiteaIntegrationService{}
}

func (s *ControllerGiteaIntegrationService) Connect(ctx context.Context, _ *connect.Request[emptypb.Empty], st *connect.ServerStream[pb.GiteaIntegrationRequest]) error {
id := domain.NewID()
log.WithField("id", id).Info("new gitea integration connection")
defer log.WithField("id", id).Info("gitea integration connection closed")

reqSender := make(chan *pb.GiteaIntegrationRequest)
conn := &giteaIntegrationConnection{reqSender: reqSender}
s.lock.Lock()
s.connections = append(s.connections, conn)
s.lock.Unlock()

defer func() {
s.lock.Lock()
defer s.lock.Unlock()
s.connections = lo.Without(s.connections, conn)
}()

loop:
for {
select {
case req, ok := <-reqSender:
if !ok {
break loop
}
err := st.Send(req)
if err != nil {
return err
}
case <-ctx.Done():
break loop
}
}

return nil
}

func (s *ControllerGiteaIntegrationService) Broadcast(req *pb.GiteaIntegrationRequest) {
s.lock.Lock()
defer s.lock.Unlock()

for _, ssgen := range s.connections {
select {
case ssgen.reqSender <- req:
default:
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package grpc

import (
"context"

"connectrpc.com/connect"
"google.golang.org/protobuf/types/known/emptypb"

"github.com/traPtitech/neoshowcase/pkg/domain"
"github.com/traPtitech/neoshowcase/pkg/domain/web"
"github.com/traPtitech/neoshowcase/pkg/infrastructure/grpc/pb"
"github.com/traPtitech/neoshowcase/pkg/infrastructure/grpc/pb/pbconnect"
)

type ControllerGiteaIntegrationServiceClient struct {
client pbconnect.ControllerGiteaIntegrationServiceClient
}

func NewControllerGiteaIntegrationServiceClient(
c ControllerServiceClientConfig,
) domain.ControllerGiteaIntegrationServiceClient {
return &ControllerGiteaIntegrationServiceClient{
client: pbconnect.NewControllerGiteaIntegrationServiceClient(web.NewH2CClient(), c.URL),
}
}

func (c *ControllerGiteaIntegrationServiceClient) Connect(ctx context.Context, onRequest func(req *pb.GiteaIntegrationRequest)) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()

st, err := c.client.Connect(ctx, connect.NewRequest(&emptypb.Empty{}))
if err != nil {
return err
}
defer st.Close()

for st.Receive() {
msg := st.Msg()
onRequest(msg)
}
return st.Err()
}
Loading

0 comments on commit 2be4215

Please sign in to comment.