From 9aa02fadc79d48529797ad6e26e8fe1f3cef5e90 Mon Sep 17 00:00:00 2001 From: greg pereira Date: Sun, 19 May 2024 12:41:19 -0700 Subject: [PATCH 1/3] GITHUB_TOKEN and GITHUB_USER passable via env worker + bot Signed-off-by: greg pereira --- gobot/Containerfile | 6 ++++++ worker/Containerfile | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/gobot/Containerfile b/gobot/Containerfile index 9c64d0ed..dcca01f0 100644 --- a/gobot/Containerfile +++ b/gobot/Containerfile @@ -1,5 +1,11 @@ FROM registry.access.redhat.com/ubi8/ubi as build +ARG GITHUB_USER +ARG GITHUB_TOKEN + +ENV GITHUB_USER=$ILWORKER_GITHUB_USERNAME +ENV GITHUB_TOKEN=$ILWORKER_GITHUB_TOKEN + RUN dnf update -qy && \ dnf install --setopt=install_weak_deps=False -qy \ go \ diff --git a/worker/Containerfile b/worker/Containerfile index 60cfc10d..efa23eb4 100644 --- a/worker/Containerfile +++ b/worker/Containerfile @@ -1,6 +1,12 @@ # Stage 1: Build the worker binary FROM golang:1.22 as builder +ARG GITHUB_USER +ARG GITHUB_TOKEN + +ENV GITHUB_USER=$ILWORKER_GITHUB_USERNAME +ENV GITHUB_TOKEN=$ILWORKER_GITHUB_TOKEN + ENV WORK_DIR /app WORKDIR ${WORK_DIR} From 823551757c2c71589a1c1ed3b3fe2665f756e7bb Mon Sep 17 00:00:00 2001 From: greg pereira Date: Sun, 19 May 2024 16:35:12 -0700 Subject: [PATCH 2/3] add docker-clean command for in-between compose-ups Signed-off-by: greg pereira --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index b088c7e5..7e507f3f 100644 --- a/Makefile +++ b/Makefile @@ -221,3 +221,12 @@ run-on-kind: $(CMD_PREFIX) kubectl create namespace instructlab-bot $(CMD_PREFIX) kubectl create -n instructlab-bot secret generic instructlab-bot --from-env-file=.env $(CMD_PREFIX) kubectl apply -k deploy/instructlab-bot/overlays/dev + +.PHONY: docker-clean +docker-clean: + @container_ids=$$(docker ps -a --format "{{.ID}}" | awk '{print $$1}'); \ + echo "removing all stopped containers (non-force)"; \ + for id in $$container_ids; do \ + echo "Removing container: $$id,"; \ + docker rm $$id; \ + done \ No newline at end of file From 64d0b97df56c0b9a4a4df7aacf6741abb301707e Mon Sep 17 00:00:00 2001 From: greg pereira Date: Sun, 19 May 2024 16:36:21 -0700 Subject: [PATCH 3/3] worker build improvement + use env after flag for worker generate + gobot Signed-off-by: greg pereira --- Makefile | 4 ++-- gobot/Containerfile | 6 ------ gobot/cmd/root.go | 6 ++++++ worker/Containerfile | 8 +------- worker/cmd/generate.go | 16 +++++++++++++++- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 7e507f3f..183c9114 100644 --- a/Makefile +++ b/Makefile @@ -224,9 +224,9 @@ run-on-kind: .PHONY: docker-clean docker-clean: - @container_ids=$$(docker ps -a --format "{{.ID}}" | awk '{print $$1}'); \ + @container_ids=$$(podman ps -a --format "{{.ID}}" | awk '{print $$1}'); \ echo "removing all stopped containers (non-force)"; \ for id in $$container_ids; do \ echo "Removing container: $$id,"; \ - docker rm $$id; \ + podman rm $$id; \ done \ No newline at end of file diff --git a/gobot/Containerfile b/gobot/Containerfile index dcca01f0..9c64d0ed 100644 --- a/gobot/Containerfile +++ b/gobot/Containerfile @@ -1,11 +1,5 @@ FROM registry.access.redhat.com/ubi8/ubi as build -ARG GITHUB_USER -ARG GITHUB_TOKEN - -ENV GITHUB_USER=$ILWORKER_GITHUB_USERNAME -ENV GITHUB_TOKEN=$ILWORKER_GITHUB_TOKEN - RUN dnf update -qy && \ dnf install --setopt=install_weak_deps=False -qy \ go \ diff --git a/gobot/cmd/root.go b/gobot/cmd/root.go index 3d4ceab5..91fc6f01 100644 --- a/gobot/cmd/root.go +++ b/gobot/cmd/root.go @@ -67,6 +67,12 @@ func init() { rootCmd.PersistentFlags().StringSliceVarP(&Maintainers, "maintainers", "", []string{}, "GitHub users or groups that are considered maintainers") rootCmd.PersistentFlags().BoolVarP(&Debug, "debug", "d", false, "Enable debug logging") rootCmd.PersistentFlags().StringVarP(&BotUsername, "bot-username", "", "@instructlab-bot", "The username of the bot") + if GithubToken == "" { + GithubToken = os.Getenv("ILWORKER_GITHUB_TOKEN") + } + if GithubUsername == "" { + GithubUsername = os.Getenv("ILWORKER_GITHUB_USERNAME") + } } var rootCmd = &cobra.Command{ diff --git a/worker/Containerfile b/worker/Containerfile index efa23eb4..d5a19b60 100644 --- a/worker/Containerfile +++ b/worker/Containerfile @@ -1,17 +1,11 @@ # Stage 1: Build the worker binary FROM golang:1.22 as builder -ARG GITHUB_USER -ARG GITHUB_TOKEN - -ENV GITHUB_USER=$ILWORKER_GITHUB_USERNAME -ENV GITHUB_TOKEN=$ILWORKER_GITHUB_TOKEN - ENV WORK_DIR /app WORKDIR ${WORK_DIR} -COPY . ${WORK_DIR}/instructlab-bot +COPY worker ${WORK_DIR}/instructlab-bot/worker # Build the worker binary WORKDIR ${WORK_DIR}/instructlab-bot/worker diff --git a/worker/cmd/generate.go b/worker/cmd/generate.go index 7e4647b8..4df7d641 100644 --- a/worker/cmd/generate.go +++ b/worker/cmd/generate.go @@ -117,7 +117,7 @@ type IlabConfig struct { func init() { generateCmd.Flags().StringVarP(&WorkDir, "work-dir", "w", "", "Directory to work in") generateCmd.Flags().StringVarP(&VenvDir, "venv-dir", "v", "", "The virtual environment directory") - generateCmd.Flags().StringVarP(&PreCheckEndpointURL, "precheck-endpoint-url", "e", "http://localhost:8000/v1", "Endpoint hosting the model API. Default, it assumes the model is served locally.") + generateCmd.Flags().StringVarP(&PreCheckEndpointURL, "precheck-endpoint-url", "e", "", "Endpoint hosting the model API. Default, it assumes the model is served locally.") generateCmd.Flags().StringVarP(&SdgEndpointURL, "sdg-endpoint-url", "", "http://localhost:8000/v1", "Endpoint hosting the model API. Default, it assumes the model is served locally.") generateCmd.Flags().IntVarP(&NumInstructions, "num-instructions", "n", 10, "The number of instructions to generate") generateCmd.Flags().StringVarP(&GitRemote, "git-remote", "", "https://github.com/instructlab/taxonomy", "The git remote for the taxonomy repo") @@ -131,6 +131,20 @@ func init() { generateCmd.Flags().StringVarP(&TlsServerCaCertPath, "tls-server-ca-cert", "", "server-ca-crt.pem2", "Path to the TLS server CA certificate. Defaults to 'server-ca-crt.pem2'") generateCmd.Flags().BoolVarP(&TlsInsecure, "tls-insecure", "", false, "Whether to skip TLS verification") generateCmd.Flags().IntVarP(&MaxSeed, "max-seed", "m", 40, "Maximum number of seed Q&A pairs to process to SDG.") + if GithubToken == "" { + GithubToken = os.Getenv("ILWORKER_GITHUB_TOKEN") + } + if GithubUsername == "" { + GithubUsername = os.Getenv("ILWORKER_GITHUB_USERNAME") + } + if PreCheckEndpointURL == "" { + preCheckEndpointURLEnvValue := os.Getenv("PECHECK_ENDPOINT") + if preCheckEndpointURLEnvValue != "" { + PreCheckEndpointURL = preCheckEndpointURLEnvValue + } else { + PreCheckEndpointURL = localEndpoint + } + } _ = generateCmd.MarkFlagRequired("github-token") rootCmd.AddCommand(generateCmd) }