diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..84c0cd2f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,29 @@ +--- +name: Bug Report +about: Create a report to help us improve +title: "[BUG]" +labels: kind/bug + +--- + + + +**What happened**: + +**What you expected to happen**: + +**How to reproduce it (as minimally and precisely as possible)**: + +**Anything else we need to know?**: + +**Environment**: +- App version: +- Kubernetes version (use `kubectl version`): +- Install details (e.g. helm install args): +- Node environment (for koordlet/runtime-proxy issue): + - Containerd/Docker version: + - OS version: + - Kernal version: + - Cgroup driver: cgroupfs/systemd +- Others: + diff --git a/.github/ISSUE_TEMPLATE/membership.yml b/.github/ISSUE_TEMPLATE/membership.yml new file mode 100644 index 00000000..6dd0f9fa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/membership.yml @@ -0,0 +1,56 @@ +name: Organization Membership Request +description: Request membership in a Koordinator Org +labels: ["kind/github-membership"] +title: "REQUEST: New membership for " +body: + - id: github + type: input + attributes: + label: GitHub Username + placeholder: e.g. @example_user + validations: + required: true + - id: requirements + type: checkboxes + attributes: + label: Requirements + options: + - label: I have reviewed the [community membership guidelines](https://github.com/koordinator-sh/koordinator/blob/main/docs/community/community-membership.md) + required: true + - label: I have [enabled 2FA on my GitHub account](https://github.com/settings/security) + required: true + - label: I am actively contributing to 1 or more Koordinator subprojects + required: true + - label: I have two sponsors that meet the sponsor requirements listed in the community membership guidelines + required: true + - label: I have spoken to my sponsors ahead of this application, and they have agreed to sponsor my application + required: true + - label: I have verified that my sponsors are a reviewer or an approver in at least one OWNERS file within one of the Koordinator GitHub organizations (excluding the contributor-playground) + required: true + - label: "**OPTIONAL:** I have taken the [Inclusive Open Source Community Orientation course](https://training.linuxfoundation.org/training/inclusive-open-source-community-orientation-lfc102/)" + - id: sponsor_1 + type: input + attributes: + label: "Sponsor 1" + description: GitHub handle of your sponsor + placeholder: e.g. @sponsor-1 + validations: + required: true + - id: sponsor_2 + type: input + attributes: + label: "Sponsor 2" + description: GitHub handle of your sponsor + placeholder: e.g. @sponsor-2 + validations: + required: true + - id: contributions + type: textarea + attributes: + label: List of contributions to the Koordinator project + placeholder: | + - PRs reviewed / authored + - Issues responded to + - Subprojects I am involved with + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/proposal.md b/.github/ISSUE_TEMPLATE/proposal.md new file mode 100644 index 00000000..368fd72b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/proposal.md @@ -0,0 +1,17 @@ +--- +name: Proposal +about: Suggest an idea for this project +title: "[proposal]" +labels: kind/proposal + +--- + + + +**What is your proposal**: + + +**Why is this needed**: + + +**Is there a suggested solution, if so, please add it**: diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 00000000..97c8665f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,25 @@ +--- +name: Question +about: Support request or question relating to Koordinator +title: "[question]" +labels: kind/question + +--- + + + +**What happened:** + +**What you expected to happen:** + +**Environment:** + + + +- Koordinator version: - v0.6.2 +- Kubernetes version (use kubectl version): v1.22.5 +- docker/containerd version: containerd 1.5.0 +- OS (e.g: cat /etc/os-release): Ubuntu 20.04.4 LTS +- Kernel (e.g. uname -a): Linux 5.10.112-11.al8.x86_64 #1 SMP Tue May 24 16:05:50 CST 2022 x86_64 x86_64 x86_64 GNU/Linux + +**Anything else we need to know:** diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..9fc3b075 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" + labels: + - "dependencies" + commit-message: + prefix: "feat" + include: "scope" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + labels: + - "dependencies" + commit-message: + prefix: "chore" + include: "scope" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..4514c604 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +### Ⅰ. Describe what this PR does + + + +### Ⅱ. Does this pull request fix one issue? + + + +### Ⅲ. Describe how to verify it + +### Ⅳ. Special notes for reviews + +### V. Checklist + +- [ ] I have written necessary docs and comments +- [ ] I have added necessary unit tests and integration tests +- [ ] All checks passed in `make test` diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 00000000..94f03686 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,40 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 90 + +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 30 + +# Issues with these labels will never be considered stale +exemptLabels: + - pinned + - security + +# Label to use when marking an issue as stale +staleLabel: lifecycle/stale + +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: | + This issue has been automatically marked as stale because it has not had recent activity. + This bot triages issues and PRs according to the following rules: + - After 90d of inactivity, `lifecycle/stale` is applied + - After 30d of inactivity since `lifecycle/stale` was applied, the issue is closed + You can: + - Mark this issue or PR as fresh with `/remove-lifecycle stale` + - Close this issue or PR with `/close` + Thank you for your contributions. + +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: | + This issue has been automatically closed because it has not had recent activity. + This bot triages issues and PRs according to the following rules: + - After 90d of inactivity, `lifecycle/stale` is applied + - After 30d of inactivity since `lifecycle/stale` was applied, the issue is closed + You can: + - Reopen this PR with `/reopen` + Thank you for your contributions. + +# Set to true to ignore issues in a project (defaults to false) +exemptProjects: true + +# Set to true to ignore issues in a milestone (defaults to false) +exemptMilestones: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..ece86cf3 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,123 @@ +name: CI + +on: + push: + branches: + - main + - release-* + pull_request: { } + workflow_dispatch: { } + +jobs: + golangci-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + cache: false + go-version-file: go.mod + - uses: golangci/golangci-lint-action@v3 + with: + version: v1.47.3 + + unit-tests: + strategy: + fail-fast: false + matrix: + include: + - step: mod + name: Check Go modules + command: go mod tidy && git add go.* && git diff --cached --exit-code || (echo 'Please run "go mod tidy" to sync Go modules' && exit 1); + - step: manifests + name: Check manifests + command: make manifests && git add config && git diff --cached --exit-code || (echo 'Please run "make manifests" to generate manifests' && exit 1); + - step: generate + name: Check auto-generated codes + command: make generate && git add pkg apis && git diff --cached --exit-code || (echo 'Please run "make generate" to generate Go codes' && exit 1); + - step: gofmt + name: Verify gofmt + command: make fmt && git add apis pkg cmd && git diff --cached --exit-code || (echo 'Please run "make fmt" to verify gofmt' && exit 1); + - step: govet + name: Verify govet + command: make vet && git add apis pkg cmd && git diff --cached --exit-code || (echo 'Please run "make vet" to verify govet' && exit 1); + - step: build + name: Run Go build + command: make build + - step: test + name: Run Go test + command: make fast-test + name: unit-tests(${{ matrix.name }}) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + cache: false + go-version-file: go.mod + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-ut-${{ matrix.step }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-ut-${{ matrix.step }}- + - name: ${{ matrix.name }} + run: | + ${{ matrix.command }} + + build-and-push: + needs: [ golangci-lint, unit-tests ] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: [ koordlet, koord-manager, koord-scheduler, koord-descheduler ] + env: + GITHUB_REG: ghcr.io + ALIYUN_BJ_REG: registry.cn-beijing.aliyuncs.com + ALIYUN_HZ_REG: registry.cn-hangzhou.aliyuncs.com + steps: + - uses: docker/setup-buildx-action@v2 + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ${{ env.GITHUB_REG }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to AliyunCS_BJ + uses: docker/login-action@v2 + with: + registry: ${{ env.ALIYUN_BJ_REG }} + username: ${{ secrets.ALIYUN_USERNAME }} + password: ${{ secrets.ALIYUN_PWD }} + - name: Login to AliyunCS_HZ + uses: docker/login-action@v2 + with: + registry: ${{ env.ALIYUN_HZ_REG }} + username: ${{ secrets.ALIYUN_USERNAME }} + password: ${{ secrets.ALIYUN_PWD }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + platforms: linux/amd64 + push: true + pull: true + file: docker/${{ matrix.target }}.dockerfile + labels: | + org.opencontainers.image.title=${{ matrix.target }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at}} + org.opencontainers.image.licenses=Apache-2.0 + tags: | + ${{ env.GITHUB_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:latest + ${{ env.GITHUB_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.sha }} + ${{ env.ALIYUN_BJ_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:latest + ${{ env.ALIYUN_BJ_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.sha }} + ${{ env.ALIYUN_HZ_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:latest + ${{ env.ALIYUN_HZ_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.sha }} + cache-from: type=gha,scope=build-${{ matrix.target }} + cache-to: type=gha,mode=max,scope=build-${{ matrix.target }} diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml new file mode 100644 index 00000000..83994a70 --- /dev/null +++ b/.github/workflows/license.yml @@ -0,0 +1,30 @@ +name: License + +on: + push: + branches: + - main + - release-* + pull_request: {} + workflow_dispatch: {} + +jobs: + license-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.6 + - name: Check license + run: | + gem install license_finder + license_finder --decisions_file .license/dependency_decisions.yml + - uses: actions/setup-go@v4 + with: + cache: false + go-version-file: go.mod + - name: Check license header + run: | + make lint-license && git add apis pkg cmd || exit 1 + git diff --cached --exit-code || (echo 'Please run "make lint-license" to verify license header' && exit 1); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..9db01d9e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,87 @@ +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: [ yarn-opeartor, yarn-copilot ] + env: + GITHUB_REG: ghcr.io + ALIYUN_BJ_REG: registry.cn-beijing.aliyuncs.com + ALIYUN_HZ_REG: registry.cn-hangzhou.aliyuncs.com + steps: + - uses: docker/setup-buildx-action@v2 + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ${{ env.GITHUB_REG }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to AliyunCS_BJ + uses: docker/login-action@v2 + with: + registry: ${{ env.ALIYUN_BJ_REG }} + username: ${{ secrets.ALIYUN_USERNAME }} + password: ${{ secrets.ALIYUN_PWD }} + - name: Login to AliyunCS_HZ + uses: docker/login-action@v2 + with: + registry: ${{ env.ALIYUN_HZ_REG }} + username: ${{ secrets.ALIYUN_USERNAME }} + password: ${{ secrets.ALIYUN_PWD }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + platforms: linux/amd64 + push: true + pull: true + file: docker/${{ matrix.target }}.dockerfile + labels: | + org.opencontainers.image.title=${{ matrix.target }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at}} + org.opencontainers.image.licenses=Apache-2.0 + tags: | + ${{ env.GITHUB_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.ref_name }} + ${{ env.ALIYUN_BJ_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.ref_name }} + ${{ env.ALIYUN_HZ_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.ref_name }} + cache-from: type=gha,scope=build-${{ matrix.target }} + cache-to: type=gha,mode=max,scope=build-${{ matrix.target }} + goreleaser: + runs-on: ubuntu-latest + env: + GITHUB_REG: ghcr.io + ALIYUN_BJ_REG: registry.cn-beijing.aliyuncs.com + ALIYUN_HZ_REG: registry.cn-hangzhou.aliyuncs.com + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Fetch all tags + run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v4 + with: + cache: false + go-version-file: go.mod + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.license/dependency_decisions.yml b/.license/dependency_decisions.yml new file mode 100644 index 00000000..f721d441 --- /dev/null +++ b/.license/dependency_decisions.yml @@ -0,0 +1,43 @@ +--- +- - :permit + - MIT + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:35:34.645031000 Z +- - :permit + - Apache 2.0 + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:19:18.243194000 Z +- - :permit + - New BSD + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:19:28.540675000 Z +- - :permit + - Simplified BSD + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:20:01.774212000 Z +- - :permit + - Mozilla Public License 2.0 + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:21:05.194536000 Z +- - :permit + - unknown + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:21:43.379269000 Z +- - :permit + - ISC + - :who: + :why: + :versions: [] + :when: 2021-03-12 07:22:07.265966000 Z diff --git a/Makefile b/Makefile index 0e4309a6..6250070c 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="$(LICENSE_HEADER_GO)" paths="./api/..." + $(CONTROLLER_GEN) object:headerFile="$(LICENSE_HEADER_GO)" paths="./apis/..." .PHONY: fmt fmt: ## Run go fmt against code. @@ -94,7 +94,7 @@ fast-test: envtest ## Run tests fast. ##@ Build .PHONY: build -build: build-yarn-copilot build-yarn-operator +build: build-yarn-operator .PHONY: build-yarn-copilot build-yarn-copilot: ## Build yarn-copilot binary. diff --git a/apis/doc.go b/apis/doc.go new file mode 100644 index 00000000..26badb20 --- /dev/null +++ b/apis/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 20b8987a..5f5230ec 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: koord-yarn-operator + name: manager-role rules: - apiGroups: - "" @@ -13,18 +13,6 @@ rules: - get - list - watch - - create - - update -- apiGroups: - - "" - resources: - - events - verbs: - - get - - list - - watch - - create - - update - apiGroups: - "" resources: @@ -33,30 +21,3 @@ rules: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: koordinator-system - name: koord-yarn-operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: koord-yarn-operator-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: koord-yarn-operator -subjects: - - kind: ServiceAccount - name: koord-yarn-operator - namespace: koordinator-system \ No newline at end of file diff --git a/pkg/yarn/client/ipc/client.go b/pkg/yarn/client/ipc/client.go index bbc7d4e4..6084f9cb 100644 --- a/pkg/yarn/client/ipc/client.go +++ b/pkg/yarn/client/ipc/client.go @@ -53,10 +53,10 @@ type connection_id struct { } type call struct { - callId int32 - procedure proto.Message - request proto.Message - response proto.Message + callId int32 + procedure proto.Message + request proto.Message + response proto.Message // err *error retryCount int32 }