Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry pick optimize pod initializer on release-0.10 #877

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cmd/craned/app/manager.go
Original file line number Diff line number Diff line change
@@ -106,7 +106,12 @@ func Run(ctx context.Context, opts *options.Options) error {
return err
}

if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
klog.ErrorS(err, "failed to add health check endpoint")
return err
}

if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
klog.ErrorS(err, "failed to add health check endpoint")
return err
}
2 changes: 1 addition & 1 deletion deploy/crane-agent/daemonset.yaml
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ spec:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: docker.io/gocrane/crane-agent:v0.9.0
image: docker.io/gocrane/crane-agent:v0.10.0
imagePullPolicy: Always
command:
- /crane-agent
4 changes: 2 additions & 2 deletions deploy/craned/deployment.yaml
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@ spec:
operator: Exists
containers:
- name: craned
image: docker.io/gocrane/craned:v0.9.0
image: docker.io/gocrane/craned:v0.10.0
imagePullPolicy: IfNotPresent
env:
- name: TZ
@@ -61,7 +61,7 @@ spec:
path: /api/healthz
port: 8082
scheme: HTTP
- image: docker.io/gocrane/dashboard:v0.8.0
- image: docker.io/gocrane/dashboard:v0.10.0
imagePullPolicy: IfNotPresent
name: dashboard
volumeMounts:
Empty file removed deploy/craned/new.yaml
Empty file.
4 changes: 4 additions & 0 deletions deploy/manifests/analysis.crane.io_recommendationrules.yaml
Original file line number Diff line number Diff line change
@@ -287,6 +287,10 @@ spec:
type: object
type: array
x-kubernetes-list-type: atomic
runNumber:
description: RunNumber is the numbers of runs
format: int32
type: integer
type: object
type: object
served: true
2 changes: 1 addition & 1 deletion deploy/metric-adapter/deployment.yaml
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ spec:
serviceAccountName: metric-adapter
containers:
- name: metric-adapter
image: docker.io/gocrane/metric-adapter:v0.9.0
image: docker.io/gocrane/metric-adapter:v0.10.0
imagePullPolicy: IfNotPresent
env:
- name: TZ
7 changes: 1 addition & 6 deletions go.mod
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ require (
github.com/google/cadvisor v0.39.2
github.com/jaypipes/ghw v0.9.0
github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.26.0
github.com/shirou/gopsutil v3.21.10+incompatible
@@ -52,12 +53,10 @@ require (
)

require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/alessio/shellescape v1.4.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
@@ -76,7 +75,6 @@ require (
github.com/emicklei/go-restful v2.15.0+incompatible // indirect
github.com/emicklei/go-restful-swagger12 v0.0.0-20201014110547-68ccff494617 // indirect
github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.2.0 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
@@ -121,8 +119,6 @@ require (
github.com/opencontainers/runc v1.0.2 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/opencontainers/selinux v1.8.2 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
@@ -169,7 +165,6 @@ require (
k8s.io/kube-scheduler v0.0.0 // indirect
k8s.io/mount-utils v0.22.3 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect
sigs.k8s.io/kind v0.11.1 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
)

15 changes: 2 additions & 13 deletions go.sum
Original file line number Diff line number Diff line change
@@ -93,8 +93,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -260,8 +258,6 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdR
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.2.0 h1:8ozOH5xxoMYDt5/u+yMTsVXydVCbTORFnOOoq2lumco=
github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -352,10 +348,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.1.0-rc.5 h1:QOAag7FoBaBYYHRqzqkhhd8fq5RTubvI4v3Ft/gDVVQ=
github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/gocrane/api v0.0.0-20230117025609-8b3a495647d9 h1:3R06LMq6zl+LyYBlmoVPXjY2E3XtsMqZ7WPBA+BMa/M=
github.com/gocrane/api v0.0.0-20230117025609-8b3a495647d9/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk=
github.com/gocrane/api v0.9.1-0.20230307114903-ce6fcd9a2eaf h1:HwjISCuAIPI/ZqzLDmw5WL1Yz0vjshzXt/3vNXEsCJs=
github.com/gocrane/api v0.9.1-0.20230307114903-ce6fcd9a2eaf/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk=
github.com/gocrane/api v0.10.0 h1:qzYFg9I2NNuvkZ7PZKCOmz4oEnRa+hvl68rEW12ZGdI=
github.com/gocrane/api v0.10.0/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk=
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -722,8 +716,6 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -822,7 +814,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
@@ -1515,8 +1506,6 @@ sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpv
sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
sigs.k8s.io/custom-metrics-apiserver v1.22.0 h1:nRrRRCq46m3y6lCp/6rfptPjX0eGsF88s66vt9TWgac=
sigs.k8s.io/custom-metrics-apiserver v1.22.0/go.mod h1:QST5+Nu7RXcDVbg19K+0UT+2QMxnw+FSB6q7sv3yDUw=
sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA=
sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA=
sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=
sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs=
sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go=
4 changes: 0 additions & 4 deletions initializer/Dockerfile

This file was deleted.

4 changes: 0 additions & 4 deletions initializer/podinfo

This file was deleted.

22 changes: 0 additions & 22 deletions initializer/qos-checking.sh

This file was deleted.

14 changes: 12 additions & 2 deletions pkg/controller/analytics/analytics_controller.go
Original file line number Diff line number Diff line change
@@ -546,8 +546,11 @@ func setReadyCondition(status *analysisv1alph1.AnalyticsStatus, conditionStatus
func ConvertToRecommendationRule(analytics *analysisv1alph1.Analytics) *analysisv1alph1.RecommendationRule {
recommendationRule := &analysisv1alph1.RecommendationRule{}
recommendationRule.Name = analytics.Name
if recommendationRule.Annotations == nil {
recommendationRule.Annotations = map[string]string{}
}
recommendationRule.Annotations[known.AnalyticsConversionAnnotation] = string(analytics.UID)
recommendationRule.Spec.ResourceSelectors = analytics.Spec.ResourceSelectors
// todo: make sure the conversion is right after recommendation refactor
recommendationRule.Spec.Recommenders = []analysisv1alph1.Recommender{{Name: string(analytics.Spec.Type)}}
if analytics.Namespace == known.CraneSystemNamespace {
recommendationRule.Spec.NamespaceSelector.Any = true
@@ -560,6 +563,11 @@ func ConvertToRecommendationRule(analytics *analysisv1alph1.Analytics) *analysis
recommendationRule.Spec.RunInterval = (time.Duration(*analytics.Spec.CompletionStrategy.PeriodSeconds) * time.Second).String()
}

recommendationRule.Status = analysisv1alph1.RecommendationRuleStatus{
LastUpdateTime: analytics.Status.LastUpdateTime,
Recommendations: analytics.Status.Recommendations,
}

return recommendationRule
}

@@ -572,8 +580,10 @@ func UpsertRecommendationRule(recommendationRule *analysisv1alph1.Recommendation
return err
}

if !reflect.DeepEqual(recommendationRule.Spec, recommendationRuleExist.Spec) {
if !reflect.DeepEqual(recommendationRule.Spec, recommendationRuleExist.Spec) ||
reflect.DeepEqual(recommendationRule.Annotations, recommendationRuleExist.Annotations) {
recommendationRuleExist.Spec = recommendationRule.Spec
recommendationRuleExist.Annotations = recommendationRule.Annotations
return client.Update(context.TODO(), recommendationRuleExist)
}

22 changes: 22 additions & 0 deletions pkg/controller/recommendation/recommendation_rule_controller.go
Original file line number Diff line number Diff line change
@@ -157,6 +157,12 @@ func (c *RecommendationRuleController) doReconcile(ctx context.Context, recommen
opts := []client.ListOption{
client.MatchingLabels(map[string]string{known.RecommendationRuleUidLabel: string(recommendationRule.UID)}),
}
if convert, uid := IsConvertFromAnalytics(recommendationRule); convert {
opts = []client.ListOption{
client.MatchingLabels(map[string]string{known.AnalyticsUidLabel: uid}),
}
}

err = c.Client.List(ctx, &currRecommendations, opts...)
if err != nil {
c.Recorder.Event(recommendationRule, corev1.EventTypeWarning, "FailedSelectResource", err.Error())
@@ -378,6 +384,9 @@ func CreateRecommendationObject(recommendationRule *analysisv1alph1.Recommendati
if id.Namespace != "" {
namespace = id.Namespace
}
if convert, _ := IsConvertFromAnalytics(recommendationRule); convert {
namespace = known.CraneSystemNamespace
}

recommendation := &analysisv1alph1.Recommendation{
ObjectMeta: metav1.ObjectMeta{
@@ -396,6 +405,9 @@ func CreateRecommendationObject(recommendationRule *analysisv1alph1.Recommendati
labels := map[string]string{}
labels[known.RecommendationRuleNameLabel] = recommendationRule.Name
labels[known.RecommendationRuleUidLabel] = string(recommendationRule.UID)
if convert, uid := IsConvertFromAnalytics(recommendationRule); convert {
labels[known.AnalyticsUidLabel] = uid
}
labels[known.RecommendationRuleRecommenderLabel] = recommenderName
labels[known.RecommendationRuleTargetKindLabel] = target.Kind
labels[known.RecommendationRuleTargetVersionLabel] = target.GroupVersionKind().Version
@@ -483,3 +495,13 @@ func executeMission(ctx context.Context, wg *sync.WaitGroup, recommenderMgr reco
mission.APIVersion = recommendation.APIVersion
}
}

func IsConvertFromAnalytics(recommendationRule *analysisv1alph1.RecommendationRule) (bool, string) {
if recommendationRule.Annotations != nil {
if uid, exist := recommendationRule.Annotations[known.AnalyticsConversionAnnotation]; exist {
return true, uid
}
}

return false, ""
}
Original file line number Diff line number Diff line change
@@ -54,7 +54,7 @@ func (c *RecommendationTriggerController) Reconcile(ctx context.Context, req ctr

recommendationRuleRef := utils.GetRecommendationRuleOwnerReference(recommendation)
if recommendationRuleRef == nil {
klog.Warningf("cannot found referred recommendation rule %s/%s", recommendation.Namespace, recommendationRuleRef.Name)
klog.Warning("cannot found referred recommendation rule")
return ctrl.Result{}, nil
}

3 changes: 2 additions & 1 deletion pkg/ensurance/analyzer/analyzer.go
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@ import (
ecache "github.com/gocrane/crane/pkg/ensurance/cache"
"github.com/gocrane/crane/pkg/ensurance/executor"
"github.com/gocrane/crane/pkg/ensurance/executor/podinfo"
"github.com/gocrane/crane/pkg/ensurance/util"
"github.com/gocrane/crane/pkg/known"
"github.com/gocrane/crane/pkg/metrics"
"github.com/gocrane/crane/pkg/utils"
@@ -490,7 +491,7 @@ func (s *AnomalyAnalyzer) filterPodQOSMatches(pods []*v1.Pod, actionName string)
}
for _, qos := range podQOSList {
for _, pod := range pods {
if !match(pod, qos) {
if !util.MatchPodAndPodQOS(pod, qos) {
klog.V(4).Infof("Pod %s/%s does not match PodQOS %s", pod.Namespace, pod.Name, qos.Name)
continue

Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package analyzer
package util

import (
"fmt"
"reflect"
"sort"
"strconv"
"strings"

@@ -69,7 +70,41 @@ func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]strin
return true
}

func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool {
func sortQOSSlice(qosSlice []*ensuranceapi.PodQOS) {
sort.Slice(qosSlice, func(i, j int) bool {
if len(qosSlice[i].Spec.LabelSelector.MatchLabels) != len(qosSlice[j].Spec.LabelSelector.MatchLabels) {
return len(qosSlice[i].Spec.LabelSelector.MatchLabels) > len(qosSlice[j].Spec.LabelSelector.MatchLabels)
}

if qosSlice[i].Spec.ScopeSelector == nil && qosSlice[j].Spec.ScopeSelector == nil {
return true
}

if qosSlice[i].Spec.ScopeSelector == nil {
return false
}

if qosSlice[j].Spec.ScopeSelector == nil {
return true
}

return len(qosSlice[i].Spec.ScopeSelector.MatchExpressions) > len(qosSlice[j].Spec.ScopeSelector.MatchExpressions)
})
}

func MatchPodAndPodQOSSlice(pod *v1.Pod, qosSlice []*ensuranceapi.PodQOS) (res *ensuranceapi.PodQOS) {
newSlice := make([]*ensuranceapi.PodQOS, len(qosSlice), len(qosSlice))
copy(newSlice, qosSlice)
sortQOSSlice(newSlice)
for _, qos := range newSlice {
if MatchPodAndPodQOS(pod, qos) {
return qos
}
}
return nil
}

func MatchPodAndPodQOS(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool {

if podQOS.Spec.ScopeSelector == nil &&
podQOS.Spec.LabelSelector.MatchLabels == nil &&
1 change: 1 addition & 0 deletions pkg/known/annotation.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ const (
ReplicasRecommendationValueAnnotation = "analysis.crane.io/replicas-recommendation"
ResourceRecommendationValueAnnotation = "analysis.crane.io/resource-recommendation"
RunNumberAnnotation = "analysis.crane.io/run-number"
AnalyticsConversionAnnotation = "analysis.crane.io/analytics-conversion"
)

const (
5 changes: 3 additions & 2 deletions pkg/providers/grpc/pb/provider.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/providers/grpc/pb/provider_grpc.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

78 changes: 66 additions & 12 deletions pkg/webhooks/pod/mutating.go
Original file line number Diff line number Diff line change
@@ -4,17 +4,29 @@ import (
"context"
"fmt"

"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog/v2"

"github.com/gocrane/api/ensurance/v1alpha1"

"github.com/gocrane/crane/pkg/ensurance/config"
"github.com/gocrane/crane/pkg/ensurance/util"
)

type MutatingAdmission struct {
Config *config.QOSConfig
Config *config.QOSConfig
listPodQOS func() ([]*v1alpha1.PodQOS, error)
}

func NewMutatingAdmission(config *config.QOSConfig, listPodQOS func() ([]*v1alpha1.PodQOS, error)) *MutatingAdmission {
return &MutatingAdmission{
Config: config,
listPodQOS: listPodQOS,
}
}

// Default implements webhook.Defaulter so a webhook will be registered for the type
@@ -23,13 +35,10 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err
if !ok {
return fmt.Errorf("expected a Pod but got a %T", obj)
}
klog.Infof("Into Pod injection %s/%s", pod.Namespace, pod.Name)

if m.Config == nil || !m.Config.QOSInitializer.Enable {
return nil
}
klog.V(2).Infof("Mutating started for pod %s/%s", pod.Namespace, pod.Name)

if pod.Labels == nil {
if !m.available() {
return nil
}

@@ -38,17 +47,62 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err
return err
}

if ls.Matches(labels.Set(pod.Labels)) {
if m.Config.QOSInitializer.InitContainerTemplate != nil {
pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate)
if !ls.Matches(labels.Set(pod.Labels)) {
klog.V(2).Infof("Injection skipped: webhook is not interested in the pod")
return nil
}

qosSlice, err := m.listPodQOS()
if err != nil {
return errors.WithMessage(err, "list PodQOS failed")
}

/****************************************************************
* Check whether the pod has a low CPUPriority (CPUPriority > 0)
****************************************************************/
qos := util.MatchPodAndPodQOSSlice(pod, qosSlice)
if qos == nil {
klog.V(2).Infof("Injection skipped: no podqos matched")
return nil
}

if qos.Spec.ResourceQOS.CPUQOS == nil ||
qos.Spec.ResourceQOS.CPUQOS.CPUPriority == nil ||
*qos.Spec.ResourceQOS.CPUQOS.CPUPriority == 0 {
klog.V(2).Infof("Injection skipped: not a low CPUPriority pod, qos %s", qos.Name)
return nil
}

for _, container := range pod.Spec.InitContainers {
if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name {
klog.V(2).Infof("Injection skipped: pod has initializerContainer already")
return nil
}
}

if m.Config.QOSInitializer.VolumeTemplate != nil {
pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate)
for _, volume := range pod.Spec.Volumes {
if volume.Name == m.Config.QOSInitializer.VolumeTemplate.Name {
klog.V(2).Infof("Injection skipped: pod has initializerVolume already")
return nil
}
}

if m.Config.QOSInitializer.InitContainerTemplate != nil {
pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate)
}

klog.Infof("Injected QOSInitializer for Pod %s/%s", pod.Namespace, pod.Name)
if m.Config.QOSInitializer.VolumeTemplate != nil {
pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate)
}

klog.V(2).Infof("Mutating completed for pod %s/%s", pod.Namespace, pod.Name)

return nil
}

func (m *MutatingAdmission) available() bool {
return m.Config != nil &&
m.Config.QOSInitializer.Enable &&
m.Config.QOSInitializer.InitContainerTemplate != nil &&
m.Config.QOSInitializer.VolumeTemplate != nil
}
104 changes: 94 additions & 10 deletions pkg/webhooks/pod/mutating_test.go
Original file line number Diff line number Diff line change
@@ -4,10 +4,14 @@ import (
"context"
"testing"

"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
"sigs.k8s.io/yaml"

"github.com/gocrane/api/ensurance/v1alpha1"

"github.com/gocrane/crane/pkg/ensurance/config"
)

@@ -20,22 +24,102 @@ func TestDefaultingPodQOSInitializer(t *testing.T) {
t.Errorf("unmarshal config failed:%v", err)
}
m := MutatingAdmission{
Config: config,
Config: config,
listPodQOS: MockListPodQOSFunc,
}

type Case struct {
Pod *v1.Pod
Inject bool
}

for _, tc := range []Case{
{Pod: MockPod("offline", "offline", "enable", "app", "nginx"), Inject: true},
{Pod: MockPod("offline-not-interested", "offline", "enable"), Inject: false},
{Pod: MockPod("online", "offline", "disable", "app", "nginx"), Inject: false},
{Pod: MockPod("online-not-interested", "offline", "disable"), Inject: false},
{Pod: MockPod("default"), Inject: false},
} {
assert.NoError(t, m.Default(context.Background(), tc.Pod))
t.Log(tc.Pod.Name)
assert.Equal(t, len(tc.Pod.Spec.InitContainers) == 1, tc.Inject)
assert.Equal(t, len(tc.Pod.Spec.Volumes) == 1, tc.Inject)
}
}

func TestPrecheck(t *testing.T) {
configYaml := "apiVersion: ensurance.crane.io/v1alpha1\nkind: QOSConfig\nqosInitializer:\n enable: true\n selector: \n matchLabels:\n app: nginx\n"

config := &config.QOSConfig{}
err := yaml.Unmarshal([]byte(configYaml), config)
if err != nil {
t.Errorf("unmarshal config failed:%v", err)
}
m := MutatingAdmission{
Config: config,
listPodQOS: MockListPodQOSFunc,
}
assert.False(t, m.available())
}

func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) {
return []*v1alpha1.PodQOS{
{
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{},
Spec: v1alpha1.PodQOSSpec{
LabelSelector: metav1.LabelSelector{
MatchLabels: map[string]string{"offline": "enable"},
},
ResourceQOS: v1alpha1.ResourceQOS{
CPUQOS: &v1alpha1.CPUQOS{
CPUPriority: pointer.Int32(7),
},
},
},
}, {
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{},
Spec: v1alpha1.PodQOSSpec{
LabelSelector: metav1.LabelSelector{
MatchLabels: map[string]string{"offline": "disable"},
},
ResourceQOS: v1alpha1.ResourceQOS{
CPUQOS: &v1alpha1.CPUQOS{
CPUPriority: pointer.Int32(0),
},
},
},
}, {
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{},
Spec: v1alpha1.PodQOSSpec{
ResourceQOS: v1alpha1.ResourceQOS{
CPUQOS: &v1alpha1.CPUQOS{
CPUPriority: pointer.Int32(7),
},
},
},
},
}, nil
}

func MockPod(name string, labels ...string) *v1.Pod {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Labels: map[string]string{
"app": "nginx",
},
Name: name,
Labels: nil,
},
}
err = m.Default(context.TODO(), pod)
if err != nil {
t.Fatalf("inject pod failed: %v", err)

if len(labels) < 2 {
return pod
}
if len(pod.Spec.InitContainers) == 0 {
t.Fatalf("should inject containers")

labelmap := map[string]string{}
for i := 0; i < len(labels)-1; i += 2 {
labelmap[labels[i]] = labels[i+1]
}
pod.Labels = labelmap
return pod
}
26 changes: 19 additions & 7 deletions pkg/webhooks/webhook.go
Original file line number Diff line number Diff line change
@@ -17,14 +17,15 @@ limitations under the License.
package webhooks

import (
corev1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"context"

analysisapi "github.com/gocrane/api/analysis/v1alpha1"
autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1"
ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1"
predictionapi "github.com/gocrane/api/prediction/v1alpha1"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"

"github.com/gocrane/crane/pkg/ensurance/config"
"github.com/gocrane/crane/pkg/webhooks/autoscaling"
@@ -109,12 +110,10 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE
klog.Errorf("Failed to load qos initializer config: %v", err)
}

podMutatingAdmission := pod.MutatingAdmission{
Config: qosConfig,
}
podMutatingAdmission := pod.NewMutatingAdmission(qosConfig, BuildPodQosListFunction(mgr))
err = ctrl.NewWebhookManagedBy(mgr).
For(&corev1.Pod{}).
WithDefaulter(&podMutatingAdmission).
WithDefaulter(podMutatingAdmission).
Complete()
if err != nil {
klog.Errorf("Failed to setup qos initializer webhook: %v", err)
@@ -124,3 +123,16 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE

return nil
}

func BuildPodQosListFunction(mgr ctrl.Manager) func() ([]*ensuranceapi.PodQOS, error) {
return func() (qosSlice []*ensuranceapi.PodQOS, err error) {
podQOSList := ensuranceapi.PodQOSList{}
if err := mgr.GetCache().List(context.Background(), &podQOSList); err != nil {
return nil, err
}
for _, qos := range podQOSList.Items {
qosSlice = append(qosSlice, qos.DeepCopy())
}
return qosSlice, err
}
}
22 changes: 15 additions & 7 deletions tools/initializer/resource.yaml
Original file line number Diff line number Diff line change
@@ -99,28 +99,36 @@ data:
kind: QOSConfig
qosInitializer:
enable: true
selector:
selector:
matchLabels:
app: nginx
initContainerTemplate:
name: crane-qos-initializer
image: docker.io/gocrane/qos-init:v0.1.4
imagePullPolicy: IfNotPresent
args:
- "while ! grep -q gocrane.io/cpu-qos /etc/podinfo/annotations; do sleep 1; done; echo cpu qos setting competed;"
command:
- sh
- -x
- /qos-checking.sh
- /bin/bash
- -c
resources:
requests:
cpu: 10m
memory: 10Mi
limits:
cpu: 10m
memory: 10Mi
volumeMounts:
- name: podinfo
- name: crane-qos-initializer-volume
mountPath: /etc/podinfo
volumeTemplate:
name: podinfo
name: crane-qos-initializer-volume
downwardAPI:
items:
- path: "annotations"
fieldRef:
fieldPath: metadata.annotations
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration