From 55f4b8c040964131bb5ca0e6fda638d852b2afdc Mon Sep 17 00:00:00 2001 From: Luca Kuendig Date: Tue, 31 Jan 2023 10:43:01 +0100 Subject: [PATCH] add replicas.idle to set idleReplicaCount on SO Signed-off-by: Luca Kuendig --- .../bases/http.keda.sh_httpscaledobjects.yaml | 8 ++ e2e/k8s.go | 28 +++++- examples/xkcd/templates/httpscaledobject.yaml | 1 + examples/xkcd/values.yaml | 1 + .../api/v1alpha1/httpscaledobject_types.go | 5 +- operator/controllers/scaled_object.go | 16 +--- operator/controllers/scaled_object_test.go | 37 ++++---- operator/controllers/suite_test.go | 6 ++ pkg/k8s/scaledobject.go | 92 ++++++++++--------- pkg/k8s/templates/scaledobject.yaml | 25 +++++ 10 files changed, 144 insertions(+), 75 deletions(-) create mode 100644 pkg/k8s/templates/scaledobject.yaml diff --git a/config/crd/bases/http.keda.sh_httpscaledobjects.yaml b/config/crd/bases/http.keda.sh_httpscaledobjects.yaml index ac806459..094cc145 100644 --- a/config/crd/bases/http.keda.sh_httpscaledobjects.yaml +++ b/config/crd/bases/http.keda.sh_httpscaledobjects.yaml @@ -33,6 +33,9 @@ spec: - jsonPath: .spec.replicas.max name: MaxReplicas type: integer + - jsonPath: .spec.replicas.idle + name: IdleReplicas + type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -66,6 +69,11 @@ spec: replicas: description: (optional) Replica information properties: + idle: + description: Amount of replicas to have in the deployment while + being idle + format: int32 + type: integer max: description: Maximum amount of replicas to have in the deployment (Default 100) diff --git a/e2e/k8s.go b/e2e/k8s.go index 1f3a7b6d..56e2c61d 100644 --- a/e2e/k8s.go +++ b/e2e/k8s.go @@ -4,7 +4,6 @@ import ( "context" "github.com/codeskyblue/go-sh" - kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/pkg/errors" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,7 +32,28 @@ func deleteNS(ns string) error { return sh.Command("kubectl", "delete", "namespace", ns).Run() } -func getScaledObject(ctx context.Context, cl client.Client, ns string, name string) error { - var scaledObject kedav1alpha1.ScaledObject - return cl.Get(ctx, k8s.ObjKey(ns, name), &scaledObject) +func getScaledObject( + ctx context.Context, + cl client.Client, + ns, + name string, +) error { + scaledObject, err := k8s.NewScaledObject( + ns, + name, + "", + "", + "", + 1, + 2, + 0, + 30, + ) + if err != nil { + return err + } + if err := cl.Get(ctx, k8s.ObjKey(ns, name), scaledObject); err != nil { + return err + } + return nil } diff --git a/examples/xkcd/templates/httpscaledobject.yaml b/examples/xkcd/templates/httpscaledobject.yaml index 613280ee..c5a8547e 100644 --- a/examples/xkcd/templates/httpscaledobject.yaml +++ b/examples/xkcd/templates/httpscaledobject.yaml @@ -12,3 +12,4 @@ spec: replicas: min: {{ .Values.autoscaling.http.minReplicas }} max: {{ .Values.autoscaling.http.maxReplicas }} + idle: {{ .Values.autoscaling.http.idleReplicas }} diff --git a/examples/xkcd/values.yaml b/examples/xkcd/values.yaml index 716eb9db..8aefe659 100644 --- a/examples/xkcd/values.yaml +++ b/examples/xkcd/values.yaml @@ -46,3 +46,4 @@ autoscaling: http: minReplicas: 0 maxReplicas: 10 + idleReplicas: 0 diff --git a/operator/api/v1alpha1/httpscaledobject_types.go b/operator/api/v1alpha1/httpscaledobject_types.go index 3e59a7c0..a4c426cb 100644 --- a/operator/api/v1alpha1/httpscaledobject_types.go +++ b/operator/api/v1alpha1/httpscaledobject_types.go @@ -83,7 +83,9 @@ type ReplicaStruct struct { // Minimum amount of replicas to have in the deployment (Default 0) Min *int32 `json:"min,omitempty" description:"Minimum amount of replicas to have in the deployment (Default 0)"` // Maximum amount of replicas to have in the deployment (Default 100) - Max *int32 `json:"max,omitempty" description:"Maximum amount of replicas to have in the deployment (Default 100)"` + Max int32 `json:"max,omitempty" description:"Maximum amount of replicas to have in the deployment (Default 100)"` + // Amount of replicas to have in the deployment while being idle + Idle int32 `json:"idle,omitempty" description:"Amount of replicas to have in the deployment while being idle"` } // HTTPScaledObjectSpec defines the desired state of HTTPScaledObject @@ -133,6 +135,7 @@ type HTTPScaledObjectStatus struct { // +kubebuilder:printcolumn:name="ScaleTargetPort",type="integer",JSONPath=".spec.scaleTargetRef" // +kubebuilder:printcolumn:name="MinReplicas",type="integer",JSONPath=".spec.replicas.min" // +kubebuilder:printcolumn:name="MaxReplicas",type="integer",JSONPath=".spec.replicas.max" +// +kubebuilder:printcolumn:name="IdleReplicas",type="integer",JSONPath=".spec.replicas.idle" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Active",type="string",JSONPath=".status.conditions[?(@.type==\"HTTPScaledObjectIsReady\")].status" diff --git a/operator/controllers/scaled_object.go b/operator/controllers/scaled_object.go index f2fdf20d..718ef168 100644 --- a/operator/controllers/scaled_object.go +++ b/operator/controllers/scaled_object.go @@ -26,22 +26,16 @@ func createOrUpdateScaledObject( httpso *v1alpha1.HTTPScaledObject, ) error { logger.Info("Creating scaled objects", "external scaler host name", externalScalerHostName) - - var minReplicaCount *int32 - var maxReplicaCount *int32 - if replicas := httpso.Spec.Replicas; replicas != nil { - minReplicaCount = replicas.Min - maxReplicaCount = replicas.Max - } - - appScaledObject := k8s.NewScaledObject( + logger.Info("This is the current httpso object", "output", httpso.Spec) + appScaledObject, appErr := k8s.NewScaledObject( httpso.GetNamespace(), fmt.Sprintf("%s-app", httpso.GetName()), // HTTPScaledObject name is the same as the ScaledObject name httpso.Spec.ScaleTargetRef.Deployment, externalScalerHostName, httpso.Spec.Host, - minReplicaCount, - maxReplicaCount, + httpso.Spec.Replicas.Min, + httpso.Spec.Replicas.Max, + httpso.Spec.Replicas.Idle, httpso.Spec.CooldownPeriod, ) diff --git a/operator/controllers/scaled_object_test.go b/operator/controllers/scaled_object_test.go index 401c4c4c..b340677a 100644 --- a/operator/controllers/scaled_object_test.go +++ b/operator/controllers/scaled_object_test.go @@ -56,32 +56,29 @@ func TestCreateOrUpdateScaledObject(t *testing.T) { metadata.Name, ) - var minReplicaCount *int32 - var maxReplicaCount *int32 - if replicas := testInfra.httpso.Spec.Replicas; replicas != nil { - minReplicaCount = replicas.Min - maxReplicaCount = replicas.Max - } - - r.EqualValues( - minReplicaCount, - spec.MinReplicaCount, + // HTTPScaledObject min/max/idle replicas are int32s, + // but the ScaledObject's spec is decoded into + // an *unsructured.Unstructured (basically a map[string]interface{}) + // which is an int64. we need to convert the + // HTTPScaledObject's values into int64s before we compare + r.Equal( + int64(testInfra.httpso.Spec.Replicas.Min), + spec["minReplicaCount"], ) r.EqualValues( maxReplicaCount, spec.MaxReplicaCount, ) + r.Equal( + int64(testInfra.httpso.Spec.Replicas.Idle), + spec["idleReplicaCount"], + ) // now update the min and max replicas on the httpso // and call createOrUpdateScaledObject again - if spec := &testInfra.httpso.Spec; spec.Replicas == nil { - spec.Replicas = &v1alpha1.ReplicaStruct{ - Min: new(int32), - Max: new(int32), - } - } - *testInfra.httpso.Spec.Replicas.Min++ - *testInfra.httpso.Spec.Replicas.Max++ + testInfra.httpso.Spec.Replicas.Min++ + testInfra.httpso.Spec.Replicas.Max++ + testInfra.httpso.Spec.Replicas.Idle++ r.NoError(createOrUpdateScaledObject( testInfra.ctx, testInfra.cl, @@ -107,6 +104,10 @@ func TestCreateOrUpdateScaledObject(t *testing.T) { *testInfra.httpso.Spec.Replicas.Max, *spec.MaxReplicaCount, ) + r.Equal( + int64(testInfra.httpso.Spec.Replicas.Idle), + spec["idleReplicaCount"], + ) } func getSO( diff --git a/operator/controllers/suite_test.go b/operator/controllers/suite_test.go index f9af2c58..eb3225ad 100644 --- a/operator/controllers/suite_test.go +++ b/operator/controllers/suite_test.go @@ -31,6 +31,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "github.com/kedacore/http-add-on/operator/api/v1alpha1" httpv1alpha1 "github.com/kedacore/http-add-on/operator/api/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -85,6 +86,11 @@ func newCommonTestInfra(namespace, appName string) *commonTestInfra { Service: appName, Port: 8081, }, + Replicas: v1alpha1.ReplicaStruct{ + Min: 0, + Max: 20, + Idle: 0, + }, }, } diff --git a/pkg/k8s/scaledobject.go b/pkg/k8s/scaledobject.go index ddc20b0f..0725a8ac 100644 --- a/pkg/k8s/scaledobject.go +++ b/pkg/k8s/scaledobject.go @@ -1,10 +1,11 @@ package k8s import ( - kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "bytes" + "text/template" + + "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) const ( @@ -22,42 +23,51 @@ func NewScaledObject( deploymentName string, scalerAddress string, host string, - minReplicas *int32, - maxReplicas *int32, - cooldownPeriod *int32, -) *kedav1alpha1.ScaledObject { - return &kedav1alpha1.ScaledObject{ - TypeMeta: metav1.TypeMeta{ - APIVersion: kedav1alpha1.SchemeGroupVersion.Identifier(), - Kind: ObjectKind(&kedav1alpha1.ScaledObject{}), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - Labels: Labels(name), - }, - Spec: kedav1alpha1.ScaledObjectSpec{ - ScaleTargetRef: &kedav1alpha1.ScaleTarget{ - APIVersion: appsv1.SchemeGroupVersion.Identifier(), - Kind: ObjectKind(&appsv1.Deployment{}), - Name: deploymentName, - }, - PollingInterval: pointer.Int32(soPollingInterval), - CooldownPeriod: cooldownPeriod, - MinReplicaCount: minReplicas, - MaxReplicaCount: maxReplicas, - Advanced: &kedav1alpha1.AdvancedConfig{ - RestoreToOriginalReplicaCount: true, - }, - Triggers: []kedav1alpha1.ScaleTriggers{ - { - Type: soTriggerType, - Metadata: map[string]string{ - mkScalerAddress: scalerAddress, - mkHost: host, - }, - }, - }, - }, + minReplicas, + maxReplicas int32, + idleReplicas int32, + cooldownPeriod int32, +) (*unstructured.Unstructured, error) { + // https://keda.sh/docs/1.5/faq/ + // https://github.com/kedacore/keda/blob/aa0ea79450a1c7549133aab46f5b916efa2364ab/api/v1alpha1/scaledobject_types.go + // + // unstructured.Unstructured only supports specific types in it. see here for the list: + // https://github.com/kubernetes/apimachinery/blob/v0.17.12/pkg/runtime/converter.go#L449-L476 + typedLabels := Labels(name) + labels := map[string]interface{}{} + for k, v := range typedLabels { + var vIface interface{} = v + labels[k] = vIface + } + + tpl, err := template.ParseFS(scaledObjectTemplateFS, "templates/scaledobject.yaml") + if err != nil { + return nil, err + } + + var scaledObjectTemplateBuffer bytes.Buffer + if tplErr := tpl.Execute(&scaledObjectTemplateBuffer, map[string]interface{}{ + "Name": name, + "Namespace": namespace, + "Labels": labels, + "MinReplicas": minReplicas, + "MaxReplicas": maxReplicas, + "IdleReplicas": idleReplicas, + "DeploymentName": deploymentName, + "ScalerAddress": scalerAddress, + "Host": host, + "CooldownPeriod": cooldownPeriod, + }); tplErr != nil { + return nil, tplErr + } + + var decodedYaml map[string]interface{} + decodeErr := yaml.Unmarshal(scaledObjectTemplateBuffer.Bytes(), &decodedYaml) + if decodeErr != nil { + return nil, decodeErr } + + return &unstructured.Unstructured{ + Object: decodedYaml, + }, nil } diff --git a/pkg/k8s/templates/scaledobject.yaml b/pkg/k8s/templates/scaledobject.yaml new file mode 100644 index 00000000..080d69d9 --- /dev/null +++ b/pkg/k8s/templates/scaledobject.yaml @@ -0,0 +1,25 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ .Name }} + namespace: {{ .Namespace }} + labels: + {{- range $key, $val := .Labels }} + {{ $key }}: {{ $val }} + {{- end }} +spec: + minReplicaCount: {{ .MinReplicas }} + maxReplicaCount: {{ .MaxReplicas }} + idleReplicaCount: {{ .IdleReplicas }} + cooldownPeriod: {{ .CooldownPeriod }} + pollingInterval: 1 + scaleTargetRef: + name: {{ .DeploymentName }} + kind: Deployment + triggers: + - type: external-push + metadata: + scalerAddress: {{ .ScalerAddress }} + host: {{ .Host }} + advanced: + restoreToOriginalReplicaCount: true