Feat/descheduler

images/descheduler/werf.inc.yaml

@@ -0,0 +1,43 @@
+{{- $version := "1.31" }}
+---
+image: {{ $.ImageName }}-builder
+final: false
+fromImage: BASE_GOLANG_23_BOOKWORM
+mount:
+- fromPath: ~/go-pkg-cache
+  to: /go/pkg
+shell:
+  install:
+    - apt-get -qq update
+    - apt-get -qq install -y --no-install-recommends git
+    - apt-get clean
+    - rm --recursive --force /var/lib/apt/lists/* /var/cache/apt/*
+    - git clone --depth 1 --branch release-{{ $version }} https://github.com/kubernetes-sigs/descheduler.git /src
+    - cd /src
+    - git checkout release-{{ $version }}
+    - rm -rf .git
+  setup:
+    - cd /src
+    - |
+      export GO111MODULE=on
+      export GOOS=linux
+      export CGO_ENABLED=0
+      export GOARCH=amd64
+    - go mod download -x
+    - go mod vendor
+    - go build -ldflags "-s -w -X sigs.k8s.io/descheduler/pkg/version.version={{ $version }}" -o /descheduler sigs.k8s.io/descheduler/cmd/descheduler
+    - chown 64535:64535 /descheduler
+    - chmod 0700 /descheduler
+---
+image: {{ $.ImageName }}
+fromImage: distroless
+import:
+  - image: {{ $.ImageName }}-builder
+    add: /descheduler
+    to: /descheduler
+    before: setup
+imageSpec:
+  config:
+    user: 64535
+    entrypoint: ["/descheduler"]
+---

openapi/config-values.yaml

@@ -248,3 +248,13 @@ properties:
     enum:
       - "text"
       - "json"
+  descheduler:
+    type: object
+    description: |
+      Configuration for the descheduler. Enables eviction of virtual machines.
+    properties:
+      enabled:
+        type: boolean
+        description: |
+          Enable or disable the descheduler. Set to true to activate VM eviction.
+        x-examples: [true, false]

openapi/doc-ru-config-values.yaml

@@ -149,3 +149,12 @@ properties:
 
       Работает для следующих компонентов:
       - `virtualization-controller`
+  descheduler:
+    type: object
+    description: |
+      Конфигурация для descheduler. Включает выселение виртуальных машин.
+    properties:
+      enabled:
+        type: boolean
+        description: |
+          Включение или отключение descheduler. Установите значение true для активации выселения виртуальных машин.

templates/descheduler/configmap.yaml

@@ -0,0 +1,44 @@
+{{- if .Values.virtualization.descheduler.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: descheduler-policy
+  namespace: d8-{{ .Chart.Name }}
+    {{- include "helm_lib_module_labels" (list $) | nindent 2 }}
+data:
+  policy.yaml: |
+    apiVersion: "descheduler/v1alpha2"
+    kind: "DeschedulerPolicy"
+    profiles:
+      - name: virtualization
+        pluginConfig:
+          - name: "DefaultEvictor"
+            args:
+              evictLocalStoragePods: true
+              evictSystemCriticalPods: false
+              ignorePvcPods: false
+              evictFailedBarePods: true
+              nodeFit: true
+              labelSelector:
+                matchExpressions:
+                  - key: "vm.kubevirt.internal.virtualization.deckhouse.io/name"
+                    operator: Exists
+          - name: "RemovePodsViolatingNodeAffinity"
+            args:
+              nodeAffinityType:
+                - requiredDuringSchedulingIgnoredDuringExecution
+          - name: "RemovePodsViolatingInterPodAntiAffinity"
+        plugins:
+          filter:
+            enabled:
+              - "DefaultEvictor"
+          preEvictionFilter:
+            enabled:
+              - "DefaultEvictor"
+          deschedule:
+            enabled:
+              - "RemovePodsViolatingNodeAffinity"
+              - "RemovePodsViolatingInterPodAntiAffinity"
+
+{{- end }}

templates/descheduler/deployment.yaml

@@ -0,0 +1,106 @@
+{{- define "descheduler_resources" }}
+cpu: 25m
+memory: 50Mi
+{{- end }}
+
+{{- if .Values.virtualization.descheduler.enabled }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: descheduler
+  namespace: d8-{{ .Chart.Name }}
+  {{- include "helm_lib_module_labels" (list $ (dict "app" "descheduler")) | nindent 2 }}
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: descheduler
+  template:
+    metadata:
+      labels:
+        app: descheduler
+    spec:
+      serviceAccountName: descheduler
+      imagePullSecrets:
+        - name: deckhouse-registry
+      {{- include "helm_lib_node_selector" (tuple $ "system") | nindent 6 }}
+      {{- include "helm_lib_tolerations" (tuple $ "system") | nindent 6 }}
+      {{- include "helm_lib_priority_class" (tuple $ "cluster-low") | nindent 6 }}
+      {{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . | nindent 6 }}
+      containers:
+      - name: descheduler
+        {{- include "helm_lib_module_container_security_context_read_only_root_filesystem" $ | nindent 8 }}
+        image: {{ include "helm_lib_module_image" (list $ "descheduler") }}
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /healthz
+            port: 10258
+            scheme: HTTPS
+          initialDelaySeconds: 3
+          periodSeconds: 10
+        volumeMounts:
+          - mountPath: /policy
+            name: policy-volume
+        args:
+          - "--bind-address"
+          - "127.0.0.1"
+          - "--policy-config-file"
+          - "/policy/policy.yaml"
+          - "--logging-format"
+          - "json"
+          - "--v"
+          - "6"
+          - "--descheduling-interval"
+          - "15m"
+        resources:
+          requests:
+          {{- include "helm_lib_module_ephemeral_storage_only_logs" $ | nindent 12 }}
+          {{- if not ($.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
+            {{- include "descheduler_resources" $ | nindent 12 }}
+          {{- end }}
+      - name: kube-rbac-proxy
+        {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . | nindent 8 }}
+        image: {{ include "helm_lib_module_common_image" (list . "kubeRbacProxy") }}
+        args:
+          - "--secure-listen-address=$(KUBE_RBAC_PROXY_LISTEN_ADDRESS):10258"
+          - "--v=2"
+          - "--logtostderr=true"
+          - "--stale-cache-interval=1h30m"
+        env:
+          - name: KUBE_RBAC_PROXY_LISTEN_ADDRESS
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: KUBE_RBAC_PROXY_CONFIG
+            value: |
+              excludePaths:
+              - /healthz
+              upstreams:
+              - upstream: https://127.0.0.1:10258/
+                upstreamInsecureSkipVerify: true
+                path: /
+                authorization:
+                  resourceAttributes:
+                    namespace: d8-{{ .Chart.Name }}
+                    apiGroup: apps
+                    apiVersion: v1
+                    resource: deployments
+                    subresource: prometheus-metrics
+                    name: descheduler
+        ports:
+          - containerPort: 10258
+            name: https-metrics
+        resources:
+          requests:
+          {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }}
+          {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
+            {{- include "helm_lib_container_kube_rbac_proxy_resources" . | nindent 12 }}
+          {{- end }}
+      volumes:
+        - name: policy-volume
+          configMap:
+            name: descheduler-policy
+{{- end }}

templates/descheduler/rbac-for-us.yaml

@@ -0,0 +1,72 @@
+{{- if .Values.virtualization.descheduler.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: descheduler
+  namespace: d8-{{ .Chart.Name }}
+  {{- include "helm_lib_module_labels" (list . (dict "app" "descheduler")) | nindent 2 }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: d8:virtualization:descheduler:descheduler
+  {{- include "helm_lib_module_labels" (list . (dict "app" "descheduler")) | nindent 2 }}
+rules:
+- apiGroups: ["events.k8s.io"]
+  resources: ["events"]
+  verbs: ["create", "update"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list", "delete"]
+- apiGroups: [""]
+  resources: ["pods/eviction"]
+  verbs: ["create"]
+- apiGroups: ["scheduling.k8s.io"]
+  resources: ["priorityclasses"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create", "update"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  resourceNames: ["descheduler"]
+  verbs: ["get", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: d8:virtualization:descheduler:descheduler
+  {{- include "helm_lib_module_labels" (list . (dict "app" "descheduler")) | nindent 2 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: d8:virtualization:descheduler:descheduler
+subjects:
+  - name: descheduler
+    kind: ServiceAccount
+    namespace: d8-{{ .Chart.Name }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: d8:virtualization:descheduler:descheduler:rbac-proxy
+  {{- include "helm_lib_module_labels" (list . (dict "app" "descheduler")) | nindent 2 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: d8:rbac-proxy
+subjects:
+  - kind: ServiceAccount
+    name: descheduler
+    namespace: d8-{{ .Chart.Name }}
+{{- end }}

templates/descheduler/rbac-to-us.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.virtualization.descheduler.enabled }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: access-to-descheduler
+  namespace: d8-{{ $.Chart.Name }}
+  {{- include "helm_lib_module_labels" (list . (dict "app" "descheduler")) | nindent 2 }}
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments/prometheus-metrics"]
+  resourceNames: ["descheduler"]
+  verbs: ["get"]
+{{- if (.Values.global.enabledModules | has "prometheus") }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: access-to-descheduler
+  namespace: d8-{{ $.Chart.Name }}
+  {{- include "helm_lib_module_labels" (list . (dict "app" "descheduler")) | nindent 2 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: access-to-descheduler
+subjects:
+- kind: User
+  name: d8-monitoring:scraper
+- kind: ServiceAccount
+  name: prometheus
+  namespace: d8-monitoring
+{{- end }}
+
+{{- end }}

tools/kubeconform/fixtures/module-values.yaml

@@ -326,6 +326,7 @@ global:
         virtualizationApi: sha256:0000000000000000000000000000000000000000000000000000000000000000
         virtualizationController: sha256:0000000000000000000000000000000000000000000000000000000000000000
         vmRouteForge: sha256:0000000000000000000000000000000000000000000000000000000000000000
+        descheduler: sha256:0000000000000000000000000000000000000000000000000000000000000000
     registry:
       CA: ""
       address: some-registry.io
@@ -335,6 +336,8 @@ global:
       scheme: https
     tags: {}
 virtualization:
+  descheduler:
+    enabled: true
   dvcr:
     storage:
       persistentVolumeClaim: