From d9e2248172239a9ecaf0f1a518bcd0cf70a11c3a Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Tue, 7 Nov 2023 23:50:43 -0600 Subject: [PATCH] Add configurable support of armada-operator for armada-api Signed-off-by: Ruslan Aliev Change-Id: I76fb41062d152bf360a85d781c19ab5b204769b8 --- .zuul.yaml | 6 +- armada/cli/apply.py | 9 +- armada/conf/default.py | 12 + armada/handlers/armada.py | 48 +++- armada/handlers/wait.py | 30 ++ .../armada.airshipit.org_armadacharts.yaml | 257 ++++++++++++++++++ charts/armada/templates/deployment-api.yaml | 39 +++ charts/armada/values.yaml | 3 +- images/armada/Dockerfile.ubuntu_focal | 3 + 9 files changed, 401 insertions(+), 6 deletions(-) create mode 100644 charts/armada/crds/armada.airshipit.org_armadacharts.yaml diff --git a/.zuul.yaml b/.zuul.yaml index c9a79546..dd41ef42 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -105,9 +105,9 @@ CLONE_ARMADA: false ARMADA_IMAGE_DISTRO: ubuntu_focal HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz - HTK_COMMIT: ae91cf3fc3f288b6d92ace4a3a405606a653638f - OSH_INFRA_COMMIT: db3537e56b182a54e7f6931ce57e2a190714019b - OSH_COMMIT: 75c30f43db44218e7842611e880fd8d7a30fa79c + HTK_COMMIT: 1c83e3a9aef8c0b40360a88125f8b11b540dd3a7 + OSH_INFRA_COMMIT: 1c83e3a9aef8c0b40360a88125f8b11b540dd3a7 + OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 irrelevant-files: - ^.*\.rst$ - ^doc/.*$ diff --git a/armada/cli/apply.py b/armada/cli/apply.py index 4fe82f78..c2f7dfe3 100644 --- a/armada/cli/apply.py +++ b/armada/cli/apply.py @@ -128,13 +128,20 @@ SHORT_DESC = "Command installs manifest charts." "which manifest to run when multiple are available."), default=None) @click.option('--bearer-token', help="User Bearer token", default=None) +@click.option( + '--enable-operator', help="Use operator for applying charts", is_flag=True) +@click.option( + '--go-wait', help="Use operator for applying charts", is_flag=True) @click.option('--debug', help="Enable debug logging.", is_flag=True) @click.pass_context def apply_create( ctx, locations, api, disable_update_post, disable_update_pre, enable_chart_cleanup, metrics_output, use_doc_ref, set, timeout, - values, wait, target_manifest, bearer_token, debug): + values, wait, target_manifest, bearer_token, enable_operator, go_wait, + debug): CONF.debug = debug + CONF.enable_operator = enable_operator + CONF.go_wait = go_wait ApplyManifest( ctx, locations, api, disable_update_post, disable_update_pre, enable_chart_cleanup, metrics_output, use_doc_ref, set, timeout, diff --git a/armada/conf/default.py b/armada/conf/default.py index 213a6040..3dd2c2c3 100644 --- a/armada/conf/default.py +++ b/armada/conf/default.py @@ -94,6 +94,18 @@ path to the private key that includes the name of the key itself.""")), """Time in seconds of how much time needs to pass since the last update of an existing lock before armada forcibly removes it and tries to acquire its own lock""")), + cfg.BoolOpt( + 'enable_operator', + default=False, + help=utils.fmt( + """Determines whether the operator has to be enabled + to apply charts instead of armada-api itself""")), + cfg.BoolOpt( + 'go_wait', + default=False, + help=utils.fmt( + """Determines whether the wait process has to be done + via armada-go using client-go library""")), ] diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index 4dfa6c5e..e2e19345 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import subprocess # nosec +import tempfile from concurrent.futures import ThreadPoolExecutor, as_completed from oslo_config import cfg from oslo_log import log as logging +import yaml from armada import const from armada.conf import set_current_chart @@ -74,6 +77,7 @@ class Armada(object): ''' self.enable_chart_cleanup = enable_chart_cleanup + self.enable_operator = CONF.enable_operator self.force_wait = force_wait self.helm = helm try: @@ -83,6 +87,7 @@ class Armada(object): except (validate_exceptions.InvalidManifestException, override_exceptions.InvalidOverrideValueException): raise + self.target_manifest = target_manifest self.manifest = Manifest( self.documents, target_manifest=target_manifest).get_manifest() self.chart_download = ChartDownload() @@ -109,7 +114,48 @@ class Armada(object): ''' manifest_name = self.manifest['metadata']['name'] with metrics.APPLY.get_context(manifest_name): - return self._sync() + if self.enable_operator: + return self._sync_with_operator() + else: + return self._sync() + + def _sync_with_operator(self): + # TODO: add actual msg + msg = { + 'install': [], + 'upgrade': [], + 'diff': [], + 'purge': [], + 'protected': [] + } + + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + yaml.safe_dump_all(self.documents, tfile) + tfile.flush() + tfile.close() + + command = ['armada-go', 'apply'] + if self.target_manifest != "": + command.extend( + ['--target-manifest', "{}".format(self.target_manifest)]) + command.append("{}".format(tfile.name)) + LOG.info('Running command=%s', command) + try: + with subprocess.Popen( # nosec + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + bufsize=1, universal_newlines=True) as sp: + for line in sp.stdout: + LOG.info(line.rstrip()) + sp.wait() + if sp.returncode != 0: + raise subprocess.CalledProcessError( + sp.returncode, command, output=sp.stdout) + except subprocess.CalledProcessError as e: + LOG.info("armada-go apply exception: %s", e) + raise armada_exceptions.ArmadaException(e) + + LOG.info('Done applying manifest.') + return msg def _sync(self): msg = { diff --git a/armada/handlers/wait.py b/armada/handlers/wait.py index 5fcfa14f..bc6572fa 100644 --- a/armada/handlers/wait.py +++ b/armada/handlers/wait.py @@ -17,9 +17,11 @@ import collections import copy import math import re +import subprocess # nosec import time from kubernetes import watch +from oslo_config import cfg from oslo_log import log as logging from retry import retry import urllib3.exceptions @@ -33,6 +35,7 @@ from armada.utils.helm import is_test_pod from armada.utils.release import label_selectors LOG = logging.getLogger(__name__) +CONF = cfg.CONF ROLLING_UPDATE_STRATEGY_TYPE = 'RollingUpdate' ASYNC_UPDATE_NOT_ALLOWED_MSG = 'Async update not allowed: ' @@ -381,6 +384,33 @@ class ResourceWait(ABC): modified = set() found_resources = False + if CONF.go_wait: + command = [ + 'armada-go', 'wait', '--resource-type', + "{}s".format(self.resource_type), '--namespace', + self.chart_wait.release_id.namespace, '--label-selector', + self.label_selector, '--timeout', "{}s".format(timeout) + ] + if hasattr(self, "min_ready"): + _, _, m_ready = self.min_ready + command.extend(['--min-ready', "{}".format(m_ready)]) + LOG.info('Running command=%s', command) + try: + with subprocess.Popen( # nosec + command, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, bufsize=1, + universal_newlines=True) as sp: + for line in sp.stdout: + LOG.info(line.rstrip()) + sp.wait() + if sp.returncode != 0: + raise subprocess.CalledProcessError( + sp.returncode, command, output=sp.stdout) + return False, [], [], False + except subprocess.CalledProcessError as e: + LOG.info("armada-go wait exception: %s", e) + raise armada_exceptions.WaitException(e) + kwargs = { 'namespace': self.chart_wait.release_id.namespace, 'label_selector': self.label_selector, diff --git a/charts/armada/crds/armada.airshipit.org_armadacharts.yaml b/charts/armada/crds/armada.airshipit.org_armadacharts.yaml new file mode 100644 index 00000000..f2f9181c --- /dev/null +++ b/charts/armada/crds/armada.airshipit.org_armadacharts.yaml @@ -0,0 +1,257 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: armadacharts.armada.airshipit.org +spec: + group: armada.airshipit.org + names: + kind: ArmadaChart + listKind: ArmadaChartList + plural: armadacharts + singular: armadachart + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.helmStatus + name: Helm Status + type: string + - jsonPath: .status.waitCompleted + name: Wait Done + type: boolean + - jsonPath: .status.tested + name: Tested + type: boolean + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Message + priority: 10 + type: string + name: v1 + schema: + openAPIV3Schema: + description: ArmadaChart is the Schema for the armadacharts API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + data: + description: ArmadaChartSpec defines the specification of ArmadaChart + properties: + chart_name: + description: ChartName is name of ArmadaChart + type: string + namespace: + description: Namespace is a namespace for ArmadaChart + type: string + release: + description: Release is a name of corresponding Helm Release of ArmadaChart + type: string + source: + description: Source is a source location of Helm Chart *.tgz + properties: + location: + type: string + subpath: + type: string + type: + type: string + type: object + test: + description: Test holds the test parameters for this Helm release. + properties: + enabled: + description: Enabled is an example field of ArmadaChart. Edit + armadachart_types.go to remove/update + type: boolean + type: object + upgrade: + description: Upgrade holds the upgrade options for this Helm release. + properties: + pre: + properties: + delete: + items: + description: ArmadaChartDeleteResource defines the delete + options of ArmadaChart + properties: + labels: + additionalProperties: + type: string + type: object + type: + type: string + type: object + type: array + type: object + type: object + values: + description: Values holds the values for this Helm release. + x-kubernetes-preserve-unknown-fields: true + wait: + description: Wait holds the wait options for this Helm release. + properties: + labels: + additionalProperties: + type: string + type: object + native: + description: ArmadaChartWaitNative defines the wait options of + ArmadaChart + properties: + enabled: + type: boolean + type: object + resources: + items: + description: ArmadaChartWaitResource defines the wait options + of ArmadaChart + properties: + labels: + additionalProperties: + type: string + type: object + min_ready: + type: string + type: + type: string + type: object + type: array + timeout: + description: Timeout is the time to wait for full reconciliation + of Helm release. + type: integer + type: object + type: object + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: ArmadaChartStatus defines the observed state of ArmadaChart + properties: + conditions: + description: Conditions holds the conditions for the ArmadaChart. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + failures: + description: Failures is the reconciliation failure count against + the latest desired state. It is reset after a successful reconciliation. + format: int64 + type: integer + helmStatus: + description: HelmStatus describes the status of helm release + type: string + installFailures: + description: InstallFailures is the install failure count against + the latest desired state. It is reset after a successful reconciliation. + format: int64 + type: integer + lastAttemptedChartSource: + description: LastAppliedChartSource is the URL of chart of the last + reconciliation attempt + type: string + lastAttemptedValuesChecksum: + description: LastAppliedValuesChecksum is the SHA1 checksum of the + values of the last reconciliation attempt. + type: string + observedGeneration: + description: ObservedGeneration is the last observed generation. + format: int64 + type: integer + tested: + description: Tested is the bool value whether the Helm Release was + successfully tested or not. + type: boolean + upgradeFailures: + description: UpgradeFailures is the upgrade failure count against + the latest desired state. It is reset after a successful reconciliation. + format: int64 + type: integer + waitCompleted: + description: WaitCompleted is the bool value whether the Helm Release + resources were waited for or not. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/armada/templates/deployment-api.yaml b/charts/armada/templates/deployment-api.yaml index 5e930311..030d8130 100644 --- a/charts/armada/templates/deployment-api.yaml +++ b/charts/armada/templates/deployment-api.yaml @@ -109,6 +109,45 @@ spec: {{ tuple $envAll "api" $mounts_armada_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{ dict "envAll" $envAll "application" "armada" "container" "armada_api_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} containers: +{{- if .Values.conf.armada.DEFAULT.enable_operator }} + - name: manager +{{ tuple $envAll "operator" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /manager + args: + - '--health-probe-bind-address=:8081' + - '--metrics-bind-address=127.0.0.1:8080' + - '--leader-elect' + livenessProbe: + httpGet: + path: /healthz + port: 8081 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 1 + periodSeconds: 20 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false +{{- end }} - name: armada-api {{ tuple $envAll "api" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} diff --git a/charts/armada/values.yaml b/charts/armada/values.yaml index d99d60e5..6d8d7ffd 100644 --- a/charts/armada/values.yaml +++ b/charts/armada/values.yaml @@ -28,7 +28,8 @@ labels: images: tags: - api: 'quay.io/airshipit/armada:latest' + api: 'quay.io/airshipit/armada:latest-ubuntu_focal' + operator: 'quay.io/airshipit/armada-operator:latest-ubuntu_focal' dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' ks_endpoints: 'docker.io/openstackhelm/heat:newton' ks_service: 'docker.io/openstackhelm/heat:newton' diff --git a/images/armada/Dockerfile.ubuntu_focal b/images/armada/Dockerfile.ubuntu_focal index ed989f53..066080d8 100644 --- a/images/armada/Dockerfile.ubuntu_focal +++ b/images/armada/Dockerfile.ubuntu_focal @@ -1,4 +1,6 @@ ARG FROM=ubuntu:20.04 +ARG ARMADA_GO=quay.io/airshipit/armada-go:8b6a87059acf6600a84b5f1314f2a69f434032b0-ubuntu_focal +FROM ${ARMADA_GO} AS armada_go FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ @@ -88,6 +90,7 @@ RUN set -ex \ /usr/share/doc-base COPY . ./ +COPY --from=armada_go /usr/local/bin/armada /usr/local/bin/armada-go # Setting the version explicitly for PBR ENV PBR_VERSION 0.8.0