From b1c4e75c3513507d0664853320d00fdc82b7b001 Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Tue, 7 Nov 2023 23:50:43 -0600 Subject: [PATCH] Add configurable support of armada-operator for armada-api Signed-off-by: Ruslan Aliev Change-Id: I76fb41062d152bf360a85d781c19ab5b204769b8 --- Makefile | 4 +- armada/cli/apply.py | 5 +- armada/conf/default.py | 6 + armada/handlers/armada.py | 50 +++- armada/handlers/wait.py | 2 +- charts/armada/templates/deployment-api.yaml | 39 +++ charts/armada/values.yaml | 1 + crd.yaml | 269 ++++++++++++++++++++ images/armada/Dockerfile.ubuntu_focal | 2 + 9 files changed, 373 insertions(+), 5 deletions(-) create mode 100644 crd.yaml diff --git a/Makefile b/Makefile index 2a137d27..876c83ec 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ _BASE_IMAGE_ARG := $(if $(UBUNTU_BASE_IMAGE),--build-arg FROM="${UBUNTU_BASE_IMA build_armada: ifeq ($(USE_PROXY), true) - docker build --network host -t $(IMAGE) --label $(LABEL) \ + docker build --no-cache --network host -t $(IMAGE) --label $(LABEL) \ --label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ @@ -108,7 +108,7 @@ ifeq ($(USE_PROXY), true) --build-arg no_proxy=$(NO_PROXY) \ --build-arg NO_PROXY=$(NO_PROXY) . else - docker build --network host -t $(IMAGE) --label $(LABEL) \ + docker build --no-cache --network host -t $(IMAGE) --label $(LABEL) \ --label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ diff --git a/armada/cli/apply.py b/armada/cli/apply.py index 4fe82f78..1d6dba51 100644 --- a/armada/cli/apply.py +++ b/armada/cli/apply.py @@ -128,13 +128,16 @@ SHORT_DESC = "Command installs manifest charts." "which manifest to run when multiple are available."), default=None) @click.option('--bearer-token', help="User Bearer token", default=None) +@click.option('--enable-operator', help="Use operator for applying charts", + is_flag=True) @click.option('--debug', help="Enable debug logging.", is_flag=True) @click.pass_context def apply_create( ctx, locations, api, disable_update_post, disable_update_pre, enable_chart_cleanup, metrics_output, use_doc_ref, set, timeout, - values, wait, target_manifest, bearer_token, debug): + values, wait, target_manifest, bearer_token, enable_operator, debug): CONF.debug = debug + CONF.enable_operator = enable_operator ApplyManifest( ctx, locations, api, disable_update_post, disable_update_pre, enable_chart_cleanup, metrics_output, use_doc_ref, set, timeout, diff --git a/armada/conf/default.py b/armada/conf/default.py index 213a6040..ac1ae814 100644 --- a/armada/conf/default.py +++ b/armada/conf/default.py @@ -94,6 +94,12 @@ path to the private key that includes the name of the key itself.""")), """Time in seconds of how much time needs to pass since the last update of an existing lock before armada forcibly removes it and tries to acquire its own lock""")), + cfg.BoolOpt( + 'enable_operator', + default=False, + help=utils.fmt( + """Determines whether the operator has to be enabled + to apply charts instead of armada-api itself""")), ] diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index 4dfa6c5e..9505bdfb 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import subprocess # nosec +import tempfile from concurrent.futures import ThreadPoolExecutor, as_completed from oslo_config import cfg from oslo_log import log as logging +import yaml from armada import const from armada.conf import set_current_chart @@ -74,6 +77,7 @@ class Armada(object): ''' self.enable_chart_cleanup = enable_chart_cleanup + self.enable_operator = CONF.enable_operator self.force_wait = force_wait self.helm = helm try: @@ -83,6 +87,7 @@ class Armada(object): except (validate_exceptions.InvalidManifestException, override_exceptions.InvalidOverrideValueException): raise + self.target_manifest = target_manifest self.manifest = Manifest( self.documents, target_manifest=target_manifest).get_manifest() self.chart_download = ChartDownload() @@ -109,7 +114,50 @@ class Armada(object): ''' manifest_name = self.manifest['metadata']['name'] with metrics.APPLY.get_context(manifest_name): - return self._sync() + if self.enable_operator: + return self._sync_with_operator() + else: + return self._sync() + + def _sync_with_operator(self): + # TODO: add actual msg + msg = { + 'install': [], + 'upgrade': [], + 'diff': [], + 'purge': [], + 'protected': [] + } + + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + yaml.safe_dump_all(self.documents, tfile) + tfile.flush() + tfile.close() + + command = ['armada-go', 'apply'] + if self.target_manifest != "": + command.extend( + ['--target-manifest', "{}".format(self.target_manifest)]) + command.append("{}".format(tfile.name)) + LOG.info('Running command=%s', command) + try: + with subprocess.Popen( # nosec + command, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, bufsize=1, + universal_newlines=True) as sp: + for line in sp.stdout: + LOG.info(line.rstrip()) + sp.wait() + if sp.returncode != 0: + raise subprocess.CalledProcessError( + sp.returncode, command, output=sp.stdout) + except subprocess.CalledProcessError as e: + LOG.info( + "Exception running cmd occurred: %s", e.output) + raise armada_exceptions.ArmadaException(e) + + LOG.info('Done applying manifest.') + return msg def _sync(self): msg = { diff --git a/armada/handlers/wait.py b/armada/handlers/wait.py index 5fcfa14f..0013085a 100644 --- a/armada/handlers/wait.py +++ b/armada/handlers/wait.py @@ -87,7 +87,7 @@ class ChartWait(): labels.update(resource_labels) resource_config['labels'] = labels - LOG.debug('Resolved `wait.resources` list: %s', resources_list) + LOG.info('Resolved `wait.resources` list: %s', resources_list) self.waits = [self.get_resource_wait(conf) for conf in resources_list] diff --git a/charts/armada/templates/deployment-api.yaml b/charts/armada/templates/deployment-api.yaml index 5e930311..030d8130 100644 --- a/charts/armada/templates/deployment-api.yaml +++ b/charts/armada/templates/deployment-api.yaml @@ -109,6 +109,45 @@ spec: {{ tuple $envAll "api" $mounts_armada_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{ dict "envAll" $envAll "application" "armada" "container" "armada_api_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} containers: +{{- if .Values.conf.armada.DEFAULT.enable_operator }} + - name: manager +{{ tuple $envAll "operator" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /manager + args: + - '--health-probe-bind-address=:8081' + - '--metrics-bind-address=127.0.0.1:8080' + - '--leader-elect' + livenessProbe: + httpGet: + path: /healthz + port: 8081 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 1 + periodSeconds: 20 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false +{{- end }} - name: armada-api {{ tuple $envAll "api" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} diff --git a/charts/armada/values.yaml b/charts/armada/values.yaml index d99d60e5..5f137e52 100644 --- a/charts/armada/values.yaml +++ b/charts/armada/values.yaml @@ -29,6 +29,7 @@ labels: images: tags: api: 'quay.io/airshipit/armada:latest' + operator: 'quay.io/airshipit/armada-operator:latest' dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' ks_endpoints: 'docker.io/openstackhelm/heat:newton' ks_service: 'docker.io/openstackhelm/heat:newton' diff --git a/crd.yaml b/crd.yaml new file mode 100644 index 00000000..3c214b86 --- /dev/null +++ b/crd.yaml @@ -0,0 +1,269 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: armadacharts.armada.airshipit.io +spec: + group: armada.airshipit.io + names: + kind: ArmadaChart + listKind: ArmadaChartList + plural: armadacharts + singular: armadachart + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ArmadaChart is the Schema for the armadacharts API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + data: + description: ArmadaChartSpec defines the desired state of ArmadaChart + properties: + chart_name: + description: ChartName is an example field of ArmadaChart. Edit armadachart_types.go + to remove/update + type: string + namespace: + description: Namespace is an example field of ArmadaChart. Edit armadachart_types.go + to remove/update + type: string + release: + description: Release is an example field of ArmadaChart. Edit armadachart_types.go + to remove/update + type: string + source: + description: Source is an example field of ArmadaChart. Edit armadachart_types.go + to remove/update + properties: + location: + description: Location is an example field of ArmadaChart. Edit + armadachart_types.go to remove/update + type: string + subpath: + description: Subpath is an example field of ArmadaChart. Edit + armadachart_types.go to remove/update + type: string + type: + description: Type is an example field of ArmadaChart. Edit armadachart_types.go + to remove/update + type: string + type: object + test: + description: Test holds the values for this Helm release. + properties: + enabled: + description: Enabled is an example field of ArmadaChart. Edit + armadachart_types.go to remove/update + type: boolean + type: object + upgrade: + description: Delete holds the values for this Helm release. + properties: + pre: + properties: + delete: + items: + description: ArmadaChartDeleteResource defines the wait + options of ArmadaChart + properties: + labels: + additionalProperties: + type: string + description: Labels is an example field of ArmadaChart. + Edit armadachart_types.go to remove/update + type: object + type: + description: Type is an example field of ArmadaChart. + Edit armadachart_types.go to remove/update + type: string + type: object + type: array + type: object + type: object + values: + description: Values holds the values for this Helm release. + x-kubernetes-preserve-unknown-fields: true + wait: + description: Wait holds the values for this Helm release. + properties: + labels: + additionalProperties: + type: string + description: Labels is an example field of ArmadaChart. Edit armadachart_types.go + to remove/update + type: object + native: + description: ArmadaChartWaitNative defines the wait options of + ArmadaChart + properties: + enabled: + description: Enabled is an example field of ArmadaChart. Edit + armadachart_types.go to remove/update + type: boolean + type: object + resources: + items: + description: ArmadaChartWaitResource defines the wait options + of ArmadaChart + properties: + labels: + additionalProperties: + type: string + description: Labels is an example field of ArmadaChart. + Edit armadachart_types.go to remove/update + type: object + min_ready: + type: string + type: + description: Type is an example field of ArmadaChart. Edit + armadachart_types.go to remove/update + type: string + type: object + type: array + timeout: + description: Timeout is the time to wait for full reconciliation + of Helm release. + type: integer + type: object + type: object + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: ArmadaChartStatus defines the observed state of ArmadaChart + properties: + conditions: + description: Conditions holds the conditions for the ArmadaChart. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + failures: + description: Failures is the reconciliation failure count against + the latest desired state. It is reset after a successful reconciliation. + format: int64 + type: integer + helmChart: + description: HelmChart is the namespaced name of the HelmChart resource + created by the controller for the ArmadaChart. + type: string + installFailures: + description: InstallFailures is the install failure count against + the latest desired state. It is reset after a successful reconciliation. + format: int64 + type: integer + lastAppliedRevision: + description: LastAppliedRevision is the revision of the last successfully + applied source. + type: string + lastAttemptedRevision: + description: LastAttemptedRevision is the revision of the last reconciliation + attempt. + type: string + lastAttemptedValuesChecksum: + description: LastAttemptedValuesChecksum is the SHA1 checksum of the + values of the last reconciliation attempt. + type: string + lastHandledReconcileAt: + description: LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value can + be detected. + type: string + lastReleaseRevision: + description: LastReleaseRevision is the revision of the last successful + Helm release. + type: integer + observedGeneration: + description: ObservedGeneration is the last observed generation. + format: int64 + type: integer + tested: + description: Tested is the bool value whether the Helm Release was + successfully tested or not. + type: boolean + upgradeFailures: + description: UpgradeFailures is the upgrade failure count against + the latest desired state. It is reset after a successful reconciliation. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/images/armada/Dockerfile.ubuntu_focal b/images/armada/Dockerfile.ubuntu_focal index ed989f53..6a68be90 100644 --- a/images/armada/Dockerfile.ubuntu_focal +++ b/images/armada/Dockerfile.ubuntu_focal @@ -75,6 +75,8 @@ RUN set -ex \ && pip3 install -r requirements-frozen.txt --no-cache-dir \ && curl -fSSL -O ${HELM_ARTIFACT_URL} \ && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \ + && curl -fSSL -o /usr/local/bin/armada-go https://artifacts-nc.mtn57z.cti.att.com/artifactory/cloud-images-local/armada-go/armada-go \ + && chmod a+x /usr/local/bin/armada-go \ && mv linux-amd64/helm /usr/local/bin \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get autoremove -yqq --purge \