From c5d39f27cacaa953be43c7d3265bb693db0939d0 Mon Sep 17 00:00:00 2001
From: Phil Sphicas <phil.sphicas@att.com>
Date: Wed, 9 Feb 2022 10:04:38 -0800
Subject: [PATCH] Create lock CRD as apiextensions.k8s.io/v1 object

Kubernetes v1.22 stopped serving the apiextensions.k8s.io/v1beta1 API
version of CustomResourceDefinition.

This change ensures that the locks.armada.process CRD is created using
the apiextensions.k8s.io/v1 API.

The kubernetes client package is also updated to take advantage of the
dynamic client.

Change-Id: Icd518ab5cbb78e8b15f63d19c51b5f5b9a67e995
---
 armada/handlers/k8s.py  |  7 +++----
 armada/handlers/lock.py | 42 +++++++++++++++++------------------------
 requirements.txt        |  4 ++--
 3 files changed, 22 insertions(+), 31 deletions(-)

diff --git a/armada/handlers/k8s.py b/armada/handlers/k8s.py
index 439b047d..1312b1d0 100644
--- a/armada/handlers/k8s.py
+++ b/armada/handlers/k8s.py
@@ -57,8 +57,7 @@ class K8s(object):
         self.batch_api = client.BatchV1Api(api_client)
         self.batch_v1beta1_api = client.BatchV1beta1Api(api_client)
         self.custom_objects = client.CustomObjectsApi(api_client)
-        self.api_extensions = client.ApiextensionsV1beta1Api(api_client)
-        self.extension_api = client.ExtensionsV1beta1Api(api_client)
+        self.api_extensions = client.ApiextensionsV1Api(api_client)
         self.apps_v1_api = client.AppsV1Api(api_client)
 
     def delete_job_action(
@@ -369,10 +368,10 @@ class K8s(object):
 
         :param crd: custom resource definition to create
 
-        :type crd: kubernetes.client.V1beta1CustomResourceDefinition
+        :type crd: kubernetes.client.V1CustomResourceDefinition
 
         :return: new custom resource definition
-        :rtype: kubernetes.client.V1beta1CustomResourceDefinition
+        :rtype: kubernetes.client.V1CustomResourceDefinition
         """
         return self.api_extensions.create_custom_resource_definition(crd)
 
diff --git a/armada/handlers/lock.py b/armada/handlers/lock.py
index 23221f48..ee084856 100644
--- a/armada/handlers/lock.py
+++ b/armada/handlers/lock.py
@@ -281,40 +281,32 @@ class LockConfig:
         return lock
 
     def create_definition(self):
-        names = client.V1beta1CustomResourceDefinitionNames(
+        names = client.V1CustomResourceDefinitionNames(
             kind="Resource", plural=LOCK_PLURAL, singular=LOCK_SINGULAR)
         metadata = client.V1ObjectMeta(
             name="{}.{}".format(LOCK_PLURAL, LOCK_GROUP),
             resource_version=LOCK_VERSION)
-        status = client.V1beta1CustomResourceDefinitionStatus(
-            accepted_names=names,
-            conditions=[],
-            stored_versions=[LOCK_VERSION])
-        spec = client.V1beta1CustomResourceDefinitionSpec(
+        spec = client.V1CustomResourceDefinitionSpec(
             group=LOCK_GROUP,
             names=names,
             scope="Namespaced",
-            version=LOCK_VERSION)
-        crd = client.V1beta1CustomResourceDefinition(
-            spec=spec,
-            status=status,
-            metadata=metadata,
-            kind="CustomResourceDefinition")
+            versions=[
+                {
+                    "name": LOCK_VERSION,
+                    "schema": {
+                        "openAPIV3Schema": {
+                            "type": "object",
+                            "x-kubernetes-preserve-unknown-fields": True
+                        }
+                    },
+                    "served": True,
+                    "storage": True,
+                }
+            ])
+        crd = client.V1CustomResourceDefinition(
+            spec=spec, metadata=metadata, kind="CustomResourceDefinition")
         try:
             self.k8s.create_custom_resource_definition(crd)
-        except ValueError as err:
-            # Because of an issue with the Kubernetes code, the API server
-            # may return `null` for the required field `conditions` in
-            # kubernetes.client.V1beta1CustomResourceDefinitionStatus
-            # This causes validation to fail which will raise the subsequent
-            # ValueError even though the CRD was created successfully
-            # https://github.com/kubernetes-client/gen/issues/52
-            # TODO if this is fixed upstream this should be removed
-            known_msg = "Invalid value for `conditions`, must not be `None`"
-            known_err = ValueError(known_msg)
-            if err.args != known_err.args:
-                raise
-            LOG.debug("Encountered known issue while creating CRD, continuing")
         except ApiException as err:
             # If a 409 is received then the definition already exists
             if err.status != 409:
diff --git a/requirements.txt b/requirements.txt
index 69753e73..0559ce77 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,7 @@ jsonschema>=3.0.1,<4
 keystoneauth1>=3.18.0
 keystonemiddleware==5.3.0
 kombu<4.7,>=4.6.10
-kubernetes>=12.0.0
+kubernetes<23,>=17.0.0
 Paste>=2.0.3
 PasteDeploy>=1.5.2
 pylibyaml~=0.1
@@ -13,7 +13,7 @@ pyyaml~=5.1
 requests
 retry
 setuptools>=40.4.3
-prometheus_client>=0.7.0
+prometheus_client<0.13.0,>=0.7.0
 
 # API
 falcon