From de9f8415d71aa3b8c632616c02ccb09c5024ecab Mon Sep 17 00:00:00 2001
From: Phil Sphicas <phil.sphicas@att.com>
Date: Fri, 23 Oct 2020 20:48:26 +0000
Subject: [PATCH] kube-apiserver: disable http2

There are several kubernetes bugs [0,1,2] involving connection problems
that seem related to the Go net/http2 library, where the stream state
and connection state can get out of sync. This can manifest as a kubelet
issue, where the node status gets stuck in a NotReady state, but can
also happen elsewhere.

In newer versions of the Go libraries some issues are fixed [3,4], but
the fixes are not present in k8s 1.18.

This change disables http2 in kube-apiserver and webhook-apiserver. This
should be sufficient to avoid the majority of the issues, as disabling
on one side of the connection is enough, and apiserver is generally
either the client or the server.

0: https://github.com/kubernetes/kubernetes/issues/87615
1: https://github.com/kubernetes/kubernetes/issues/80313
2: https://github.com/kubernetes/client-go/issues/374
3: https://github.com/golang/go/issues/40423
4: https://github.com/golang/go/issues/40201

Change-Id: Id693a7201acffccbc4b3db8f4e4b96290fd50288
---
 charts/apiserver-webhook/templates/deployment.yaml            | 1 +
 charts/apiserver-webhook/values.yaml                          | 3 +++
 charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl | 1 +
 charts/apiserver/values.yaml                                  | 3 +++
 4 files changed, 8 insertions(+)

diff --git a/charts/apiserver-webhook/templates/deployment.yaml b/charts/apiserver-webhook/templates/deployment.yaml
index 1689ac7c..0ce2b8e7 100644
--- a/charts/apiserver-webhook/templates/deployment.yaml
+++ b/charts/apiserver-webhook/templates/deployment.yaml
@@ -141,6 +141,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.apiserver | indent 12 }}
           command:
             {{- range .Values.command_prefix }}
             - {{ . }}
diff --git a/charts/apiserver-webhook/values.yaml b/charts/apiserver-webhook/values.yaml
index 211e1e8b..71b5d81b 100644
--- a/charts/apiserver-webhook/values.yaml
+++ b/charts/apiserver-webhook/values.yaml
@@ -238,6 +238,9 @@ pod:
       failureThreshold: 3
       initialDelaySeconds: 15
       periodSeconds: 20
+  env:
+    apiserver:
+      GODEBUG: http2server=0,http2client=0
   lifecycle:
     upgrades:
       daemonsets:
diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
index acd7f5e1..b2c648e4 100644
--- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
+++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
@@ -89,6 +89,7 @@ spec:
           value: {{ .Values.network.kubernetes_apiserver.port | quote }}
         - name: ETCD_ENDPOINTS
           value: {{ .Values.apiserver.etcd.endpoints | quote }}
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.apiserver | indent 8 }}
 
       command:
         {{- range .Values.const.command_prefix }}
diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml
index 0549111e..cb4bf098 100644
--- a/charts/apiserver/values.yaml
+++ b/charts/apiserver/values.yaml
@@ -350,6 +350,9 @@ pod:
             initialDelaySeconds: 10
             periodSeconds: 5
             timeoutSeconds: 5
+  env:
+    apiserver:
+      GODEBUG: http2server=0,http2client=0
 
 manifests:
   configmap_bin: true