(airflow) - Apache airflow uplift to 1.10.4

Updated airflow config and Dockerfiles for apache airflow 1.10.4
For details see: https://github.com/apache/airflow/blob/master/UPDATING.md

Also updated the kubectl to 1.16.2, as part of kubernetes 1.16 uplift.

Change-Id: Ib24ff0304b6279ff0be749141854d6a604473597

Makefile

@@ -32,6 +32,7 @@ NO_PROXY                   ?= localhost,127.0.0.1,.svc.cluster.local
 USE_PROXY                  ?= false
 
 AIRFLOW_SRC                ?=
+AIRFLOW_HOME               ?=
 DISTRO_BASE_IMAGE          ?=
 DISTRO                     ?= ubuntu_xenial
 
@@ -86,6 +87,7 @@ run:
 _BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,)
 ifeq ($(IMAGE_NAME), airflow)
 	_AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,)
+	_AIRFLOW_HOME_ARG := $(if $(AIRFLOW_HOME),--build-arg AIRFLOW_HOME="${AIRFLOW_HOME}" ,)
 endif
 
 .PHONY: build
@@ -98,6 +100,7 @@ ifeq ($(USE_PROXY), true)
 		-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
 		$(_BASE_IMAGE_ARG) \
 		$(_AIRFLOW_SRC_ARG) \
+		$(_AIRFLOW_HOME_ARG) \
 		--build-arg http_proxy=$(PROXY) \
 		--build-arg https_proxy=$(PROXY) \
 		--build-arg HTTP_PROXY=$(PROXY) \
@@ -113,6 +116,7 @@ else
 		-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
 		$(_BASE_IMAGE_ARG) \
 		$(_AIRFLOW_SRC_ARG) \
+		$(_AIRFLOW_HOME_ARG) \
 		--build-arg ctx_base=$(BUILD_CTX) .
 endif
 ifeq ($(PUSH_IMAGE), true)

charts/shipyard/values.yaml

@@ -454,7 +454,9 @@ conf:
     # NOTE: Airflow 1.10 introduces a need to declare all config options:
     #     https://issues.apache.org/jira/browse/AIRFLOW-3099
     core:
-      airflow_home: /usr/local/airflow
+      # core.airflow_home is not used in 1.10.3 and later.
+      # Envrionment variable AIRFLOW_HOME is used instead.
+      #airflow_home: /usr/local/airflow
       dags_folder: /usr/local/airflow/dags
       base_log_folder: /usr/local/airflow/logs
       remote_logging: "False"
@@ -497,15 +499,17 @@ conf:
       parallelism: 32
       dag_concurrency: 8
       dags_are_paused_at_creation: "False"
-      non_pooled_task_slot_count: 128
+      # In 1.10.4, non_pooled_task_slot_count and non_pooled_backfill_task_slot_count are
+      # removed in favor of default_pool, which is initialized with 128 slots by default.
+      #non_pooled_task_slot_count: 128
       max_active_runs_per_dag: 8
       load_examples: "False"
       plugins_folder: /usr/local/airflow/plugins
       fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
       donot_pickle: "False"
       dagbag_import_timeout: 30
-      task_runner: "BashTaskRunner"
-      # task_runner: "StandardTaskRunner"  -- coming soon?
+      # BashTaskRunner has been renamed to StandardTaskRunner from 1.10.3
+      task_runner: "StandardTaskRunner"
       default_impersonation: ""
       security: ""
       secure_mode: "True"
@@ -683,9 +687,9 @@ conf:
       hide_sensitive_variable_fields: "True"
     elasticsearch:
       # Shipyard is not using this
-      elasticsearch_host: ""
-      elasticsearch_log_id_template: ""
-      elasticsearch_end_of_log_mark: ""
+      host: ""
+      log_id_template: ""
+      end_of_log_mark: ""
     kubernetes:
       # Shipyard is not using this (maybe future for spawning own workers)
       worker_container_repository: ""

images/airflow/Dockerfile.opensuse_15

@@ -40,11 +40,14 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
-ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
+ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
 ARG ctx_base=src/bin
 
 # Kubectl version
-ARG KUBECTL_VERSION=1.10.2
+ARG KUBECTL_VERSION=1.16.2
+
+# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
+ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
 RUN set -ex && \
     zypper -q update -y ;\

images/airflow/Dockerfile.ubuntu_xenial

@@ -40,12 +40,15 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
-ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
+ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG ctx_base=src/bin
 
 # Kubectl version
-ARG KUBECTL_VERSION=1.10.2
+ARG KUBECTL_VERSION=1.16.2
+
+# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
+ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
 RUN set -ex && \
     apt-get -qq update && \