(airflow) - Apache airflow uplift to 1.10.4
Updated airflow config and Dockerfiles for apache airflow 1.10.4 For details see: https://github.com/apache/airflow/blob/master/UPDATING.md Also updated the kubectl to 1.16.2, as part of kubernetes 1.16 uplift. Change-Id: Ib24ff0304b6279ff0be749141854d6a604473597
This commit is contained in:
parent
0cbb5cf9b7
commit
f0f876afef
4
Makefile
4
Makefile
|
@ -32,6 +32,7 @@ NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local
|
||||||
USE_PROXY ?= false
|
USE_PROXY ?= false
|
||||||
|
|
||||||
AIRFLOW_SRC ?=
|
AIRFLOW_SRC ?=
|
||||||
|
AIRFLOW_HOME ?=
|
||||||
DISTRO_BASE_IMAGE ?=
|
DISTRO_BASE_IMAGE ?=
|
||||||
DISTRO ?= ubuntu_xenial
|
DISTRO ?= ubuntu_xenial
|
||||||
|
|
||||||
|
@ -86,6 +87,7 @@ run:
|
||||||
_BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,)
|
_BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,)
|
||||||
ifeq ($(IMAGE_NAME), airflow)
|
ifeq ($(IMAGE_NAME), airflow)
|
||||||
_AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,)
|
_AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,)
|
||||||
|
_AIRFLOW_HOME_ARG := $(if $(AIRFLOW_HOME),--build-arg AIRFLOW_HOME="${AIRFLOW_HOME}" ,)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: build
|
.PHONY: build
|
||||||
|
@ -98,6 +100,7 @@ ifeq ($(USE_PROXY), true)
|
||||||
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
|
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
|
||||||
$(_BASE_IMAGE_ARG) \
|
$(_BASE_IMAGE_ARG) \
|
||||||
$(_AIRFLOW_SRC_ARG) \
|
$(_AIRFLOW_SRC_ARG) \
|
||||||
|
$(_AIRFLOW_HOME_ARG) \
|
||||||
--build-arg http_proxy=$(PROXY) \
|
--build-arg http_proxy=$(PROXY) \
|
||||||
--build-arg https_proxy=$(PROXY) \
|
--build-arg https_proxy=$(PROXY) \
|
||||||
--build-arg HTTP_PROXY=$(PROXY) \
|
--build-arg HTTP_PROXY=$(PROXY) \
|
||||||
|
@ -113,6 +116,7 @@ else
|
||||||
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
|
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
|
||||||
$(_BASE_IMAGE_ARG) \
|
$(_BASE_IMAGE_ARG) \
|
||||||
$(_AIRFLOW_SRC_ARG) \
|
$(_AIRFLOW_SRC_ARG) \
|
||||||
|
$(_AIRFLOW_HOME_ARG) \
|
||||||
--build-arg ctx_base=$(BUILD_CTX) .
|
--build-arg ctx_base=$(BUILD_CTX) .
|
||||||
endif
|
endif
|
||||||
ifeq ($(PUSH_IMAGE), true)
|
ifeq ($(PUSH_IMAGE), true)
|
||||||
|
|
|
@ -454,7 +454,9 @@ conf:
|
||||||
# NOTE: Airflow 1.10 introduces a need to declare all config options:
|
# NOTE: Airflow 1.10 introduces a need to declare all config options:
|
||||||
# https://issues.apache.org/jira/browse/AIRFLOW-3099
|
# https://issues.apache.org/jira/browse/AIRFLOW-3099
|
||||||
core:
|
core:
|
||||||
airflow_home: /usr/local/airflow
|
# core.airflow_home is not used in 1.10.3 and later.
|
||||||
|
# Envrionment variable AIRFLOW_HOME is used instead.
|
||||||
|
#airflow_home: /usr/local/airflow
|
||||||
dags_folder: /usr/local/airflow/dags
|
dags_folder: /usr/local/airflow/dags
|
||||||
base_log_folder: /usr/local/airflow/logs
|
base_log_folder: /usr/local/airflow/logs
|
||||||
remote_logging: "False"
|
remote_logging: "False"
|
||||||
|
@ -497,15 +499,17 @@ conf:
|
||||||
parallelism: 32
|
parallelism: 32
|
||||||
dag_concurrency: 8
|
dag_concurrency: 8
|
||||||
dags_are_paused_at_creation: "False"
|
dags_are_paused_at_creation: "False"
|
||||||
non_pooled_task_slot_count: 128
|
# In 1.10.4, non_pooled_task_slot_count and non_pooled_backfill_task_slot_count are
|
||||||
|
# removed in favor of default_pool, which is initialized with 128 slots by default.
|
||||||
|
#non_pooled_task_slot_count: 128
|
||||||
max_active_runs_per_dag: 8
|
max_active_runs_per_dag: 8
|
||||||
load_examples: "False"
|
load_examples: "False"
|
||||||
plugins_folder: /usr/local/airflow/plugins
|
plugins_folder: /usr/local/airflow/plugins
|
||||||
fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
|
fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
|
||||||
donot_pickle: "False"
|
donot_pickle: "False"
|
||||||
dagbag_import_timeout: 30
|
dagbag_import_timeout: 30
|
||||||
task_runner: "BashTaskRunner"
|
# BashTaskRunner has been renamed to StandardTaskRunner from 1.10.3
|
||||||
# task_runner: "StandardTaskRunner" -- coming soon?
|
task_runner: "StandardTaskRunner"
|
||||||
default_impersonation: ""
|
default_impersonation: ""
|
||||||
security: ""
|
security: ""
|
||||||
secure_mode: "True"
|
secure_mode: "True"
|
||||||
|
@ -683,9 +687,9 @@ conf:
|
||||||
hide_sensitive_variable_fields: "True"
|
hide_sensitive_variable_fields: "True"
|
||||||
elasticsearch:
|
elasticsearch:
|
||||||
# Shipyard is not using this
|
# Shipyard is not using this
|
||||||
elasticsearch_host: ""
|
host: ""
|
||||||
elasticsearch_log_id_template: ""
|
log_id_template: ""
|
||||||
elasticsearch_end_of_log_mark: ""
|
end_of_log_mark: ""
|
||||||
kubernetes:
|
kubernetes:
|
||||||
# Shipyard is not using this (maybe future for spawning own workers)
|
# Shipyard is not using this (maybe future for spawning own workers)
|
||||||
worker_container_repository: ""
|
worker_container_repository: ""
|
||||||
|
|
|
@ -40,11 +40,14 @@ EXPOSE $WORKER_PORT
|
||||||
|
|
||||||
# Set ARG for usage during build
|
# Set ARG for usage during build
|
||||||
ARG AIRFLOW_HOME=/usr/local/airflow
|
ARG AIRFLOW_HOME=/usr/local/airflow
|
||||||
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
|
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
|
||||||
ARG ctx_base=src/bin
|
ARG ctx_base=src/bin
|
||||||
|
|
||||||
# Kubectl version
|
# Kubectl version
|
||||||
ARG KUBECTL_VERSION=1.10.2
|
ARG KUBECTL_VERSION=1.16.2
|
||||||
|
|
||||||
|
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
|
||||||
|
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
|
||||||
|
|
||||||
RUN set -ex && \
|
RUN set -ex && \
|
||||||
zypper -q update -y ;\
|
zypper -q update -y ;\
|
||||||
|
|
|
@ -40,12 +40,15 @@ EXPOSE $WORKER_PORT
|
||||||
|
|
||||||
# Set ARG for usage during build
|
# Set ARG for usage during build
|
||||||
ARG AIRFLOW_HOME=/usr/local/airflow
|
ARG AIRFLOW_HOME=/usr/local/airflow
|
||||||
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
|
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
|
||||||
ARG DEBIAN_FRONTEND=noninteractive
|
ARG DEBIAN_FRONTEND=noninteractive
|
||||||
ARG ctx_base=src/bin
|
ARG ctx_base=src/bin
|
||||||
|
|
||||||
# Kubectl version
|
# Kubectl version
|
||||||
ARG KUBECTL_VERSION=1.10.2
|
ARG KUBECTL_VERSION=1.16.2
|
||||||
|
|
||||||
|
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
|
||||||
|
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
|
||||||
|
|
||||||
RUN set -ex && \
|
RUN set -ex && \
|
||||||
apt-get -qq update && \
|
apt-get -qq update && \
|
||||||
|
|
Loading…
Reference in New Issue