Install Python dependencies to docker-compose cluster without re-build images

pip install --target=
from datetime import datetime
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator

from slack import *

with DAG(dag_id='dependency_dag',
start_date=datetime(2019, 12, 17),
schedule_interval=None) as dag:
DummyOperator(task_id='dummy')
import error in Airflow UI
/usr/local/airflow/packages

site-packages folder

import sys
print(sys.path)
site-packages in Apache Airflow container
docker exec -it #container_id /bin/bash
/usr/local/lib/python3.7/site-packages

Docker volume

- ./airflow_files/packages:/usr/local/airflow/packages
volumes:
- ./airflow_files/dags:/usr/local/airflow/dags
- ./airflow_files/logs:/usr/local/airflow/logs
- ./airflow_files/packages:/usr/local/airflow/packages

Dockerfile

RUN mkdir /usr/local/airflow/packages
COPY ./packages.pth /usr/local/lib/python3.7/site-packages
FROM puckel/docker-airflow:1.10.6
RUN pip install --user psycopg2-binary
ENV AIRFLOW_HOME=/usr/local/airflow
RUN mkdir /usr/local/airflow/packages
COPY ./packages.pth /usr/local/lib/python3.7/site-packages
COPY ./airflow.cfg /usr/local/airflow/airflow.cfg

.dockerignore

airflow_files/
data/
docs/
LICENSE
README.md
airflow/packages added to site-packages

pip install with ‘target’ flag

pip install --target=./airflow_files/packages slackclient
pip install with target flag
packages installed in target folder
no import error UI

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store