準備工作

下載docker：https://download.docker.com/linux/static/stable/x86_64

我下載的是docker-20.10.6.tgz

下載docker-compose：https://github.com/docker/compose/releases

我下載的是docker-compose-linux-x86_64

構建airflow映象：

Dockerfile

FROM python:3.8.12-buster
ENV TZ="Asia/Shanghai"

RUN sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list
RUN sed -i s@/security.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list
RUN apt-get clean
RUN apt-get -y update
RUN apt-get -y install vim
RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
RUN pip install --upgrade pip
COPY requirements.txt /requirements.txt
RUN pip install -r /requirements.txt
RUN rm /requirements.txt
 

requirements

apache_airflow==2.2.2
apache-airflow-providers-docker==2.3.0
psycopg2==2.9.2
pandas==1.3.5

build和save

 docker build -t airflow:1.0 .
 docker save -o ./airflow.tar airflow:1.0

下載postgres資料庫

docker pull postgres:13
docker save -o ./ postgres:13

開始安裝

安裝docker

建立docker.service

[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
#TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
# restart the docker process if it exits prematurely
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target
 

依次執行以下命令

# 解壓檔案
tar -xvf docker-20.10.6.tgz
# 移動docker到bin下
cp docker/* /usr/bin/
# 移動docker.service
cp docker.service /etc/systemd/system/
# 設定許可權
chmod +x /etc/systemd/system/docker.service
# 重新載入配置檔案
systemctl daemon-reload
# 啟動docker
systemctl start docker
# 開機自啟
systemctl enable docker.service
# 完成
docker -v
 

安裝docker-compose

mv docker-compose-linux-x86_64 /usr/local/bin/docker-compose

新增許可權

chmod +x /usr/local/bin/docker-compose

測試

docker-compose -v

安裝airflow

建立docker-compose.yml

version: '3.8'
x-airflow-common:
  &airflow-common
  image: airflow:1.0
  environment:
    - AIRFLOW__CORE__EXECUTOR=LocalExecutor
    - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
    - AIRFLOW__CORE__LOAD_EXAMPLES=False
    - AIRFLOW__CORE__DEFAULT_TIMEZONE=Asia/Shanghai
    - AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE=Asia/Shanghai
    - TZ=Asia/Shanghai
    - POSTGRES_ENGINE_STRING=postgresql+psycopg2://postgres:[email protected]:5432/stock
    - DOCKER_URL=tcp://192.168.31.214:2375
  volumes:
    - /root/airflow/dags:/root/airflow/dags
    - /root/airflow/logs:/root/airflow/logs
  depends_on:
    - postgres

services:
  postgres:
    container_name: airflow_postgres
    image: postgres:13
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
      - POSTGRES_PORT=5432
    volumes:
      - /root/airflow/postgres/data:/var/lib/postgresql/data
    ports:
      - 54321:5432
  airflow-init:
    <<: *airflow-common
    container_name: airflow_init
    entrypoint: /bin/bash
    command:
      - -c
      - airflow db check-migrations || ( airflow db init &&
        airflow users create
        --role Admin
        --username airflow
        --password airflow
        --email [email protected]
        --firstname airflow
        --lastname airflow )
    restart: on-failure

  airflow-webserver:
    <<: *airflow-common
    command: airflow webserver
    ports:
      - 8080:8080
    container_name: airflow_webserver
    restart: always

  airflow-scheduler:
    <<: *airflow-common
    command: airflow scheduler
    container_name: airflow_scheduler
    restart: always
    depends_on:
      airflow-init:
        condition: service_completed_successfully

匯入docker映象

docker load --input airflow.tar
docker load --input postgres.tar

啟動docker

docker-compose up -d

啟動完成後，開啟http://192.168.31.214:8080就可以看到airflow了。