chore: remove Airflow completely from the stack
Drop all four Airflow containers (db, init, webserver, scheduler) from the mlops compose profile, leaving MLflow as the sole mlops service. Remove AIRFLOW_* env vars, config fields, health-check entries, DAG trigger code in admin/bench routes, the airflow_dag_run_id schema column, Airflow nav links and DAG-run links in the admin UI, the two Airflow DAG files (bench_dag.py, sim_dag.py), and all related docs/ADR references. Simulations now run exclusively via the subprocess path. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,9 +13,6 @@ services:
|
||||
NODE_ENV: production
|
||||
ML_SERVING_URL: "http://ml-serving:8000"
|
||||
MLFLOW_URL: "http://mlflow:5000"
|
||||
AIRFLOW_URL: "http://airflow-webserver:8080"
|
||||
AIRFLOW_API_USER: "admin"
|
||||
AIRFLOW_API_PASSWORD: "${AIRFLOW_ADMIN_PASSWORD:-admin}"
|
||||
INTERNAL_API_TOKEN: "${INTERNAL_API_TOKEN:-}"
|
||||
volumes:
|
||||
- /mnt/ssd/dbs/oo:/mnt/ssd/dbs/oo
|
||||
@@ -56,7 +53,6 @@ services:
|
||||
HOSTNAME: "0.0.0.0"
|
||||
NEXT_PUBLIC_API_URL: ""
|
||||
NEXT_PUBLIC_MLFLOW_URL: "/mlflow"
|
||||
NEXT_PUBLIC_AIRFLOW_URL: "/airflow"
|
||||
INTERNAL_API_URL: "http://api:3078"
|
||||
ports:
|
||||
- "127.0.0.1:3080:3080"
|
||||
@@ -85,100 +81,9 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── mlops profile — MLflow + Airflow ──────────────────────────────────────
|
||||
# ── mlops profile — MLflow ────────────────────────────────────────────────
|
||||
# Start: docker compose --profile mlops up
|
||||
# MLflow UI: http://localhost:5000 or https://o.alogins.net/mlflow (admin / password — change via basic_auth.ini)
|
||||
# Airflow UI: http://localhost:8080/airflow or https://o.alogins.net/airflow (admin / AIRFLOW_ADMIN_PASSWORD)
|
||||
# Caddy routes /mlflow* and /airflow* inside the o.alogins.net block
|
||||
|
||||
airflow-db:
|
||||
image: postgres:16-alpine
|
||||
profiles: [mlops]
|
||||
environment:
|
||||
POSTGRES_DB: airflow
|
||||
POSTGRES_USER: airflow
|
||||
POSTGRES_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow}
|
||||
volumes:
|
||||
- /mnt/ssd/dbs/oo/airflow-db:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U airflow"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
airflow-init:
|
||||
image: apache/airflow:2.9.3
|
||||
profiles: [mlops]
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- -c
|
||||
- |
|
||||
airflow db migrate
|
||||
airflow users create \
|
||||
--username admin \
|
||||
--firstname Admin \
|
||||
--lastname User \
|
||||
--role Admin \
|
||||
--email admin@oo.local \
|
||||
--password "$${AIRFLOW_ADMIN_PASSWORD:-admin}"
|
||||
environment:
|
||||
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:-airflow}@airflow-db/airflow
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_SECRET_KEY:-change-me-in-prod}
|
||||
AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW_BASE_URL:-https://o.alogins.net/airflow}
|
||||
depends_on:
|
||||
airflow-db:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
airflow-webserver:
|
||||
image: apache/airflow:2.9.3
|
||||
profiles: [mlops]
|
||||
command: webserver
|
||||
environment:
|
||||
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:-airflow}@airflow-db/airflow
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_SECRET_KEY:-change-me-in-prod}
|
||||
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-}
|
||||
AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW_BASE_URL:-https://o.alogins.net/airflow}
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth"
|
||||
_PIP_ADDITIONAL_REQUIREMENTS: "mlflow==2.14.3 httpx"
|
||||
MLFLOW_TRACKING_URI: "http://mlflow:5000/mlflow"
|
||||
MLFLOW_TRACKING_USERNAME: "admin"
|
||||
MLFLOW_TRACKING_PASSWORD: "${MLFLOW_ADMIN_PASSWORD:-password}"
|
||||
volumes:
|
||||
- ../../ml/pipelines:/opt/airflow/dags:ro
|
||||
- ../../ml:/opt/airflow/ml:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
depends_on:
|
||||
airflow-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
airflow-scheduler:
|
||||
image: apache/airflow:2.9.3
|
||||
profiles: [mlops]
|
||||
command: scheduler
|
||||
environment:
|
||||
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:-airflow}@airflow-db/airflow
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-}
|
||||
_PIP_ADDITIONAL_REQUIREMENTS: "mlflow==2.14.3 httpx"
|
||||
MLFLOW_TRACKING_URI: "http://mlflow:5000/mlflow"
|
||||
MLFLOW_TRACKING_USERNAME: "admin"
|
||||
MLFLOW_TRACKING_PASSWORD: "${MLFLOW_ADMIN_PASSWORD:-password}"
|
||||
volumes:
|
||||
- ../../ml/pipelines:/opt/airflow/dags:ro
|
||||
- ../../ml:/opt/airflow/ml:ro
|
||||
depends_on:
|
||||
airflow-init:
|
||||
condition: service_completed_successfully
|
||||
# MLflow UI: http://localhost:5000 or https://o.alogins.net/mlflow
|
||||
|
||||
# ── events profile — NATS JetStream ─────────────────────────────────────
|
||||
# Start: docker compose --profile events up
|
||||
@@ -201,7 +106,7 @@ services:
|
||||
retries: 5
|
||||
|
||||
mlflow:
|
||||
image: ghcr.io/mlflow/mlflow:v2.14.3
|
||||
image: ghcr.io/mlflow/mlflow:v3.11.1
|
||||
profiles: [mlops]
|
||||
command: >
|
||||
mlflow server
|
||||
@@ -209,17 +114,15 @@ services:
|
||||
--default-artifact-root /mlflow/artifacts
|
||||
--host 0.0.0.0
|
||||
--port 5000
|
||||
--app-name basic-auth
|
||||
--static-prefix /mlflow
|
||||
environment:
|
||||
MLFLOW_AUTH_CONFIG_PATH: /mlflow/basic_auth.ini
|
||||
--allowed-hosts o.alogins.net,localhost
|
||||
--cors-allowed-origins https://o.alogins.net
|
||||
volumes:
|
||||
- /mnt/ssd/dbs/oo/mlflow:/mlflow
|
||||
- ../../infra/mlflow/basic_auth.ini:/mlflow/basic_auth.ini:ro
|
||||
ports:
|
||||
- "127.0.0.1:5000:5000"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:5000/health',timeout=3).status==200 else 1)"]
|
||||
test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:5000/mlflow/health',timeout=3).status==200 else 1)"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
Reference in New Issue
Block a user