Simplify the test environment (#15503)

* Simplify the test environment

* test 2.6.2

airflow/hatch.toml

@@ -2,10 +2,13 @@
 
 [[envs.default.matrix]]
 python = ["2.7", "3.9"]
-version = ["v2"]
+version = ["2.1", "2.6"]
 
 [envs.default.overrides]
-matrix.version.env-vars = "AIRFLOW_VERSION"
+matrix.version.env-vars = [
+  { key = "AIRFLOW_VERSION", value = "2.1.4", if = ["2.1"] },
+  { key = "AIRFLOW_VERSION", value = "2.6.2", if = ["2.6"] },
+]
 
 [envs.latest.env-vars]
 AIRFLOW_VERSION = "latest"

airflow/tests/compose/v2/Dockerfile → airflow/tests/compose/Dockerfile

@@ -1,4 +1,6 @@
-FROM apache/airflow:2.1.4-python3.8
+ARG AIRFLOW_VERSION
+
+FROM apache/airflow:$AIRFLOW_VERSION
 USER root
 # INSTALL TOOLS
 RUN apt-get update \

airflow/tests/compose/v2/docker-compose.yaml → airflow/tests/compose/docker-compose.yaml

@@ -3,6 +3,8 @@ version: '3.5'
 services:
   postgres:
     image: postgres:12-alpine
+    hostname: airflow-postgres
+    container_name: airflow-postgres
     environment:
       POSTGRES_USER: airflow
       POSTGRES_PASSWORD: airflow
@@ -14,7 +16,12 @@ services:
       - "5432:5432"
 
   server:
-    build: .
+    hostname: airflow-server
+    container_name: airflow-server
+    build:
+      context: .
+      args:
+        AIRFLOW_VERSION: ${AIRFLOW_VERSION}
     environment:
       # -- CORE
       AIRFLOW__CORE__EXECUTOR: LocalExecutor
@@ -32,8 +39,6 @@ services:
       DB__NAME: airflow
     ports:
       - "8080:8080"
-    volumes:
-       - ./dags:/opt/airflow/dags
 
 volumes:
   postgres:

airflow/tests/conftest.py

@@ -6,19 +6,13 @@
 
 import pytest
 
-from datadog_checks.dev import docker_run
+from datadog_checks.dev import TempDir, docker_run
 from datadog_checks.dev.conditions import CheckEndpoints
 
-from .common import AIRFLOW_VERSION, URL
+from .common import URL
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 
-TMP_DATA_FOLDER = path.join(HERE, 'compose', AIRFLOW_VERSION, 'tmp_data')
-
-E2E_METADATA = {
-    'docker_volumes': ['{}/datadog.yaml:/etc/datadog-agent/datadog.yaml'.format(TMP_DATA_FOLDER)],
-}
-
 
 def get_readme_mappings():
     with open(path.join(HERE, '..', 'README.md'), 'r') as f:
@@ -29,29 +23,23 @@ def get_readme_mappings():
     return readme[start : start + end]
 
 
-def create_datadog_config(datadog_config):
-    with open(path.join(TMP_DATA_FOLDER, 'datadog.yaml'), 'w') as f:
-        f.write(datadog_config)
-
-
 @pytest.fixture(scope='session')
 def dd_environment(instance):
     datadog_config = """
 dogstatsd_metrics_stats_enable: true
 """
-    create_datadog_config(datadog_config + get_readme_mappings())
-
-    endpoint = URL
-    if AIRFLOW_VERSION == "v2":
-        endpoint += "/api/v1/health"
-    else:
-        endpoint += "/api/experimental/test"
-
-    with docker_run(
-        os.path.join(HERE, 'compose', AIRFLOW_VERSION, 'docker-compose.yaml'),
-        conditions=[CheckEndpoints(endpoint, attempts=120)],
-    ):
-        yield instance, E2E_METADATA
+    with TempDir() as temp_dir:
+        with open(path.join(temp_dir, 'datadog.yaml'), 'w') as f:
+            f.write(datadog_config + get_readme_mappings())
+
+        with docker_run(
+            os.path.join(HERE, 'compose', 'docker-compose.yaml'),
+            build=True,
+            conditions=[CheckEndpoints(URL + "/api/v1/health", attempts=120)],
+        ):
+            yield instance, {
+                'docker_volumes': ['{}/datadog.yaml:/etc/datadog-agent/datadog.yaml'.format(temp_dir)],
+            }
 
 
 @pytest.fixture(scope='session')