Separating common environment variables in docker compose

Question

So I have few environment variables which are common across a few services. Now replicating them is prone to mistakes. Somebody recently informed me about yaml anchors but somehow I am not being able to use them correctly. It could be great if someone can help me fix it.

https://docs.docker.com/compose/compose-file/11-extension/

version: '3'

x-common-environment: &common-environment
  - PORT=8080
  - HOST_SERVER=localhost
  - TZ=Asia/Kolkata
  - PROTOCOL=http

x-keycloak-service: &keycloak-service
  - KEYCLOAK_SERVICE=keycloak

services:
  nginx_proxy:
    image: nginx
    container_name: nginx
    ports:
      - "8080:80"
    volumes:
      - ./nginx:/etc/nginx
      - ./nginx/log:/var/log/nginx
      - ./https:/https
    networks:
      - proxy
    profiles: [ "auth", "irasus" ]
    command: /bin/bash -c "envsubst < /etc/nginx/templates/nginx.conf.template > /etc/nginx/nginx.conf && nginx -g 'daemon off;'"
    environment:
      <<: *common-environment
      - DOLLAR=$$
      # - CERTIFICATE_FILE_NAME=auth_server.crt
      # - CERTIFICATE_KEY_FILE_NAME=auth_server.key

  keycloak:
    image: quay.io/keycloak/keycloak:21.0.2
    container_name: keycloak
    volumes:
      - ./https:/https
    networks:
      - proxy
      - keycloak
    profiles: [ "auth", "irasus" ]
    restart: on-failure:3
    environment:
        - KEYCLOAK_ADMIN=admin
        - KEYCLOAK_ADMIN_PASSWORD=admin
        - KC_PROXY=edge
        - KC_HOSTNAME_STRICT=false
        - KC_DB_URL=jdbc:postgresql://db_keycloak:5432/keycloak
        - KC_DB_USERNAME=admin
        - KC_DB_PASSWORD=admin
        - KC_DB=postgres
    command: "start"
    


  db_keycloak:
    image: postgres
    container_name: db_keycloak
    volumes:
      - ./postgres-data:/var/lib/postgresql/data
    networks:
      - keycloak
    profiles: [ "auth", "irasus" ]
    environment:
      - POSTGRES_PASSWORD=admin
      - POSTGRES_USER=admin
      - POSTGRES_DB=keycloak


  grafana:
    image: grafana/grafana-oss
    container_name: grafana
    volumes:
      - ./grafana:/etc/grafana
      - ./https:/https
    networks:
      - proxy
      - grafana
    profiles: [ "irasus" ]
    restart: on-failure:3
    environment:    
      <<: *keycloak-service
      - SERVICE_NAME=grafana
      - CLIENT_ID=grafana
      - CLIENT_SECRET=kSQ8xBdS8ONpsVngSyJpl409AQXMmdiW
      - ADMIN_USER=admin
      - ADMIN_PASSWORD=admin
      - REALM=master
      - SCOPES=openid profile offline_access roles

  db_grafana:
    image: postgres
    container_name: db_grafana
    networks:
      - grafana
    profiles: ["irasus" ]
    environment:
      - POSTGRES_PASSWORD=admin
      - POSTGRES_USER=admin
      - POSTGRES_DB=grafana



  nodered:
    image: teut2711/node-red-auth-oidc
    container_name: nodered
    volumes:
      - ./nodered:/data
    networks:
      - proxy
    profiles: [ "irasus" ]
    environment:
      <<: *keycloak-service
      <<: *common-environment
      - SERVICE_NAME=nodered
      - CLIENT_ID=node-red-editor
      - CLIENT_SECRET=9MyyuFF13ddO0F7UbXODBEgO9lF2ybFy
      - SSL_REQUIRED=none
      - SCOPE=openid email profile offline_access roles
      - HTTP_PROXY=http://nginx
      - STRATEGY=keycloak
      - REALM=master

networks:
  proxy:
    driver: bridge
  keycloak:
      driver: bridge
  grafana:
      driver: bridge

which I want to be equivalent to:

version: '3'




services:
  nginx_proxy:
    image: nginx
    container_name: nginx
    ports:
      - "8080:80"
    volumes:
      - ./nginx:/etc/nginx
      - ./nginx/log:/var/log/nginx
      - ./https:/https
    networks:
      - proxy
    profiles: [ "auth", "irasus" ]
    command: /bin/bash -c "envsubst < /etc/nginx/templates/nginx.conf.template > /etc/nginx/nginx.conf && nginx -g 'daemon off;'"
    environment:
        - PORT=8080
        - HOST_SERVER=localhost
        - TZ=Asia/Kolkata
        - PROTOCOL=http
        - DOLLAR=$$
        # - CERTIFICATE_FILE_NAME=auth_server.crt
        # - CERTIFICATE_KEY_FILE_NAME=auth_server.key

  keycloak:
    image: quay.io/keycloak/keycloak:21.0.2
    container_name: keycloak
    volumes:
      - ./https:/https
    networks:
      - proxy
      - keycloak
    profiles: [ "auth", "irasus" ]
    restart: on-failure:3
    environment:
        - KEYCLOAK_ADMIN=admin
        - KEYCLOAK_ADMIN_PASSWORD=admin
        - KC_PROXY=edge
        - KC_HOSTNAME_STRICT=false
        - KC_DB_URL=jdbc:postgresql://db_keycloak:5432/keycloak
        - KC_DB_USERNAME=admin
        - KC_DB_PASSWORD=admin
        - KC_DB=postgres
    command: "start"
    


  db_keycloak:
    image: postgres
    container_name: db_keycloak
    volumes:
      - ./postgres-data:/var/lib/postgresql/data
    networks:
      - keycloak
    profiles: [ "auth", "irasus" ]
    environment:
      - POSTGRES_PASSWORD=admin
      - POSTGRES_USER=admin
      - POSTGRES_DB=keycloak


  grafana:
    image: grafana/grafana-oss
    container_name: grafana
    volumes:
      - ./grafana:/etc/grafana
      - ./https:/https
    networks:
      - proxy
      - grafana
    profiles: [ "irasus" ]
    restart: on-failure:3
    environment:    
      - KEYCLOAK_SERVICE=keycloak
      - SERVICE_NAME=grafana
      - CLIENT_ID=grafana
      - CLIENT_SECRET=kSQ8xBdS8ONpsVngSyJpl409AQXMmdiW
      - ADMIN_USER=admin
      - ADMIN_PASSWORD=admin
      - REALM=master
      - SCOPES=openid profile offline_access roles

  db_grafana:
    image: postgres
    container_name: db_grafana
    networks:
      - grafana
    profiles: ["irasus" ]
    environment:
      - POSTGRES_PASSWORD=admin
      - POSTGRES_USER=admin
      - POSTGRES_DB=grafana



  nodered:
    image: teut2711/node-red-auth-oidc
    container_name: nodered
    volumes:
      - ./nodered:/data
    networks:
      - proxy
    profiles: [ "irasus" ]
    environment:
      - KEYCLOAK_SERVICE=keycloak
      - PORT=8080
      - HOST_SERVER=localhost
      - TZ=Asia/Kolkata
      - PROTOCOL=http
      - SERVICE_NAME=nodered
      - CLIENT_ID=node-red-editor
      - CLIENT_SECRET=9MyyuFF13ddO0F7UbXODBEgO9lF2ybFy
      - SSL_REQUIRED=none
      - SCOPE=openid email profile offline_access roles
      - HTTP_PROXY=http://nginx
      - STRATEGY=keycloak
      - REALM=master

networks:
  proxy:
    driver: bridge
  keycloak:
      driver: bridge
  grafana:
      driver: bridge

I get the linter error(also parsing errors when I tried to parse it with python's yaml module)

Traceback (most recent call last):
  File "/home/vishesh/Desktop/irasus/./a.py", line 22, in <module>
    config = yaml.safe_load(f)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/__init__.py", line 125, in safe_load
    return load(stream, SafeLoader)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/__init__.py", line 81, in load
    return loader.get_single_data()
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/constructor.py", line 49, in get_single_data
    node = self.get_single_node()
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 36, in get_single_node
    document = self.compose_document()
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 55, in compose_document
    node = self.compose_node(None, None)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 84, in compose_node
    node = self.compose_mapping_node(anchor)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 133, in compose_mapping_node
    item_value = self.compose_node(node, item_key)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 84, in compose_node
    node = self.compose_mapping_node(anchor)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 133, in compose_mapping_node
    item_value = self.compose_node(node, item_key)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 84, in compose_node
    node = self.compose_mapping_node(anchor)
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/composer.py", line 127, in compose_mapping_node
    while not self.check_event(MappingEndEvent):
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/parser.py", line 98, in check_event
    self.current_event = self.state()
  File "/home/vishesh/Desktop/irasus/venv/lib/python3.10/site-packages/yaml/parser.py", line 438, in parse_block_mapping_key
    raise ParserError("while parsing a block mapping", self.marks[-1],
yaml.parser.ParserError: while parsing a block mapping
  in "docker-compose.yaml", line 15, column 5
expected <block end>, but found '<block sequence start>'
  in "docker-compose.yaml", line 28, column 9

Also running docker compose up

vishesh@vishesh-HP-Laptop-15s-fr4xxx:~/Desktop/irasus$ sudo docker compose --profile irasus up
yaml: line 28: did not find expected key

I tried asking on IRC but I was told that the is correct.

I have even opened up a GitHub issue feature request in regards to a much flexible approach to attain what's required: https://github.com/docker/compose/issues/10545

EDIT: The solution proposed by ndeloof works but there is a slight issue.

Error in following line Bpaste

Answer 1

yaml merge only applies to mappings, you can't use it to append to a sequence. In practice, you can use anchors to share common environment, but need to adopt the mapping format:

x-common-environment: &common-environment
  PORT: 8080
  HOST_SERVER: localhost
  TZ: Asia/Kolkata
  PROTOCOL: http

x-keycloak-service: &keycloak-service
  - KEYCLOAK_SERVICE=keycloak

services:
  nginx_proxy:
    ...
    environment:
      <<: *common-environment
      DOLLAR: $$
  nodered:
    ...
    environment:
      <<: [ *common-environment, *keycloak-service ]
      SERVICE_NAME: nodered