Search code examples
dockerdocker-composeyaml

how to merge compose file fragments correctly


I would like to merge the seaweed-master-defaults and deploy_test01 fragments into the seaweed_master_test01 service, see the example below:

version: "3.9"

x-service-defaults: &service-defaults
    # common defaults for all services
    deploy:
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s
      endpoint_mode: dnsrr

x-seaweeed-master-defaults: &seaweed-master-defaults
  image: chrislusf/seaweedfs
  volumes:
    - ${seaweed_dir}/master:/data
  <<: *service-defaults

x-seaweeed-volume-defaults: &seaweed-volume-defaults
  image: chrislusf/seaweedfs
  volumes:
    - ${seaweed_dir}/volume:/data
  <<: *service-defaults

x-deploy-test01: &deploy_test01
    # place instances on node01 only
    placement:
      constraints:
        - "node.labels.node==test01"

x-deploy-test02: &deploy_test02
    # place instances on node02 only
    placement:
      constraints:
        - "node.labels.node==test02"

x-deploy-test03: &deploy_test03
    # place instances on node03 only
    placement:
      constraints:
        - "node.labels.node==test03"
services:
  seaweed_master_test01:
    hostname: seaweed_master_test01
    command: 'master ${seaweed_master_opts} -mdir=/data -peers="seaweed_master_test02,seaweed_master_test03"'
    # service defaults PLUS place instances on node01 only
    <<: *seaweed-master-defaults
    deploy: *deploy_test01
  seaweed_master_test02:
    hostname: seaweed_master_test02
    command: 'master ${seaweed_master_opts} -mdir=/data -peers="seaweed_master_test01,seaweed_master_test03"'
    <<: *seaweed-master-defaults
    deploy: *deploy_test02
  seaweed_master_test03:
    hostname: seaweed_master_test03
    command: 'master ${seaweed_master_opts} -mdir=/data -peers="seaweed_master_test01,seaweed_master_test03"'
    <<: *seaweed-master-defaults
    deploy: *deploy_test03
  volume01:
    hostname: volume01
    command: 'volume -dataCenter=test01 -rack=rack1 ${seaweed_volume_opts} -dir=/data -mserver="seaweed_master_test01:9333,seaweed_master_test02:9333,seaweed_master_test03:9333"'
    <<: *seaweed-volume-defaults
    deploy: *deploy_test01
  volume02:
    hostname: volume02
    command: 'volume -dataCenter=test02 -rack=rack1 ${seaweed_volume_opts} -dir=/data -mserver="seaweed_master_test01:9333,seaweed_master_test02:9333,seaweed_master_test03:9333"'
    <<: *seaweed-volume-defaults
    deploy: *deploy_test02
  volume03:
    hostname: volume03
    command: 'volume -dataCenter=test03 -rack=rack1 ${seaweed_volume_opts} -dir=/data -mserver="seaweed_master_test01:9333,seaweed_master_test02:9333,seaweed_master_test03:9333"'
    <<: *seaweed-volume-defaults
    deploy: *deploy_test03
networks:
  default:
    driver: overlay
    attachable: true

My idea was to merge service defaults with instance placement, as follows:

    # service defaults PLUS place instances on node01 only
    <<: *seaweed-master-defaults
    deploy: *deploy_test01

However, if I try docker stack config --compose-file compose.yaml then I see that the generated file looks like this:

  seaweed_master_test01:
    command:
    - master
    - -defaultReplication=000
    - -volumeSizeLimitMB=1024
    - -mdir=/data
    - -peers=seaweed_master_test02,seaweed_master_test03
    deploy:
      placement:
        constraints:
        - node.labels.node==test01
    hostname: seaweed_master_test01
    image: chrislusf/seaweedfs
    volumes:
    - type: bind
      source: /srv/data/dev/seaweed/master
      target: /data

In other words, *deploy_test01 overwrites everything under deploy:, instead of merging it.

How can I merge the two fragments so that the result is like this:

    deploy:
      # https://docs.docker.com/compose/compose-file/compose-file-v3/#restart_policy
      restart_policy:
        condition: on-failure
        delay: 10s
        #max_attempts: 3
        window: 120s
      endpoint_mode: dnsrr
      placement:
        constraints:
          - "node.labels.node==test01"

Solution

  • As @David Maze pointed out in the comments: The YAML content gets merged at the top-level and will overwrite your settings.

    To achieve your desired behavior, one could use a combination of extensions and extends. Unfortunately, extends is not available for docker stack (see this open GitHub issue).

    Another possible solution would be to use extensions on each separate level like:

    version: "3.9"
    
    x-restart-policy: &restart-policy
      condition: on-failure
      delay: 10s
      max_attempts: 3
      window: 120s
    
    x-endpoint-mode: &endpoint-mode
      endpoint_mode: dnsrr
    
    x-seaweeed-master-defaults: &seaweed-master-defaults
      image: chrislusf/seaweedfs
      volumes:
        - ${seaweed_dir}/master:/data
    
    x-deploy-test01: &deploy_test01
        # place instances on node01 only
          constraints: [node.labels.node==test01]
    
    services:
      seaweed_master_test01:
        hostname: seaweed_master_test01
        command: 'master ${seaweed_master_opts} -mdir=/data -peers="seaweed_master_test02,seaweed_master_test03"'
        # service defaults PLUS place instances on node01 only
        <<: *seaweed-master-defaults
        deploy:
          <<: *endpoint-mode
          restart_policy:
            <<: *restart-policy
          placement:
            <<: *deploy_test01
    

    output:

    version: "3.9"
    services:
      seaweed_master_test01:
        command:
        - master
        - -mdir=/data
        - -peers=seaweed_master_test02,seaweed_master_test03
        deploy:
          restart_policy:
            condition: on-failure
            delay: 10s
            max_attempts: 3
            window: 2m0s
          placement:
            constraints:
            - node.labels.node==test01
          endpoint_mode: dnsrr
        hostname: seaweed_master_test01
        image: chrislusf/seaweedfs
        volumes:
        - type: bind
          source: /master
          target: /data