Vault scaling using Consul

Question

I have the following configuration

#vault1-config.hcl

storage "consul" {
  address = "http://consul1:8500"
  path    = "vault/"
}

listener "tcp" {
  address     = "0.0.0.0:8200"
  tls_disable = 1
}

service_registration "consul" {
  address = "vault1"
}

api_addr = "http://vault1:8200"
cluster_addr = "http://vault1:8201"

disable_mlock = true
ui = true

#vault2-config.hcl

storage "consul" {
  address = "http://consul1:8500"
  path    = "vault/"
}

listener "tcp" {
  address     = "0.0.0.0:8200"
  tls_disable = 1
}

service_registration "consul" {
  address = "vault2"
}

api_addr = "http://vault2:8200"
cluster_addr = "http://vault2:8201"

disable_mlock = true
ui = true

#init-vault.sh

#!/bin/sh

# Wait for Vault to be ready
while ! nc -z vault1 8200; do
  sleep 1
done

# Function to initialize Vault
init() {
  echo "Initializing Vault..."
  vault operator init > /vault/data/init.txt
}

# Function to unseal Vault
unseal() {
  echo "Unsealing Vault..."
  # Print unseal keys and root token
  echo "Unseal keys and root token:"
  cat /vault/data/init.txt

  vault operator unseal $(grep 'Unseal Key 1:' /vault/data/init.txt | awk '{print $NF}')
  vault operator unseal $(grep 'Unseal Key 2:' /vault/data/init.txt | awk '{print $NF}')
  vault operator unseal $(grep 'Unseal Key 3:' /vault/data/init.txt | awk '{print $NF}')
}

# Function to log in with the root token
log_in() {
  VAULT_TOKEN=$(grep 'Initial Root Token:' /vault/data/init.txt | awk '{print $NF}')
  export VAULT_TOKEN
  vault login $VAULT_TOKEN
}

# Function to create the default token
create_default_token() {
  if [ ! -z "$VAULT_DEFAULT_TOKEN" ]; then
    echo "Creating default token..."
    vault token create -id $VAULT_DEFAULT_TOKEN
  fi
}

# Function to enable secrets engines and set up policies
enable_secrets() {
  echo "Enabling secrets engines and setting up policies..."
  # Enable the KV secrets engine for the 'secret/' path (using v2, which is compatible with Spring Boot)
  vault secrets enable -path=secret kv-v2

  # Write the policies
  vault policy write user-policy /vault/config/user-policy.hcl
  vault policy write kv-access /vault/config/kv-access.hcl
  vault policy write list-secrets-engines /vault/config/list-secrets-engines.hcl

  # Enable userpass authentication and create a user
  vault auth enable userpass
  vault write auth/userpass/users/$VAULT_USER password=$VAULT_PASSWORD policies=default,list-secrets-engines,kv-access,user-policy
}

# Function to join a standby node to the active cluster
join_standby() {
  echo "Joining standby node to the active cluster..."
  vault operator raft join http://vault1:8200
}

# Initialize Vault only if it has not been initialized already
if vault status | grep -q 'Initialized.*false'; then
  init
  unseal
  log_in
  create_default_token
  enable_secrets
else
  unseal
  log_in
  if vault status | grep -q 'Standby.*true'; then
    join_standby
  fi
fi

vault status > /vault/file/status

#docker-compose.yml

version: '3.7'

services:
  consul1:
    image: hashicorp/consul:latest
    container_name: consul1
    ports:
      - "8500:8500"
    command: agent -server -bootstrap-expect=3 -ui -client=0.0.0.0 -retry-join=consul2 -retry-join=consul3
    networks:
      - consul_net

  consul2:
    image: hashicorp/consul:latest
    container_name: consul2
    ports:
      - "8501:8500"
    command: agent -server -ui -client=0.0.0.0 -retry-join=consul1 -retry-join=consul3
    depends_on:
      - consul1
    networks:
      - consul_net

  consul3:
    image: hashicorp/consul:latest
    container_name: consul3
    ports:
      - "8502:8500"
    command: agent -server -ui -client=0.0.0.0 -retry-join=consul1 -retry-join=consul2
    depends_on:
      - consul1
      - consul2
    networks:
      - consul_net

  vault1:
    image: hashicorp/vault:latest
    container_name: vault1
    environment:
      VAULT_ADDR: http://0.0.0.0:8200
    volumes:
      - vault-data:/vault/data
      - ./vault-config:/vault/config
    ports:
      - "8200:8200"
    entrypoint: ["vault", "server", "-config=/vault/config/vault1-config.hcl"]
    cap_add:
      - IPC_LOCK
    depends_on:
      - consul1
      - consul2
      - consul3
    networks:
      - consul_net

  vault2:
    image: hashicorp/vault:latest
    container_name: vault2
    environment:
      VAULT_ADDR: http://0.0.0.0:8200
    volumes:
      - vault-data2:/vault/data
      - ./vault-config:/vault/config
    ports:
      - "8201:8200"
    entrypoint: ["vault", "server", "-config=/vault/config/vault2-config.hcl"]
    cap_add:
      - IPC_LOCK
    depends_on:
      - consul1
      - consul2
      - consul3
    networks:
      - consul_net

  vault-init:
    image: hashicorp/vault:latest
    container_name: vault-init
    depends_on:
      - vault1
      - vault2
    entrypoint: ["/vault/config/init-vault.sh"]
    environment:
      VAULT_ADDR: http://vault1:8200
      VAULT_USER: ${VAULT_USER}
      VAULT_PASSWORD: ${VAULT_PASSWORD}
      VAULT_DEFAULT_TOKEN: ${VAULT_DEFAULT_TOKEN}
    volumes:
      - ./vault-config:/vault/config
      - vault-data:/vault/data
      - vault-data2:/vault/data2
    networks:
      - consul_net

networks:
  consul_net:
    driver: bridge

volumes:
  vault-data:
  vault-data2:

Vault instances are able to share information using consul storage, and consul works as cluster, they defined a leader and the followers. But, if I stop the container consul1, which is used as storage address for each vault instance; they, (vault1 and vault2) stop immediately. I tried to define multiple containers as address for vault instances:

storage "consul" {
  address = "http://consul1:8500,http://consul2:8500,http://consul3:8500"
  path    = "vault/"
}

but in that case they doesn't start. What Am I doing wrong?

Answer 1

I solved, my solution is available in https://github.com/eduard2diaz/testing-hashicorp/tree/main/vault/consul/consul-replica-vaultproxy