Search code examples
dockerelasticsearchopensearch

Running Elasticsearch and OpenSearch on Docker-compose

I have the following docker-compose.yml file:

services:
  elasticsearch:
    image: elasticsearch:7.10.1
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - http.host=0.0.0.0
      - http.port=9201
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=false
    ports:
      - "9201:9201"
      - "9300:9300"
    expose:
      - "9201"
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --fail localhost:9201/_cluster/health || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 3
    networks:
      - elastic
    command: >
      bash -c '
        until curl -sS "http://localhost:9201/_cat/health?h=status" | grep -q "green\|yellow"; do
          sleep 1
        done
        curl -X PUT "http://localhost:9201/products" -H "Content-Type: application/json" -d'
        {
          "settings": {
            "number_of_shards": 1,
            "number_of_replicas": 1,
            "analysis": {
              "analyzer": "standard"
            }
          },
          "mappings": {
            "properties": {
              "title": {
                "type": "text",
                "analyzer": "standard"
              },
              "author": {
                "type": "text",
                "analyzer": "standard"
              },
              "description": {
                "type": "text",
                "analyzer": "standard"
              },
              "published_date": {
                "type": "date"
              },
              "purchase_url": {
                "type": "keyword"
              }
            }
          }
        }'
      '
    deploy:
      resources:
        limits:
          memory: 2GB
  opensearch-node1A:
    image: opensearchproject/opensearch:1.2.3
    container_name: opensearch-node1A
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node1A
      - discovery.seed_hosts=opensearch-node1A,opensearch-node2A
      - cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
        hard: 65536
    volumes:
      - opensearch-data1:/usr/share/opensearch/data
    ports:
      - 9200:9200
      - 9600:9600 # required for Performance Analyzer
    networks:
      - opensearch-net
  opensearch-node2A:
    image: opensearchproject/opensearch:1.2.3
    container_name: opensearch-node2A
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node2A
      - discovery.seed_hosts=opensearch-node1A,opensearch-node2A
      - cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
      - bootstrap.memory_lock=true
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - opensearch-data2:/usr/share/opensearch/data
    networks:
      - opensearch-net
  opensearch-dashboardsA:
    image: opensearchproject/opensearch-dashboards:1.1.0
    container_name: opensearch-dashboardsA
    ports:
      - 5601:5601
    expose:
      - "5601"
    environment:
      OPENSEARCH_HOSTS: '["https://opensearch-node1A:9200","https://opensearch-node2A:9200"]'
    networks:
      - opensearch-net
  logstash-with-plugin:
    image: opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
    container_name: logstash-with-plugin
    networks:
      - opensearch-net

volumes:
  opensearch-data1:
  opensearch-data2:

networks:
  opensearch-net:
    driver: bridge
  elastic:
    driver: bridge

I want to do a POC about transferring data from Elasticsearch to Opensearch, but I want them running over containers with docker-compose.yml.

So I set Elasticsearch to run on port 9201 while Opensearch runs on port 9200.

However, from the logs, I see that the Elasticsearch container fails to to start:

logstash-with-plugin    | [2024-09-26T04:29:39,365][INFO ][logstash.javapipeline    ][main] Pipeline Java execution initialization time {"seconds"=>0.32}
logstash-with-plugin    | [2024-09-26T04:29:39,376][INFO ][logstash.inputs.beats    ][main] Starting input listener {:address=>"0.0.0.0:5044"}
logstash-with-plugin    | [2024-09-26T04:29:39,384][INFO ][logstash.javapipeline    ][main] Pipeline started {"pipeline.id"=>"main"}
logstash-with-plugin    | [2024-09-26T04:29:39,413][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
logstash-with-plugin    | [2024-09-26T04:29:39,418][INFO ][org.logstash.beats.Server][main][0710cad67e8f47667bc7612580d5b91f691dd8262a4187d9eca8cf87229d04aa] Starting server on port: 5044
elasticsearch           | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch           | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch           | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch           | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch           | curl: (7) Failed to connect to localhost port 9201: Connection refused

How can I have both running ?

Solution

  • In elasticsearch service section you have to remove command part and create products index manually or you can use docker compose file below:

    services:
  elasticsearch:
    image: elasticsearch:7.10.1
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - http.host=0.0.0.0
      - http.port=9201
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=false
    ports:
      - "9201:9201"
      - "9300:9300"
    expose:
      - "9201"
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --fail localhost:9201/_cluster/health || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 3
    networks:
      - elastic
    command: >
      bash -c '
        /usr/local/bin/docker-entrypoint.sh &
        until [ "$(curl -sS "http://localhost:9201/_cat/health?h=status" 2>&1 | grep "green")" = "green" ]; do
          echo "Status is not green, retrying..."
          sleep 1
        done
        echo "Status is green"
        curl -X PUT "http://localhost:9201/products" -H "Content-Type: application/json" -d "{\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":1,\"analysis\":{\"analyzer\":\"standard\"}},\"mappings\":{\"properties\":{\"title\":{\"type\":\"text\",\"analyzer\":\"standard\"},\"author\":{\"type\":\"text\",\"analyzer\":\"standard\"},\"description\":{\"type\":\"text\",\"analyzer\":\"standard\"},\"published_date\":{\"type\":\"date\"},\"purchase_url\":{\"type\":\"keyword\"}}}}"
        exec tail -f /dev/null
      '
    deploy:
      resources:
        limits:
          memory: 2GB

  opensearch-node1A:
    image: opensearchproject/opensearch:1.2.3
    container_name: opensearch-node1A
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node1A
      - discovery.seed_hosts=opensearch-node1A,opensearch-node2A
      - cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
        hard: 65536
    volumes:
      - opensearch-data1:/usr/share/opensearch/data
    ports:
      - 9200:9200
      - 9600:9600 # required for Performance Analyzer
    networks:
      - opensearch-net
  opensearch-node2A:
    image: opensearchproject/opensearch:1.2.3
    container_name: opensearch-node2A
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node2A
      - discovery.seed_hosts=opensearch-node1A,opensearch-node2A
      - cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
      - bootstrap.memory_lock=true
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - opensearch-data2:/usr/share/opensearch/data
    networks:
      - opensearch-net
  opensearch-dashboardsA:
    image: opensearchproject/opensearch-dashboards:1.1.0
    container_name: opensearch-dashboardsA
    ports:
      - 5601:5601
    expose:
      - "5601"
    environment:
      OPENSEARCH_HOSTS: '["https://opensearch-node1A:9200","https://opensearch-node2A:9200"]'
    networks:
      - opensearch-net
  logstash-with-plugin:
    image: opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
    container_name: logstash-with-plugin
    networks:
      - opensearch-net

volumes:
  opensearch-data1:
  opensearch-data2:

networks:
  opensearch-net:
    driver: bridge
  elastic:
    driver: bridge

    In the above docker compose file, I just edited the command section of the elasticsearch service to match.