Send email from Python when etcd cluster is down or unhealthy

Question

I am working on the etcd cluster monitoring where I have to send the email if the cluster is down. When cluster is healthy and I use the sendEmail() function in my code, it works fine, but when cluster is down/unhealthy or I have killed the process, it says:

requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=2379): Max retries exceeded with url: /health (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x1f6de50>: Failed to establish a new connection: [Errno 111] Connection refused',))

I tried to use the status code and also requests.exception so that it reaches my code, but not able to do so. Below is my code:

#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import requests
import sys
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from subprocess import Popen, PIPE

def getClusterHealth():
    response = requests.get('http://localhost:2379/health')
    data = response.json()

    if response.status_code == 111:
        sendEmail() 

    elif data['health']=="true":
        print("Cluster is healthy")

    else:
        print ("Cluster is not healthy")
        sendEmail()

def sendEmail():
    msg = MIMEText("etcd Cluster Down Sample Mail")
    sender = "example@server.com"
    recipients = ["example1@server.com,example2@servr.com"]
    msg["Subject"] = "etcd Cluster Monitoring Test Multiple ID"  
    msg['From'] = sender
    msg['To'] = ", ".join(recipients)
    s = smtplib.SMTP('localhost')
    s.sendmail(sender,recipients,msg.as_string())
    s.quit()
    #p = Popen(["/usr/sbin/sendmail", "-t", "-oi"], stdin=PIPE, universal_newlines=True)
    #p.communicate(msg.as_string())  


if __name__ == "__main__":

    if(len(sys.argv) < 2):
        print("Usage : python etcdMonitoring.py [health|metrics|all]")
    elif(sys.argv[1] == "health"):
        getClusterHealth() 

What is the possible solution for this?

Answer 1

You can catch the ConnectionError exception and evaluate the error message and send the email as required:

def getClusterHealth():
     try:
        response = requests.get('http://localhost:2379/health')
     except ConnectionError as e:
     // You can use the value of e to check for specific error message and trigger the email
       if str(e) == 'Max retries exceeded with url':
         sendEmail() 

        data = response.json()

        if response.status_code == 111:
            sendEmail() 

        elif data['health']=="true":
            print("Cluster is healthy")

        else:
            print ("Cluster is not healthy")
            sendEmail()