Search code examples
python-3.xlinuxjqaws-cli

Syntax error with os module while using pipe delimeter with jq commnad


I am using aws-cli command within python os module, while using jq and output redirection i am getting an error .

Error:

SyntaxError: invalid syntax

code:

iport os
os.system('aws ce get-cost-and-usage --profile dev --time-period Start=2022-10-01,End=2022-10-31 + 1 month  - 1 second" -I) --granularity MONTHLY --metrics USAGE_QUANTITY BLENDED_COST  --group-by Type=DIMENSION,Key=SERVICE | /bin/jq  '[ .ResultsByTime[].Groups[] | select(.Metrics.BlendedCost.Amount > "0") | { (.Keys[0]): .Metrics.BlendedCost } ] | sort_by(.Amount)| add' > /tmp/cost.json')

Is there a way to fix it, please let me know.


Solution

  • As @glenn jackman mentioned in the comment section.. you can use ''' to escape.

    I am using below code to send an attachment and HTML view over the e-mail.

    #!/usr/bin/env python3
    from __future__ import print_function
    from signal import signal, SIGPIPE, SIG_DFL
    signal(SIGPIPE,SIG_DFL)
    import os
    import time
    import io
    import smtplib
    import pandas as pd
    from email.message import EmailMessage
    from email.mime.application import MIMEApplication
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
    
    #################################################################
    pd.set_option('display.max_rows', None)
    pd.set_option('display.max_columns', None)
    pd.set_option('display.width', None)
    pd.set_option('expand_frame_repr', True)
    #################################################################
    
    # Create message container - the correct MIME type is msg/alternative.
    mail_server = 'mail.example.nxp.com'
    from_addr   = 'someEmailAddressa@example.com'
    to_addr     = ', '.join(['karn.kumar@example.com'])
    cc_addr     = ', '.join(['otherAddress@example.com'])
    subject     = 'Devlopment Cost Report Seoul'
    
    # Create mail body as template to be send
    EMAIL_TEMPLATE = """\
    <html>
      <head>
      <style>
      table, th, td {{font-size:9pt; border:1px solid black; border-collapse:collapse; text-align:left; background-color:LightGray;}}
      th, td {{padding: 5px;}}
      </style>
      </head>
      <body>
         Dear All,<br><br>
    
         Please Find the attached CSV file attached along with HTML view for the diffrent AW Service cost . <br><br>
    
         {} <br><br>
        Kind regards.<br>
        Karn
      </body>
    </html>"""
    # collect the data into Json and save it into a file under /tmp
    os.system('''aws ce get-cost-and-usage --profile dev --time-period Start=$(date "+%Y-%m-01"),End=$(date --date="$(date +'%Y-%m-01') + 1 month  - 1 second" -I) --granularity MONTHLY --metrics USAGE_QUANTITY BLENDED_COST  --group-by Type=DIMENSION,Key=SERVICE | jq -r '[ .ResultsByTime[].Groups[] | select(.Metrics.BlendedCost.Amount > "0") | { (.Keys[0]): .Metrics.BlendedCost } ] | sort_by(.Amount)| add'  > /tmp/cost.json''')
    
    # Once the data is created now its easy to extract using the pandas.
    data = pd.read_json("/tmp/cost.json")
    df = (data.T)
    df = df.drop('Unit', axis=1)
    df.rename(columns={'Amount': 'Amount(USD)'}, inplace=True)
    
    
    def df_to_csv(df):
        with io.StringIO() as buffer:
            df.to_csv(buffer)
            return buffer.getvalue()
    
    def send_email():
        msg = MIMEMultipart()
        msg['Subject'] = subject
        msg['From']    = from_addr
        msg['To']      = to_addr
        msg['Cc']      = cc_addr
        attachment = MIMEApplication(df_to_csv(df))
        attachment['Content-Disposition'] = 'attachment; filename="cost-details.csv"'
        msg.attach(attachment)
        msg.attach(MIMEText(EMAIL_TEMPLATE.format(df.to_html(index=True)), 'html'))
        server = smtplib.SMTP(mail_server)
        server.sendmail(to_addr, cc_addr, msg.as_string())
        server.quit()
    
    if __name__ == '__main__':
        send_email()
    
    # remove the data file from "/tmp" after data parsing.
    time.sleep(30)
    os.unlink("/tmp/cost.json")
    

    Result is Blow: enter image description here