Search code examples
pythongrafana-loki

Loki throws unmarshalerDecoder error for json payload


I get this error

loghttp.PushRequest.Streams: []*loghttp.Stream: unmarshalerDecoder: Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol, error found in #10 byte of ...| ]
      }
   ]
}|..., bigger context ...|
               }
            }
         ]
      }
   ]
}|...

when uploading the json

{
   "streams":[
      {
         "stream":{
            "application":"fabric-sso",
            "job":"aws-lambda",
            "level":"info",
            "namespace":"oauth"
         },
         "values":[
            {
               "ts":"2022-12-10T01:36:44.971933+05:30",
               "message":{
                  "type":"h2",
                  "timestamp":"2022-09-17T11:00:03.828554Z",
                  "alb":"app/fabric-sso/sdjsjhdjhdshksdhf",
                  "client_ip":"999.999.999.999",
                  "client_port":"7392",
                  "backend_ip":"",
                  "backend_port":"",
                  "request_processing_time":"-1",
                  "backend_processing_time":"-1",
                  "response_processing_time":"-1",
                  "alb_status_code":"404",
                  "backend_status_code":"-",
                  "received_bytes":"706",
                  "sent_bytes":"67",
                  "request_verb":"GET",
                  "request_url":"https://foozy.dev.gabbar.com:443/gabbar",
                  "request_proto":"HTTP/2.0",
                  "user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
                  "ssl_cipher":"ECDHE-RSA-AES128-GCM-SHA256",
                  "ssl_protocol":"TLSv1.2",
                  "target_group_arn":"-",
                  "trace_id":"Root=1-6325a8b3-1980ccbd244b83c35ec5b543",
                  "domain_name":"foozy.dev.gabbar.com",
                  "chosen_cert_arn":"arn:aws:acm:us-east-1:23232323:certificate/0000",
                  "matched_rule_priority":"0",
                  "request_creation_time":"2022-09-17T11:00:03.818000Z",
                  "actions_executed":"waf,fixed-response",
                  "redirect_url":"-",
                  "lambda_error_reason":"-",
                  "target_port_list":"-",
                  "target_status_code_list":"-",
                  "classification":"-",
                  "classification_reason":"-",
                  "application":"fabric-sso",
                  "env":"dev"
               }
            }
         ]
      }
   ]
}

alb_log_data is a dictionary, I create a list of dictionaries with timestamp and the actual alb log message (as json) and post it like so

def build_loki_request_payload(alb_log_data):
    entries = []
    for entry in alb_log_data:
        curr_datetime = datetime.datetime.now(pytz.timezone('Asia/Kolkata'))
        curr_datetime = curr_datetime.isoformat('T')
        entries.append({'ts': curr_datetime, 'message': entry})

    payload = {
        'streams': [{
            "stream": {
                "application": alb_log_data[0]['application'],
                "job": "aws-lambda",
                "level": "info",
                "namespace": "oauth"
            },
            "values": entries
        }]
    }
    payload = json.dumps(payload)
    logger.debug('Created Payload %s', payload)
    return payload

Solution

  • I missed the right format, as in the example in Loki documentation

    https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki

    {
      "streams": [
        {
          "stream": {
            "label": "value"
          },
          "values": [
              [ "<unix epoch in nanoseconds>", "<log line>" ],
              [ "<unix epoch in nanoseconds>", "<log line>" ]
          ]
        }
      ]
    }
    

    changed the method to

    def build_loki_request_payload(alb_log_data):
        entries = []
        for entry in alb_log_data:
            entries.append([time.time_ns(), json.dumps(entry)])
    
        payload = {
            'streams': [{
                "stream": {
                    "application": alb_log_data[0]['application'],
                    "job": "aws-lambda",
                    "level": "info",
                    "namespace": "oauth"
                },
                "values": entries
            }]
        }
        payload = json.dumps(payload)
        logger.debug('Created Payload %s', payload)
        return payload
    

    which now gives

    {
       "streams":[
          {
             "stream":{
                "application":"fabric-sso",
                "job":"aws-lambda",
                "level":"info",
                "namespace":"oauth"
             },
             "values":[
                [
                   "1670645653278432000",
                   "{\"type\": \"h2\", \"timestamp\": \"2022-09-17T11:00:03.828554Z\", \"alb\": \"app/fabric-sso/123456\", \"client_ip\": \"230.82.1.129\", \"client_port\": \"7392\", \"backend_ip\": \"\", \"backend_port\": \"\", \"request_processing_time\": \"-1\", \"backend_processing_time\": \"-1\", \"response_processing_time\": \"-1\", \"alb_status_code\": \"404\", \"backend_status_code\": \"-\", \"received_bytes\": \"706\", \"sent_bytes\": \"67\", \"request_verb\": \"GET\", \"request_url\": \"https://foozy.dev.gabbar.com:443/gabbar\", \"request_proto\": \"HTTP/2.0\", \"user_agent\": \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36\", \"ssl_cipher\": \"ECDHE-RSA-AES128-GCM-SHA256\", \"ssl_protocol\": \"TLSv1.2\", \"target_group_arn\": \"-\", \"trace_id\": \"Root=1-6325a8b3-1980ccbd244b83c35ec5b543\", \"domain_name\": \"foozy.dev.gabbar.com\", \"chosen_cert_arn\": \"arn:aws:acm:us-east-1:3246283467784628:certificate/814e-d1e61e9e7f9b\", \"matched_rule_priority\": \"0\", \"request_creation_time\": \"2022-09-17T11:00:03.818000Z\", \"actions_executed\": \"waf,fixed-response\", \"redirect_url\": \"-\", \"lambda_error_reason\": \"-\", \"target_port_list\": \"-\", \"target_status_code_list\": \"-\", \"classification\": \"-\", \"classification_reason\": \"-\", \"application\": \"fabric-sso\", \"env\": \"dev\"}"
                ]
             ]
          }
       ]
    }