I get this error
loghttp.PushRequest.Streams: []*loghttp.Stream: unmarshalerDecoder: Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol, error found in #10 byte of ...| ]
}
]
}|..., bigger context ...|
}
}
]
}
]
}|...
when uploading the json
{
"streams":[
{
"stream":{
"application":"fabric-sso",
"job":"aws-lambda",
"level":"info",
"namespace":"oauth"
},
"values":[
{
"ts":"2022-12-10T01:36:44.971933+05:30",
"message":{
"type":"h2",
"timestamp":"2022-09-17T11:00:03.828554Z",
"alb":"app/fabric-sso/sdjsjhdjhdshksdhf",
"client_ip":"999.999.999.999",
"client_port":"7392",
"backend_ip":"",
"backend_port":"",
"request_processing_time":"-1",
"backend_processing_time":"-1",
"response_processing_time":"-1",
"alb_status_code":"404",
"backend_status_code":"-",
"received_bytes":"706",
"sent_bytes":"67",
"request_verb":"GET",
"request_url":"https://foozy.dev.gabbar.com:443/gabbar",
"request_proto":"HTTP/2.0",
"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
"ssl_cipher":"ECDHE-RSA-AES128-GCM-SHA256",
"ssl_protocol":"TLSv1.2",
"target_group_arn":"-",
"trace_id":"Root=1-6325a8b3-1980ccbd244b83c35ec5b543",
"domain_name":"foozy.dev.gabbar.com",
"chosen_cert_arn":"arn:aws:acm:us-east-1:23232323:certificate/0000",
"matched_rule_priority":"0",
"request_creation_time":"2022-09-17T11:00:03.818000Z",
"actions_executed":"waf,fixed-response",
"redirect_url":"-",
"lambda_error_reason":"-",
"target_port_list":"-",
"target_status_code_list":"-",
"classification":"-",
"classification_reason":"-",
"application":"fabric-sso",
"env":"dev"
}
}
]
}
]
}
alb_log_data is a dictionary, I create a list of dictionaries with timestamp and the actual alb log message (as json) and post it like so
def build_loki_request_payload(alb_log_data):
entries = []
for entry in alb_log_data:
curr_datetime = datetime.datetime.now(pytz.timezone('Asia/Kolkata'))
curr_datetime = curr_datetime.isoformat('T')
entries.append({'ts': curr_datetime, 'message': entry})
payload = {
'streams': [{
"stream": {
"application": alb_log_data[0]['application'],
"job": "aws-lambda",
"level": "info",
"namespace": "oauth"
},
"values": entries
}]
}
payload = json.dumps(payload)
logger.debug('Created Payload %s', payload)
return payload
I missed the right format, as in the example in Loki documentation
https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
{
"streams": [
{
"stream": {
"label": "value"
},
"values": [
[ "<unix epoch in nanoseconds>", "<log line>" ],
[ "<unix epoch in nanoseconds>", "<log line>" ]
]
}
]
}
changed the method to
def build_loki_request_payload(alb_log_data):
entries = []
for entry in alb_log_data:
entries.append([time.time_ns(), json.dumps(entry)])
payload = {
'streams': [{
"stream": {
"application": alb_log_data[0]['application'],
"job": "aws-lambda",
"level": "info",
"namespace": "oauth"
},
"values": entries
}]
}
payload = json.dumps(payload)
logger.debug('Created Payload %s', payload)
return payload
which now gives
{
"streams":[
{
"stream":{
"application":"fabric-sso",
"job":"aws-lambda",
"level":"info",
"namespace":"oauth"
},
"values":[
[
"1670645653278432000",
"{\"type\": \"h2\", \"timestamp\": \"2022-09-17T11:00:03.828554Z\", \"alb\": \"app/fabric-sso/123456\", \"client_ip\": \"230.82.1.129\", \"client_port\": \"7392\", \"backend_ip\": \"\", \"backend_port\": \"\", \"request_processing_time\": \"-1\", \"backend_processing_time\": \"-1\", \"response_processing_time\": \"-1\", \"alb_status_code\": \"404\", \"backend_status_code\": \"-\", \"received_bytes\": \"706\", \"sent_bytes\": \"67\", \"request_verb\": \"GET\", \"request_url\": \"https://foozy.dev.gabbar.com:443/gabbar\", \"request_proto\": \"HTTP/2.0\", \"user_agent\": \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36\", \"ssl_cipher\": \"ECDHE-RSA-AES128-GCM-SHA256\", \"ssl_protocol\": \"TLSv1.2\", \"target_group_arn\": \"-\", \"trace_id\": \"Root=1-6325a8b3-1980ccbd244b83c35ec5b543\", \"domain_name\": \"foozy.dev.gabbar.com\", \"chosen_cert_arn\": \"arn:aws:acm:us-east-1:3246283467784628:certificate/814e-d1e61e9e7f9b\", \"matched_rule_priority\": \"0\", \"request_creation_time\": \"2022-09-17T11:00:03.818000Z\", \"actions_executed\": \"waf,fixed-response\", \"redirect_url\": \"-\", \"lambda_error_reason\": \"-\", \"target_port_list\": \"-\", \"target_status_code_list\": \"-\", \"classification\": \"-\", \"classification_reason\": \"-\", \"application\": \"fabric-sso\", \"env\": \"dev\"}"
]
]
}
]
}