Python requests - pull from S3 to multipart Type Error

Question

Hi trying to pull a file from S3 and ingest to a multipart post endpoint on another service, code looks something like this

def execute(event, context):
    if "emailId" in event:
        props = {
            "fieldOne": "someValue",
            "fieldTwo": "someOtherValue",
            "contentType": "eml"
        }
        props = json.dumps(props)
        file = download_s3_file_to_tmp(event["emailId"])
        multipart_request(props, file, event["emailId"])


def download_s3_file_to_tmp(message_id):
    s3_key = "".join([_email_bucket_folder_name, "/", str(message_id), ".eml"])
    s3_client = boto3.client("s3", _region)
    s3_client.download_file(_my_bucket, s3_key, "/tmp/"+str(message_id)+".eml")
    downloaded_eml_file = open("/tmp/"+str(message_id)+".eml", 'rb')
    return downloaded_eml_file

def multipart_request(props, file_content, message_id):
    my_id = *****
    secret = *****
    url = f"{_url}/...."
    payload = {"props": props}
    files = [{"fileContent", file_content}]
    tmp_file_path = "/tmp/"+message_id+".eml"
    if os.path.exists(tmp_file_path):
        os.remove(tmp_file_path)
        print("Removed the file %s" % tmp_file_path)     
    else:
        print("File %s does not exist." % tmp_file_path)
    LOGGER.info(f"payload: {payload}")
    resp = requests.post(url,data=payload,files=files,auth=requests.auth.HTTPBasicAuth(my_id, secret))
    LOGGER.info(f"Request Headers: {resp.request.headers}")
    return resp.status_code, resp.text, filenote_id

The problem is(when testing with the same file from the S3 bucket so should be no inconsistencies there) that intermittently I am getting the error

  "errorMessage": "expected string or bytes-like object",
  "errorType": "TypeError",

on the requests.post call. Sometimes it is fine I am getting 200, but lot of time i am getting the above error.

here is the stack trace regarding the requests module

{
  "errorMessage": "expected string or bytes-like object",
  "errorType": "TypeError",
  "stackTrace": [
    "  File \"/var/task/my_lambda.py\", line 42, in execute\n    multipart_request(props, file, event["emailId"])\n",
    "  File \"/var/task/mime_retrieval_parser.py\", line 75, in multipart_request\n    resp = requests.post(url,data=payload,files=files,auth=requests.auth.HTTPBasicAuth(edm_id, edm_secret))\n",
    "  File \"/opt/python/requests/api.py\", line 119, in post\n    return request('post', url, data=data, json=json, **kwargs)\n",
    "  File \"/opt/python/requests/api.py\", line 61, in request\n    return session.request(method=method, url=url, **kwargs)\n",
    "  File \"/opt/python/requests/sessions.py\", line 516, in request\n    prep = self.prepare_request(req)\n",
    "  File \"/opt/python/requests/sessions.py\", line 449, in prepare_request\n    p.prepare(\n",
    "  File \"/opt/python/requests/models.py\", line 317, in prepare\n    self.prepare_body(data, files, json)\n",
    "  File \"/opt/python/requests/models.py\", line 505, in prepare_body\n    (body, content_type) = self._encode_files(files, data)\n",
    "  File \"/opt/python/requests/models.py\", line 166, in _encode_files\n    rf.make_multipart(content_type=ft)\n",
    "  File \"/opt/python/urllib3/fields.py\", line 267, in make_multipart\n    self._render_parts(\n",
    "  File \"/opt/python/urllib3/fields.py\", line 225, in _render_parts\n    parts.append(self._render_part(name, value))\n",
    "  File \"/opt/python/urllib3/fields.py\", line 205, in _render_part\n    return self.header_formatter(name, value)\n",
    "  File \"/opt/python/urllib3/fields.py\", line 116, in format_header_param_html5\n    value = _replace_multiple(value, _HTML5_REPLACEMENTS)\n",
    "  File \"/opt/python/urllib3/fields.py\", line 89, in _replace_multiple\n    result = pattern.sub(replacer, value)\n"
  ]
}

Answer 1

Solved this with the below approach

def execute(event, context):
    if "emailId" in event:
        props = {
            "fieldOne": "someValue",
            "fieldTwo": "someOtherValue",
            "contentType": "eml"
        }
        tmp_file_path = "/tmp/" + message_id + ".eml"
        props = json.dumps(props)
        file = download_s3_file_to_tmp(event["emailId"])
        multipart_request(props, file, tmp_file_path)

def download_s3_file_to_tmp(message_id, tmp_file_path):
    s3_key = "".join([_bucket_folder_name, "/", str(message_id), ".eml"])
    s3_client.download_file(
        _my_bucket, s3_key, "/tmp/" + str(message_id) + ".eml"
    )
    return open(tmp_file_path).read()

def multipart_request(props, opened_eml_file, tmp_file_path):
    my_id = *****
    secret = *****
    url = f"{_url}/...."
    payload = {"props": props}
    files = {"fileContent": ("whatEverYouWantToNameFile.eml", opened_eml_file)}

    if os.path.exists(tmp_file_path):
        os.remove(tmp_file_path)
        print("Removed the file %s" % tmp_file_path)     
    else:
        print("File %s does not exist." % tmp_file_path)

    resp = requests.post(url,data=payload,files=files,auth=requests.auth.HTTPBasicAuth(my_id, secret))
    return resp.status_code, resp.text, filenote_id