Unit test a function which downloads json file from s3

Question

I have a function given as below:

def read_json(bucket, key):
    """

    :param bucket:
    :param key:
    :return:
    """
    s3 = boto3.resource('s3')
    content_object = S3.Object(bucket, key)
    file_content = content_object.get()['Body'].read().decode('utf-8')
    json_content = json.loads(file_content)
    return json_content

I am testing this function as given below:

@mock.patch('boto3.resource')
def test_read_json(mock_resource):

    mock_resource.Object.return_value.return_value = '{ "key":"value", ' \
                                              '"key_1":value_1, ' \
                                              '"key_2":"value_2"}'
    json = helpers.read_json('bucket', 'key')
    mock_resource.assert_called_once_with('s3')
    tools.assert_is_instance(json, 'json')

TypeError: the JSON object must be str, bytes or bytearray, not MagicMock

Any ideas what I might be doing wrong over here ?

Answer 1

I changed my function slightly to achieve the same results:

def read_json(bucket, key):
    """

    :param bucket:
    :param key:
    :return:
    """
    try:
        response = s3.get_object(bucket, key)
        file_content = response.get('Body').read().decode('utf-8')
        json_content = json.loads(file_content)
    except ClientError as ex:
        LOGGER.error("Received error: %s", ex)
        raise
    return json_content

Corresponding unit tests:

@mock.patch('delivery_qa.utils.helpers.s3.get_object')
def test_read_json(s3_get_mock):
    body_mock = Mock()
    body_mock.read.return_value.decode.return_value = json.dumps('response')
    # When first time function would be called, it would return body_mock
    s3_get_mock.side_effect = [{'Body': body_mock}]
    first_call = helpers.read_json('bucket', 'key')
    tools.assert_equal(first_call, 'response')


@raises(ClientError)
@mock.patch('delivery_qa.utils.helpers.s3.get_object')
def test_read_json_exception(s3_get_mock):
    body_mock = Mock()
    body_mock.read.return_value.decode.return_value = json.dumps('response')
    # When first time function would be called, it would return ClientError
    s3_get_mock.side_effect = [ClientError(MagicMock(), MagicMock())]
    helpers.read_json('bucket', 'key')