Search code examples
pythonjsonpandascsvextract

Extract specific JSON keys and convert to CSV in Python


I'm converting several JSON files into a CSV using the following code below, it works as intended, but it converts all of the data in the JSON file. Instead, I want it to do the following:

  1. Load JSON file [done]
  2. Extract certain nested data in the JSON file [wip]
  3. Convert to CSV [done]

Current Code

import json, pandas
from flatten_json import flatten
# Enter the path to the JSON and the filename without appending '.json'
file_path = r'C:\Path\To\file_name'
# Open and load the JSON file
dic = json.load(open(file_path + '.json', 'r', encoding='utf-8', errors='ignore'))
# Flatten and convert to a data frame
dic_flattened = (flatten(d, '.') for d in dic)
df = pandas.DataFrame(dic_flattened)
# Export to CSV in the same directory with the original file name
export_csv = df.to_csv (file_path + r'.csv', sep=',', encoding='utf-8', index=None, header=True)

In the example at the bottom, I only want everything under the following keys: created, emails, and identities. The rest is useless information (such as statusCode) or it's duplicated under a different key name (such as profile and userInfo).

I know it requires a for loop and if statement to specify the key names later on, but not sure the best way to implement it. This is what I have so far when I want to test it:

Attempted Code

import json, pandas
from flatten_json import flatten
# Enter the path to the JSON and the filename without appending '.json'
file_path = r'C:\Path\To\file_name'
# Open and load the JSON file
json_file = open(file_path + '.json', 'r', encoding='utf-8', errors='ignore')
dic = json.load(json_file)
# List keys to extract
key_list = ['created', 'emails', 'identities']
for d in dic:
    #print(d['identities']) #Print all 'identities'
    #if 'identities' in d: #Check if 'identities' exists
    if key_list in d:
        # Flatten and convert to a data frame
        #dic_flattened = (flatten(d, '.') for d in dic)
        #df = pandas.DataFrame(dic_flattened)
    else:
        # Skip
# Export to CSV in the same directory with the original file name
        #export_csv = df.to_csv (file_path + r'.csv', sep=',', encoding='utf-8', index=None, header=True)

Is this the right logic?

file_name.json Example

[
    {
        "callId": "abc123",
        "errorCode": 0,
        "apiVersion": 2,
        "statusCode": 200,
        "statusReason": "OK",
        "time": "2020-12-14T12:00:32.744Z",
        "registeredTimestamp": 1417731582000,
        "UID": "_guid_abc123==",
        "created": "2014-12-04T22:19:42.894Z",
        "createdTimestamp": 1417731582000,
        "data": {},
        "preferences": {},
        "emails": {
            "verified": [],
            "unverified": []
        },
        "identities": [
            {
                "provider": "facebook",
                "providerUID": "123",
                "allowsLogin": true,
                "isLoginIdentity": true,
                "isExpiredSession": true,
                "lastUpdated": "2014-12-04T22:26:37.002Z",
                "lastUpdatedTimestamp": 1417731997002,
                "oldestDataUpdated": "2014-12-04T22:26:37.002Z",
                "oldestDataUpdatedTimestamp": 1417731997002,
                "firstName": "John",
                "lastName": "Doe",
                "nickname": "John Doe",
                "profileURL": "https://www.facebook.com/John.Doe",
                "age": 30,
                "birthDay": 31,
                "birthMonth": 12,
                "birthYear": 1969,
                "city": "City, State",
                "education": [
                    {
                        "school": "High School Name",
                        "schoolType": "High School",
                        "degree": null,
                        "startYear": 0,
                        "fieldOfStudy": null,
                        "endYear": 0
                    }
                ],
                "educationLevel": "High School",
                "followersCount": 0,
                "gender": "m",
                "hometown": "City, State",
                "languages": "English",
                "locale": "en_US",
                "name": "John Doe",
                "photoURL": "https://graph.facebook.com/123/picture?type=large",
                "timezone": "-8",
                "thumbnailURL": "https://graph.facebook.com/123/picture?type=square",
                "username": "john.doe",
                "verified": "true",
                "work": [
                    {
                        "companyID": null,
                        "isCurrent": null,
                        "endDate": null,
                        "company": "Company Name",
                        "industry": null,
                        "title": "Company Title",
                        "companySize": null,
                        "startDate": "2010-12-31T00:00:00"
                    }
                ]
            }
        ],
        "isActive": true,
        "isLockedOut": false,
        "isRegistered": true,
        "isVerified": false,
        "lastLogin": "2014-12-04T22:26:33.002Z",
        "lastLoginTimestamp": 1417731993000,
        "lastUpdated": "2014-12-04T22:19:42.769Z",
        "lastUpdatedTimestamp": 1417731582769,
        "loginProvider": "facebook",
        "loginIDs": {
            "emails": [],
            "unverifiedEmails": []
        },
        "rbaPolicy": {
            "riskPolicyLocked": false
        },
        "oldestDataUpdated": "2014-12-04T22:19:42.894Z",
        "oldestDataUpdatedTimestamp": 1417731582894
        "registered": "2014-12-04T22:19:42.956Z",
        "regSource": "",
        "socialProviders": "facebook"
    }
]

Solution

  • As mentioned by juanpa.arrivillaga, I simply need to add the following line after the key_list:

    json_list = [{k:d[k] for k in key_list} for d in json_list]
    

    This is the full working code:

    import json, pandas
    from flatten_json import flatten
    
    # Enter the path to the JSON and the filename without appending '.json'
    file_path = r'C:\Path\To\file_name'
    
    # Open and load the JSON file
    json_list = json.load(open(file_path + '.json', 'r', encoding='utf-8', errors='ignore'))
    
    # Extract data from the defined key names
    key_list = ['created', 'emails', 'identities']
    json_list = [{k:d[k] for k in key_list} for d in json_list]
    
    # Flatten and convert to a data frame
    json_list_flattened = (flatten(d, '.') for d in json_list)
    df = pandas.DataFrame(json_list_flattened)
    
    # Export to CSV in the same directory with the original file name
    export_csv = df.to_csv (file_path + r'.csv', sep=',', encoding='utf-8', index=None, header=True)