Trying to flatten a Reddit JSON into many "conversations"

Question

I am trying to use the comments of Reddit threads as a training set for a machine learning program. An example of an input would be https://old.reddit.com/r/bayarea/comments/cxxl9y/billionaires_yacht_docked_in_embarcadero.json.

I am filtering out the body, id, and parent_id in hopes of turning the nested JSON into many conversations.

For example, if the input is ["A", ["B",["C", "D"]]], the output should be ["A", "B", "C"], ["A","B","D"].

Below is my current code:

json_url = "https://old.reddit.com/r/bayarea/comments/cxxl9y/billionaires_yacht_docked_in_embarcadero.json"
            r = requests.get(json_url, headers={"user-agent": "PostmanRuntime/7.15.2"})

            comments_tree_raw = fltr(r.json(), ["ups", "body", "id", "parent_id"])[1]["data"]

            comments_tree_raw = flatten([], comments_tree_raw["children"])

def remove_all_after(node, index):
    target = node.index(index)
    return node[:target]




training_threads = []
# input the children list
def flatten(output, children):
    global training_threads


    for child in children:
        try:
            child_obj = child["data"] if "body" in child["data"] else child
            child_comment = {
                "body": child_obj["body"],
                "id": child_obj["id"],
                "parent": child_obj["parent_id"]
            }
            output.append(child_comment)
        except KeyError:
            continue

        if "replies" not in child["data"]:

            training_threads.append(output.copy())

            parent_id = child_comment["parent"].split("_")[1]
            for i in output:
                if i["id"] == parent_id:
                    output = remove_all_after(output, i)
                    break


            continue

        flatten(output, child["data"]["replies"]["data"]["children"])

Here, I am trying to solve the problem recursively, but it isn't producing the output I need. This is the output I am getting: https://pastebin.com/GkpwGUtK.

Would greatly appreciate the help! Thanks!

Answer 1

You can use simple recursion with a generator:

data = ["A", ["B",["C", "D"]]]
def group(d, c = []):
   a, b = d
   if all(not isinstance(i, list) for i in b):
     yield from [c+[a, i] for i in b]
   else:
     yield from group(b, c+[a])

print(list(group(data)))

Output:

[['A', 'B', 'C'], ['A', 'B', 'D']]

---

Edit: more complete version using itertools.groupby:

from itertools import groupby
def group(d, c = []):
  _d = [list(b) for _, b in groupby(d, key=lambda x:isinstance(x, list))]
  if len(_d) == 1:
    for i in _d[0]:
      if not isinstance(i, list):
         yield c+[i]
      else:
         yield from group(i, c)
  else:
     for i in range(0, len(_d), 2):
       for k in _d[i]:
         yield from group(_d[i+1], c+[k])

print(list(group([["C", ["D", "E"], ["C", ["D", "E"], ["C", ["D", "E"]]]]])))

Output:

[['C', 'D'], ['C', 'E'], ['C', 'C', 'D'], ['C', 'C', 'E'], ['C', 'C', 'C', 'D'], ['C', 'C', 'C', 'E']]