Search code examples
jsonscriptinguniquejq

return each element from a list-of-dictionaries if a list-of-dictionaries contained within has a specific key or list is empty

Considering the following data structure, I'm looking to use jq to return each document based on the following criteria:

  1. Return all documents whose members array contains a key subPath
  2. Return all documents whose members array does NOT contain a key subPath
  3. Return all documents whose members array is empty
[
  {
    "alternate_mode": "global-availability",
    "dynamic_ratio": "no",
    "enabled": "yes",
    "fallback_mode": "return-to-dns",
    "full_path": "/Common/a2ws-test.ree.se.net_pool",
    "load_balancing_mode": "global-availability",
    "manual_resume": "no",
    "max_answers_returned": 1,
    "members": [
      {
        "name": "a2ws-test.ree.se.net-443-vs",
        "partition": "Common",
        "subPath": "f5-tstltm-dmz-dc1-pair:/Common",
        "enabled": "yes",
        "limitMaxBps": 0,
        "limitMaxBpsStatus": "disabled",
        "limitMaxConnections": 0,
        "limitMaxConnectionsStatus": "disabled",
        "limitMaxPps": 0,
        "limitMaxPpsStatus": "disabled",
        "monitor": "default",
        "ratio": 1,
        "disabled": "no",
        "member_order": 0
      },
      {
        "name": "dc2-a2ws-test.ree.se.net-443-vs",
        "partition": "Common",
        "subPath": "f5-tstltm-dmz-dc2-pair:/Common",
        "enabled": "yes",
        "limitMaxBps": 0,
        "limitMaxBpsStatus": "disabled",
        "limitMaxConnections": 0,
        "limitMaxConnectionsStatus": "disabled",
        "limitMaxPps": 0,
        "limitMaxPpsStatus": "disabled",
        "monitor": "default",
        "ratio": 1,
        "disabled": "no",
        "member_order": 1
      }
    ],
    "name": "a2ws-test.ree.se.net_pool",
    "partition": "Common",
    "qos_hit_ratio": 5,
    "qos_hops": 0,
    "qos_kilobytes_second": 3,
    "qos_lcs": 30,
    "qos_packet_rate": 1,
    "qos_rtt": 50,
    "qos_topology": 0,
    "qos_vs_capacity": 0,
    "qos_vs_score": 0,
    "ttl": 30,
    "verify_member_availability": "yes"
  },
  {
    "alternate_mode": "round-robin",
    "dynamic_ratio": "no",
    "enabled": "yes",
    "fallback_mode": "return-to-dns",
    "full_path": "/Common/aci-apic.ree.se.net_pool",
    "load_balancing_mode": "round-robin",
    "manual_resume": "no",
    "max_answers_returned": 1,
    "members": [
      {
        "name": "prd_dc1_servers:aci-apic01.ree.se.net",
        "partition": "Common",
        "enabled": "yes",
        "limitMaxBps": 0,
        "limitMaxBpsStatus": "disabled",
        "limitMaxConnections": 0,
        "limitMaxConnectionsStatus": "disabled",
        "limitMaxPps": 0,
        "limitMaxPpsStatus": "disabled",
        "monitor": "default",
        "ratio": 1,
        "disabled": "no",
        "member_order": 0
      },
      {
        "name": "prd_dc1_servers:aci-apic02.ree.se.net",
        "partition": "Common",
        "enabled": "yes",
        "limitMaxBps": 0,
        "limitMaxBpsStatus": "disabled",
        "limitMaxConnections": 0,
        "limitMaxConnectionsStatus": "disabled",
        "limitMaxPps": 0,
        "limitMaxPpsStatus": "disabled",
        "monitor": "default",
        "ratio": 1,
        "disabled": "no",
        "member_order": 1
      },
      {
        "name": "prd_dc2_servers:aci-apic03.ree.se.net",
        "partition": "Common",
        "enabled": "yes",
        "limitMaxBps": 0,
        "limitMaxBpsStatus": "disabled",
        "limitMaxConnections": 0,
        "limitMaxConnectionsStatus": "disabled",
        "limitMaxPps": 0,
        "limitMaxPpsStatus": "disabled",
        "monitor": "default",
        "ratio": 1,
        "disabled": "no",
        "member_order": 2
      }
    ],
    "name": "aci-apic.ree.se.net_pool",
    "partition": "Common",
    "qos_hit_ratio": 5,
    "qos_hops": 0,
    "qos_kilobytes_second": 3,
    "qos_lcs": 30,
    "qos_packet_rate": 1,
    "qos_rtt": 50,
    "qos_topology": 0,
    "qos_vs_capacity": 0,
    "qos_vs_score": 0,
    "ttl": 30,
    "verify_member_availability": "yes"
  },
  {
    "alternate_mode": "global-availability",
    "dynamic_ratio": "no",
    "enabled": "yes",
    "fallback_mode": "return-to-dns",
    "full_path": "/Common/b2b.ree.se.net_pool",
    "load_balancing_mode": "global-availability",
    "manual_resume": "no",
    "max_answers_returned": 1,
    "members": [],
    "name": "b2b.ree.se.net_pool",
    "partition": "Common",
    "qos_hit_ratio": 5,
    "qos_hops": 0,
    "qos_kilobytes_second": 3,
    "qos_lcs": 30,
    "qos_packet_rate": 1,
    "qos_rtt": 50,
    "qos_topology": 0,
    "qos_vs_capacity": 0,
    "qos_vs_score": 0,
    "ttl": 30,
    "verify_member_availability": "yes"
  }
]

For brevity, I'm picking off the name key to demonstrate what's returned. Note multiples / duplicates are being returned for #1 and #2, when what I want is the single document.

For #1:

 ➜  jq -r '.[] | select(.members[] | has("subPath")).name' test.json 
a2ws-test.ree.se.net_pool
a2ws-test.ree.se.net_pool

For #2:

 ➜  jq -r '.[] | select(.members[] | has("subPath") | not).name' test.json    
aci-apic.ree.se.net_pool
aci-apic.ree.se.net_pool
aci-apic.ree.se.net_pool

For #3:

 ➜  jq -r '.[] | select(.members[] | length == 0)' test.json
 ➜

In the end, I'm wanting to query this json as aforementioned, and create yaml output from it, something like:

echo "# Discovered Members"
## Discovered Members have the key "subPath"
## look for and only return those json documents
cat test.json | jq -r '.[] | select(.members[] | has("subPath")) |
      "- name: " + .name,
      "  state: " + .enabled,
      "  type: a",
      "  preferred_lb_method: " + .load_balancing_mode,
      "  alternate_lb_method: " + .alternate_mode,
      "  fallback_lb_method: " + .fallback_mode,
      "  max_answers_returned: " + (.max_answers_returned | tostring),
      "  ttl: " + (.ttl | tostring),
      "  members:", (.members[] |
      "    - server: " + (.subPath | split(":") | first),
      "      virtual_server: /Common/" + .name,
      "      state: " + .enabled,
      "      member_order: " + (.member_order | tostring),
      "      ratio: " + (.ratio | tostring),
      "      monitor: " + .monitor)' \
    | sed 's/state: yes/state: present/g' \
    | sed 's/state: no/state: disabled/g'

echo "# Static Members"
# Static Members DO NOT have the key "subPath"
# look for and only return those json documents
jq -r '.[] | select(.members[] | has("subPath") | not) |
      "- name: " + .name,
      "  state: " + .enabled,
      "  type: a",
      "  preferred_lb_method: " + .load_balancing_mode,
      "  alternate_lb_method: " + .alternate_mode,
      "  fallback_lb_method: " + .fallback_mode,
      "  max_answers_returned: " + (.max_answers_returned | tostring),
      "  ttl: " + (.ttl | tostring),
      "  members:", (.members[] |
      "    - server: " + (.name | split(":") | first),
      "      virtual_server: " + (.name | split(":") | last),
      "      state: " + .enabled,
      "      member_order: " + (.member_order | tostring),
      "      ratio: " + (.ratio | tostring),
      "      monitor: " + .monitor)' test.json \
    | sed 's/dev_dc1_servers/static-f5-dev-dc1-servers/g' \
    | sed 's/dev_dc2_servers/static-f5-dev-dc2-servers/g' \
    | sed 's/tst_dc1_servers/static-f5-tst-dc1-servers/g' \
    | sed 's/tst_dc2_servers/static-f5-tst-dc2-servers/g' \
    | sed 's/prd_dc1_servers/static-f5-prd-dc1-servers/g' \
    | sed 's/prd_dc2_servers/static-f5-prd-dc2-servers/g' \
    | sed 's/state: yes/state: present/g' \
    | sed 's/state: no/state: disabled/g'

# those that do not have members defined, but still need to be represented as objects to consider
# figure out the jq first

I've tried piping to unique with no success which tells me I'm querying incorrectly.

How would I go about returning each document from a list of dictionaries based on whether or not a list contained within each dictionary does or does not contain a specific key name, and find those whose list is empty?

Solution

  • For #1 and #2, it's not clear to me whether you want the first item satisfying the condition, or the collection of distinct items that satisfy the condition.

    For the first item, you could use first:

    first(.[] | select(.members[] | has("subPath")).name)

    For the distinct items, you could of course use unique:

    map(select(.members[] | has("subPath")).name ) | unique[]

    Similarly for #2.

    For #3, you need to check .members|length, not .members|length:

    .[] | select(.members | length == 0)