KeyError accessing inner_hits in elasticsearch_dsl Python client

Question

I've encountered a problem when accessing inner_hits data using the Python elasticsearch_dsl client. Any attempt to use the embedded Response object within meta.inner_hits yields a KeyError on "_type" in the container object. The following code is completely self-contained so anyone should be able to reproduce the same result using Python 2.7 and elasticsearch_dsl 5.0.0:

from elasticsearch import Elasticsearch
from elasticsearch_dsl import Index, Mapping, Nested, Search, Q
from elasticsearch_dsl.connections import connections

index = "test_index"

es_con = connections.create_connection(hosts=["localhost:9200"])
es_index = Index(index)
if es_index.exists():
    es_index.delete()
es_index.create()

para_mapping = Mapping("paragraph")
para_mapping.field("sentences", Nested())
para_mapping.save(index)

test_paras = {}
for a in range(2):
    test_paras[a] = {
        "label": "Test Paragraph p{}".format(a),
        "sentences": []
    }

    for b in range(2):
        test_sent = {
            "text": "Test Sentence p{}s{}".format(a, b),
        }

        test_paras[a]["sentences"].append(test_sent)

for idx, para in test_paras.iteritems():
    para_id = "para_id_p{}".format(idx)
    es_con.create(
        index=index,
        doc_type="paragraph",
        id=para_id,
        body=para
    )

es_index.flush()

q_1 = Search(using=es_con).index(index).doc_type('paragraph')
q_2 = q_1 = q_1.query(
    'nested', path='sentences', query=Q('term', **{"sentences.text": "p0s1"}), inner_hits={}
)
q_1 = q_1.execute()
# We got the expected paragraph
print "PARA_ID:                            ", q_1.hits[0].meta.id
# With all sentences
print "PARA[SENTENCES]:                    ", q_1.hits[0].sentences
# We can see inner_hits is included in para.meta
print "DIR PARA.META:                      ", dir(q_1.hits[0].meta)
# And it contains a "sentences" object
print "DIR PARA.META.INNER_HITS:           ", dir(q_1.hits[0].meta.inner_hits)
# Of type elasticsearch_dsl.result.Response
print "TYPE PARA.META.INNER_HITS.SENTENCES:", type(q_1.hits[0].meta.inner_hits.sentences)
# That contains a "hits" object
print "DIR PARA.META.INNER_HITS.SENTENCES: ", dir(q_1.hits[0].meta.inner_hits.sentences)
# But every attempted action yields a KeyError: '_type' in result.AttrList()
try:
    print q_1.hits[0].meta.inner_hits.sentences
except KeyError as e:
    print "\nException:", type(e)
    print "Message:", e

# Uncomment the following line to see full exception detail
# print dir(q.hits[0].meta.inner_hits.sentences.hits)

# The same query using the base-level API
es = Elasticsearch()

results = es.search(index=index, body=q_2.to_dict())
# This works just fine.
print "\nES RESULT:"
print results["hits"]["hits"][0]["inner_hits"]["sentences"]["hits"]["hits"][0]["_source"]["text"]

Is this a bug in the API?

Answer 1

This is a bug where we only account for inner_hits across parent/child relationship and not nested. I created an issue to track the fix: https://github.com/elastic/elasticsearch-dsl-py/issues/565

Thanks for the report!