pandas: calculate overlapping words between rows only if values in another column match (issue with multiple instances)

Question

I have a dataframe that looks like the following, but with many rows:

import pandas as pd

data = {'intent':  ['order_food', 'order_food','order_taxi','order_call','order_call','order_call','order_taxi'],
'Sent': ['i need hamburger','she wants sushi','i need a cab','call me at 6','she called me','order call','i would like a new taxi' ],
'key_words': [['need','hamburger'], ['want','sushi'],['need','cab'],['call','6'],['call'],['order','call'],['new','taxi']]}

df = pd.DataFrame (data, columns = ['intent','Sent','key_words'])

I have calculated the jaccard similarity using the code below (not my solution):

def lexical_overlap(doc1, doc2): 
    words_doc1 = set(doc1) 
    words_doc2 = set(doc2)

    intersection = words_doc1.intersection(words_doc2)    
    return intersection

and modified the code given by @Amit Amola to compare overlapping words between every possible two rows and created a dataframe out of it:

overlapping_word_list=[]

for val in list(combinations(range(len(data_new)), 2)):
     overlapping_word_list.append(f"the shared keywords between {data_new.iloc[val[0],0]} and {data_new.iloc[val[1],0]} sentences are: {lexical_overlap(data_new.iloc[val[0],1],data_new.iloc[val[1],1])}")
#creating an overlap dataframe
banking_overlapping_words_per_sent = DataFrame(overlapping_word_list,columns=['overlapping_list'])

@gold_cy 's answer has helped me and i made some changes to it to get the output i like:

for intent in df.intent.unique():
    # loc returns a DataFrame but we need just the column
    rows = df.loc[df.intent == intent,['intent','key_words','Sent']].values.tolist()
    combos = combinations(rows, 2)
    for combo in combos:
        x, y = rows
        overlap = lexical_overlap(x[1], y[1])
        print(f"Overlap of intent ({x[0]}) for ({x[2]}) and ({y[2]}) is {overlap}")

the issue is that when there are more instances of the same intent, i run into the error: ValueError: too many values to unpack (expected 2)

and I do not know how to handle that for many more examples that i have in my dataset

Answer 1

Do you want this?

from itertools import combinations
from operator import itemgetter
items_to_consider = []
for item in list(combinations(zip(df.Sent.values, map(set,df.key_words.values)),2)):
    keywords = (list(map(itemgetter(1),item)))
    intersect = keywords[0].intersection(keywords[1])
    if len(intersect) > 0:
        str_list = list(map(itemgetter(0),item))
        str_list.append(intersect)
        items_to_consider.append(str_list)


for i in items_to_consider:
    for item in i[2]:
        if item in i[0] and item in i[1]:
            print(f"Overlap of intent (order_food) for ({i[0]}) and ({i[1]}) is {item}")