I am a beginner to AI and sentimental analysis. I'm doing sentimental analysis between two documents. This code is working perfectly fine when I add only one source document rather than a list of multiple source documents to compare it with multiple target documents.
Can someone please tell me what I need to change to work it with multiple source documents list?
#Loading pre=trained word2vec model
from gensim.models.keyedvectors import KeyedVectors
# You need to dowload google pre-trained model using below link
# https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit
#Change the path according to your directory
model_path = 'E:\GoogleNews_vectors_negative300.bin'
w2v_model = KeyedVectors.load_word2vec_format(model_path, binary=True)
#Setting Parameters for model
class DocSim(object):
def __init__(self, w2v_model , stopwords=[]):
self.w2v_model = w2v_model
self.stopwords = stopwords
def vectorize(self, doc):
"""Identify the vector values for each word in the given document"""
doc = doc.lower()
words = [w for w in doc.split(" ") if w not in self.stopwords]
word_vecs = []
for word in words:
try:
vec = self.w2v_model[word]
word_vecs.append(vec)
except KeyError:
# Ignore, if the word doesn't exist in the vocabulary
pass
# Assuming that document vector is the mean of all the word vectors
vector = np.mean(word_vecs, axis=0)
return vector
def _cosine_sim(self, vecA, vecB):
"""Find the cosine similarity distance between two vectors."""
csim = np.dot(vecA, vecB) / (np.linalg.norm(vecA) * np.linalg.norm(vecB))
if np.isnan(np.sum(csim)):
return 0
return csim
def calculate_similarity(self, source_doc, target_docs=[], threshold=0):
"""Calculates & returns similarity scores between given source document & all
the target documents."""
if isinstance(target_docs, str):
target_docs = [target_docs]
source_vec = self.vectorize(source_doc)
results = []
for doc in target_docs:
target_vec = self.vectorize(doc)
sim_score = self._cosine_sim(source_vec, target_vec)
if sim_score > threshold:
results.append({
'score' : sim_score,
'doc' : doc
})
# Sort results by score in desc order
results.sort(key=lambda k : k['score'] , reverse=True)
return results
ds = DocSim(w2v_model)
#Calculate the similarity score between a source rule & a target rule.
source_rule = [ '2.1.1 Context','2.2.3 Value']
target_rule = [ '2.1.1 Context','2.1.2.4 Assist Failed Train']
# This will return one target rules text with a similarity score
sim_scores = ds.calculate_similarity(source_rule, target_rule)
print(sim_scores)
This is the error I am getting right now.
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-22-041084a3f599> in <module>
6 # This will return one target rules text with similarity score
7
----> 8 sim_scores = ds.calculate_similarity(source_rule, target_rule)
9
10 print(sim_scores)
<ipython-input-20-055f5d25808f> in calculate_similarity(self, source_doc, target_docs, threshold)
41 source_doc=[source_doc]
42
---> 43 source_vec = self.vectorize(source_doc)
44 results = []
45 for doc in target_docs:
<ipython-input-20-055f5d25808f> in vectorize(self, doc)
8 def vectorize(self, doc):
9 """Identify the vector values for each word in the given document"""
---> 10 doc = doc.lower()
11 words = [w for w in doc.split(" ") if w not in self.stopwords]
12 word_vecs = []
AttributeError: 'list' object has no attribute 'lower'
Rather than sending the whole list to the function, make sure the source_rule is a list and then iterate over it and then execute the calculate_similarity() function on it
#Loading pre=trained word2vec model
from gensim.models.keyedvectors import KeyedVectors
# You need to dowload google pre-trained model using below link
# https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit
#Change the path according to your directory
model_path = 'E:\GoogleNews_vectors_negative300.bin'
w2v_model = KeyedVectors.load_word2vec_format(model_path, binary=True)
#Setting Parameters for model
class DocSim(object):
def __init__(self, w2v_model , stopwords=[]):
self.w2v_model = w2v_model
self.stopwords = stopwords
def vectorize(self, doc):
"""Identify the vector values for each word in the given document"""
doc = doc.lower()
words = [w for w in doc.split(" ") if w not in self.stopwords]
word_vecs = []
for word in words:
try:
vec = self.w2v_model[word]
word_vecs.append(vec)
except KeyError:
# Ignore, if the word doesn't exist in the vocabulary
pass
# Assuming that document vector is the mean of all the word vectors
vector = np.mean(word_vecs, axis=0)
return vector
def _cosine_sim(self, vecA, vecB):
"""Find the cosine similarity distance between two vectors."""
csim = np.dot(vecA, vecB) / (np.linalg.norm(vecA) * np.linalg.norm(vecB))
if np.isnan(np.sum(csim)):
return 0
return csim
def calculate_similarity(self, source_doc, target_docs=[], threshold=0):
"""Calculates & returns similarity scores between given source document & all
the target documents."""
if isinstance(target_docs, str):
target_docs = [target_docs]
source_vec = self.vectorize(source_doc)
results = []
for doc in target_docs:
target_vec = self.vectorize(doc)
sim_score = self._cosine_sim(source_vec, target_vec)
if sim_score > threshold:
results.append({
'score' : sim_score,
'doc' : doc
})
# Sort results by score in desc order
results.sort(key=lambda k : k['score'] , reverse=True)
return results
ds = DocSim(w2v_model)
#Calculate the similarity score between a source rule & a target rule.
source_rule = [ '2.1.1 Context','2.2.3 Value']
target_rule = [ '2.1.1 Context','2.1.2.4 Assist Failed Train']
if isinstance(source_rule, str):
source_rule = [source_rule]
# This will return one target rules text with a similarity score
for rule in source_rule:
sim_scores = ds.calculate_similarity(rule, target_rule)
print("Similarity with {} is {}".format(rule, sim_scores))