I found the following code in Kaggle.
import re
from nltk.corpus import stopwords # Import the stop word list
def description_to_words(review_text):
# 2. Remove non-letters
letters_only = re.sub("[^a-zA-Z]", " ", review_text)
# 3. Convert to lower case, split into individual words
words = letters_only.lower().split()
# 4. In Python, searching a set is much faster than searching
# a list, so convert the stop words to a set
stops = set(stopwords.words("english"))
# 5. Remove stop words
meaningful_words = [w for w in words if not w in stops]
# 6. Join the words back into one string separated by space,
# and return the result.
return( " ".join( meaningful_words ))
The above code works well with the following function call
clean_review = description_to_words(df['MaterialDescription'][3] )
print(clean_review)
But when I try the above same thing like assigning DataFrame to another variable as shown below,
X = df['MaterialDescription']
clean_review = description_to_words(X[3] )
print(clean_review)
I am getting the following error which is totally ridiculous. I am sure that I need some clarity in Pandas
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
C:\Anaconda\envs\tensorflow\lib\site-packages\pandas\indexes\base.py in get_loc(self, key, method, tolerance)
2133 try:
-> 2134 return self._engine.get_loc(key)
2135 except KeyError:
pandas\index.pyx in pandas.index.IndexEngine.get_loc (pandas\index.c:4433)()
pandas\index.pyx in pandas.index.IndexEngine.get_loc (pandas\index.c:4279)()
pandas\src\hashtable_class_helper.pxi in pandas.hashtable.PyObjectHashTable.get_item (pandas\hashtable.c:13742)()
pandas\src\hashtable_class_helper.pxi in pandas.hashtable.PyObjectHashTable.get_item (pandas\hashtable.c:13696)()
KeyError: 3
During handling of the above exception, another exception occurred:
KeyError Traceback (most recent call last)
<ipython-input-15-5c63f93c009a> in <module>()
----> 1 clean_review = description_to_words(X[3] )
2 print(clean_review)
C:\Anaconda\envs\tensorflow\lib\site-packages\pandas\core\frame.py in __getitem__(self, key)
2057 return self._getitem_multilevel(key)
2058 else:
-> 2059 return self._getitem_column(key)
2060
2061 def _getitem_column(self, key):
C:\Anaconda\envs\tensorflow\lib\site-packages\pandas\core\frame.py in _getitem_column(self, key)
2064 # get column
2065 if self.columns.is_unique:
-> 2066 return self._get_item_cache(key)
2067
2068 # duplicate columns & possible reduce dimensionality
C:\Anaconda\envs\tensorflow\lib\site-packages\pandas\core\generic.py in _get_item_cache(self, item)
1384 res = cache.get(item)
1385 if res is None:
-> 1386 values = self._data.get(item)
1387 res = self._box_item_values(item, values)
1388 cache[item] = res
C:\Anaconda\envs\tensorflow\lib\site-packages\pandas\core\internals.py in get(self, item, fastpath)
3541
3542 if not isnull(item):
-> 3543 loc = self.items.get_loc(item)
3544 else:
3545 indexer = np.arange(len(self.items))[isnull(self.items)]
C:\Anaconda\envs\tensorflow\lib\site-packages\pandas\indexes\base.py in get_loc(self, key, method, tolerance)
2134 return self._engine.get_loc(key)
2135 except KeyError:
-> 2136 return self._engine.get_loc(self._maybe_cast_indexer(key))
2137
2138 indexer = self.get_indexer([key], method=method, tolerance=tolerance)
pandas\index.pyx in pandas.index.IndexEngine.get_loc (pandas\index.c:4433)()
pandas\index.pyx in pandas.index.IndexEngine.get_loc (pandas\index.c:4279)()
pandas\src\hashtable_class_helper.pxi in pandas.hashtable.PyObjectHashTable.get_item (pandas\hashtable.c:13742)()
pandas\src\hashtable_class_helper.pxi in pandas.hashtable.PyObjectHashTable.get_item (pandas\hashtable.c:13696)()
KeyError: 3
I have also tried giving slice in the pandas object. That gives another error
clean_review = description_to_words(X[:3] )
print(clean_review)
The following is the stack trace for the above 2 lines of the code
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-18-f8b01af18e4b> in <module>()
----> 1 clean_review = description_to_words(X[:3] )
2 print(clean_review)
<ipython-input-6-70647cd7caba> in description_to_words(review_text)
6
7 # 2. Remove non-letters
----> 8 letters_only = re.sub("[^a-zA-Z]", " ", review_text)
9 # 3. Convert to lower case, split into individual words
10 words = letters_only.lower().split()
C:\Anaconda\envs\tensorflow\lib\re.py in sub(pattern, repl, string, count, flags)
180 a callable, it's passed the match object and must return
181 a replacement string to be used."""
--> 182 return _compile(pattern, flags).sub(repl, string, count)
183
184 def subn(pattern, repl, string, count=0, flags=0):
TypeError: expected string or bytes-like object
IF someone helps me in understanding, what really goes on here, it would be of a great help.
The following lines
X = df['MaterialDescription']
clean_review = description_to_words(X[3] )
give in python description_to_words(df['MaterialDescription'][3] )
You have to locate your index through:
clean_review = description_to_words(df.iloc[3]['MaterialDescription'] )