How to pass Reuters-21578 dataset as an input parameter for tokenize funktion in Python

Question

I try to pass Reuters-21578 dataset as an input parameter into tokenize funktion def tokenize(text): which should delete stop words, tokenize ,stem and lowercase.

#!/usr/bin/python3
import nltk
import pandas as pd
import numpy as np
import string
from nltk.corpus import reuters
from nltk import word_tokenize
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
import re
cachedStopWords = stopwords.words("english")


 for index, i in  enumerate(reuters.fileids()):
    text = reuters.raw(fileids=[i])

    #output in a txt file
#print(text, file=open("output.txt", "a"))


def tokenize(text):
    min_length = 3
    words = map(lambda word: word.lower(), word_tokenize(text));
    words = [word for word in words
                  if word not in cachedStopWords]
    tokens =(list(map(lambda token: PorterStemmer().stem(token),
                  words)));
    p = re.compile('[a-zA-Z]+');
    filtered_tokens =list(filter(lambda token:
                  p.match(token) and len(token)>=min_length,
         tokens));
    return filtered_tokens

result=tokenize(text)
print(result)

As a result I get only the following:

['a.h.a', 'automot', 'technolog', 'corp', 'year', 'net', 'shr', 'shr', 'dilut', 'net', 'rev', 'mln', 'mln']

How could that be if I passed the entire dataset into tokenize function?

Answer 1

you are overwriting text for every for loop , that why you are getting output which belong to last record in reuter dataset. just do small modification in your code..

text = ''
for index, i in  enumerate(reuters.fileids()):
    text += reuters.raw(fileids=[i])