Here is 2 functions in my code that are running REALLY SLOW. Basically i read in a document name, open the document, then process it one word at a time. I need to split up the document into sentences, and give each sentence a hash table that represents the number of times the word appears in the sentence. I also need to keep track of all the new words, and a hash table for the total document.
When i run my code now on 10 documents, that have a total of 8000word, and 2100 uniq words it takes about 8000+ seconds to run... almost 1 second per word.
can you tell me how long if(istream.good()) should take?
Or, if you can tell when what is delaying my code. Please let me know if a section is not clear, i will help.
P.S. You can see in the code where i have a start = clock() and end = clock() commented it constantly returns < 1ms. And that is mind-boggleing
void DocProcess::indexString(string sentenceString, hash * sent){
stringstream iss;
string word;
iss.clear();
iss << sentenceString;
while(iss.good())
{
iss >> word;
word = formatWord(word);
std::unordered_map<std::string,int>::const_iterator IsNewWord = words.find(word);
if(IsNewWord == words.end())
{
std::pair<std::string,int> newWordPair (word,0);
std::pair<std::string,int> newWordPairPlusOne (word,1);
words.insert(newWordPair);
sent->insert(newWordPairPlusOne);
}
else
{
std::pair<std::string,int> newWordPairPlusOne (word,1);
sent->insert(newWordPairPlusOne);
}
}
} void DocProcess::indexFile(string iFileName){
hash newDocHash;
hash newSentHash;
scoreAndInfo sentenceScore;
scoreAndInfo dummy;
fstream iFile;
fstream dFile;
string word;
string newDoc;
string fullDoc;
int minSentenceLength = 5;
int docNumber = 1;
int runningLength = 0;
int ProcessedWords = 0;
stringstream iss;
iFile.open(iFileName.c_str());
if(iFile.is_open())
{
while(iFile.good())
{
iFile >> newDoc;
dFile.open(newDoc.c_str());
DocNames.push_back(newDoc);
if(dFile.is_open())
{
scoreAndInfo documentScore;
//iss << dFile.rdbuf();
while(dFile.good())
{
//start = clock();
dFile >> word;
++ProcessedWords;
std::unordered_map<std::string,int>::const_iterator IsStopWord = stopWords.find(word);
if(runningLength >= minSentenceLength && IsStopWord != stopWords.end() || word[word.length()-1] == '.')
{
/* word is in the stop list, process the string*/
documentScore.second.second.append(" "+word);
sentenceScore.second.second.append(" "+word);
indexString(sentenceScore.second.second, &sentenceScore.second.first);
sentenceScore.first=0.0;
SentList.push_back(sentenceScore);
sentenceScore.second.first.clear(); //Clear hash
sentenceScore.second.second.clear(); // clear string
//sentenceScore = dummy;
runningLength = 0;
}
else
{
++runningLength;
sentenceScore.second.second.append(" "+word);
documentScore.second.second.append(" "+word);
}
//end = clock();
system("cls");
cout << "Processing doc number: " << docNumber << endl
<< "New Word count: " << words.size() << endl
<< "Total words: " << ProcessedWords << endl;
//<< "Last process time****: " << double(diffclock(end,start)) << " ms"<< endl;
}
indexString(documentScore.second.second, &documentScore.second.first);
documentScore.first=0.0;
DocList.push_back(documentScore);
dFile.close();
//iss.clear();
//documentScore = dummy;
++docNumber;
//end = clock();
system("cls");
cout << "Processing doc number: " << docNumber << endl
<< "Word count: " << words.size();
//<< "Last process time: " << double(diffclock(end,start)) << " ms"<< endl;
}
}
iFile.close();
}
else{ cout << "Unable to open index file: "<<endl <<iFileName << endl;}
} `
Can you try it without
system("cls");
in any of the loops? That surely isn't helping, it's an expensive call.