Is there a way to get the word-count from a Lucene index?

Question

I'm using the latest Apache-Lucene version 8.1.1

Is it possible and how can I get the word-count of all the (non stop words) terms stored in a Lucene index? The result should be:

term1 453443
term2 445484
term3 443333

etc.

I need this in Java or Scala but any language will be fine to illustrate the API for it ...

Answer 1

You can find an example implementation below.

Note that count gives number of documents not number of occurences (lucene word count 4, document count 3). Also stop words are not omitted.

import java.io.IOException;
import java.nio.file.Paths;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.misc.HighFreqTerms;
import org.apache.lucene.misc.TermStats;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

public class LuceneTest2 {
    final static String index = "index";
    final static String field = "text";

    public static void index() {
        try {
            Directory dir = FSDirectory.open(Paths.get(index));
            Analyzer analyzer = new StandardAnalyzer();
            IndexWriterConfig iwc = new IndexWriterConfig(analyzer);

            iwc.setOpenMode(OpenMode.CREATE);

            IndexWriter writer = new IndexWriter(dir, iwc);

            String[] lines = {
                    "lucene java lucene mark",
                    "lucene",
                    "lucene an example",
                    "java python"
            };
            for (int i = 0; i < lines.length; i++) {
                String line = lines[i];
                Document doc = new Document();
                doc.add(new StringField("id", "" + i, Field.Store.YES));
                doc.add(new TextField(field, line.trim(), Field.Store.YES));
                writer.addDocument(doc);
            }

            System.out.println("indexed " + lines.length + " sentences");
            writer.close();
        } catch (IOException e) {
            System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
        }
    }

    public static void count() {
        try {
            IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(index)));
            int numTerms = 100;
            TermStats[] stats = HighFreqTerms.getHighFreqTerms(reader, numTerms, field, new HighFreqTerms.DocFreqComparator());
            for (TermStats termStats : stats) {
                String termText = termStats.termtext.utf8ToString();
                System.out.println(termText + " " + termStats.docFreq);
            }
            reader.close();
        } catch (Exception e) {
            System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
        }
    }

    public static void main(String[] args) {
        index();
        count();
    }
}

This outputs:

lucene 3
java 2
python 1
mark 1
example 1
an 1