Search code examples
pythonmysqlwhoosh

Writer.add_document() function error Whoosh - mysql loop


I'm trying to index a large amount of articles from a db which is encoded in latin1. I've solved the encoding issue with charset, but I not able to add the to the index each row.

I've tried : 1)

writer.add_document(Id = unicode(row["Id"]),Body = unicode(row["Body"]), Name = unicode(row["Name"]), Brand = unicode(row["Brand"]), Familia = unicode(row["Familia"]))    

This indexes the docs but not respects the index labels.

2)

writer.add_document(doc)

This reports add_document() takes exactly 1 argument (2 given) Error

Here is the full code:

# Open a writer for the index
with ix.writer() as writer:

con= mdb.connect(host="myhost",
                      user="myuser",
                      passwd="pass",
                      db="db",
                      charset="utf8",
                      use_unicode=True)
with con:

    cur = con.cursor(mdb.cursors.DictCursor)
    #cur.execute("SELECT  Id, Body, Name, Brand, Familia FROM articles")
    rows = cur.fetchall()
    for row in rows:
        print row
        doc6 = row["Brand"]
        doc2 = row["Name"]
        print doc2
        print 'body'
        doc3 = row["Body"].replace("á", "a")
        doc3 = doc3.replace("é", "e")
        doc3 = doc3.replace("í", "i")
        doc3 = doc3.replace("ó", "o")
        doc3 = doc3.replace("ú", "u")
        doc3 = doc3.replace("ñ", "n")
        doc3 = doc3.replace(""", "")
        print doc3
        print 'familia'
        doc4 = row["Familia"]
        print doc4
        print 'id'
        doc5 = row["Id"]
        print doc5

        writer.add_document(Id = unicode(row["Id"]),Body = unicode(row["Body"]), Name = unicode(row["Name"]), Brand = unicode(row["Brand"]), Familia = unicode(row["Familia"]))
       #
       # doc = unicode(doc5),unicode(doc3), unicode(doc2), unicode(doc6), unicode(doc4)
       # writer.add_document(doc) #reports add_document() takes exactly 1 argument (2 given) Error
       #writer.add_document(Id = unicode(doc5),Body = unicode(doc3), Name = unicode(doc2), Brand = unicode(doc6), Familia = unicode(doc4))


numdocs = ix.doc_count_all()
print "docs indexed =", numdocs

Thank you all in advance!


Solution

  • Solved this way:

    with con:
    
       cur = con.cursor(mdb.cursors.DictCursor)
       #cur.execute("SELECT  Id, Body, Name, Brand, Familia FROM articles")
       rows = cur.fetchall()
       for row in rows:
             #print row
             row["Body"]= row["Body"].replace("á", "a")
             row["Body"]= row["Body"].replace("é", "e")
             row["Body"]= row["Body"].replace("í", "i")
             row["Body"]= row["Body"].replace("ó", "o")
             row["Body"]= row["Body"].replace("ú", "u")
             row["Body"]= row["Body"].replace("ñ", "n")
             row["Body"]= row["Body"].replace(""", "")
    
             writer.add_document(Id=unicode(row["Id"]),
                            Body=unicode(row["Body"]),
                            Name=unicode(row["Name"]),
                            Brand=unicode(row["Brand"]),
                            Familia=unicode(row["Familia"]),
                            Relevancia=row["Relevancia"])
    
    numdocs = ix.doc_count_all()
    print "docs indexed =", numdocs
    

    Special thank to the Whoosh team who patient and kindly solve all my doubts.