How to use python to commit data in batches to SQLServer database?

Question

The data I want to insert into the database likes this:

datalist =[['2012', '1', '3', '1', '832.0', '261.0', '100.00'],
            ['2012', '1', '5', '1', '507.0', '193.0', '92.50'],
            ['2012', '2', '3', '1', '412.0', '200.0', '95.00'],
            ['2012', '2', '5', '1', '560.0', '335.0', '90.00'],
            ['2012', '3', '3', '1', '584.0', '205.0', '100.00'],
            ['2012', '3', '5', '1', '595.0', '162.0', '92.50'],
            ['2012', '4', '3', '1', '504.0', '227.0', '100.00'],
            ['2012', '4', '5', '1', '591.0', '264.0', '92.50']]

But in fact, there are 500,000 rows in datalist. So I just listed a part of it.

The code I insert into the database likes this：

import pymssql

server = '127.0.0.1'
user = "test"
password = "test"
database='SQLTest'
datalist = [['2012', '1', '3', '1', '832.0', '261.0', '100.00'],
            ['2012', '1', '5', '1', '507.0', '193.0', '92.50'],
            ['2012', '2', '3', '1', '412.0', '200.0', '95.00'],
            ['2012', '2', '5', '1', '560.0', '335.0', '90.00'],
            ['2012', '3', '3', '1', '584.0', '205.0', '100.00'],
            ['2012', '3', '5', '1', '595.0', '162.0', '92.50'],
            ['2012', '4', '3', '1', '504.0', '227.0', '100.00'],
            ['2012', '4', '5', '1', '591.0', '264.0', '92.50']]

#But in fact, there are 500,000 rows in datalist

try:
    conn = pymssql.connect(server, user, password, database)
    cursor = conn.cursor()
    for one_row in datalist:
        val1 = one_row[4]
        val2 = one_row[5]
        val3 = one_row[6]
        sql = "insert into table_for_test values(col1, col2, col3)" % (val1, val2,val3)
        cursor.execute(sql)
        conn.commit()
except Exception as ex:
    conn.rollback()
    raise ex
finally:
    conn.close()

Because of the amount of data is too large,So I want to insert data in batchs,how to modify the code?

Answer 1

Now I know how to do it. Use executeMany.The element must be tuple in the list.

import pymssql

server = '127.0.0.1'
user = "test"
password = "test"
database='SQLTest'
datalist = [('2012', '1', '3', '1', '832.0', '261.0', '100.00'),
            ('2012', '1', '5', '1', '507.0', '193.0', '92.50'),
            ('2012', '2', '3', '1', '412.0', '200.0', '95.00'),
            ('2012', '2', '5', '1', '560.0', '335.0', '90.00'),
            ('2012', '3', '3', '1', '584.0', '205.0', '100.00'),
            ('2012', '3', '5', '1', '595.0', '162.0', '92.50'),
            ('2012', '4', '3', '1', '504.0', '227.0', '100.00'),
            ('2012', '4', '5', '1', '591.0', '264.0', '92.50')]

try:
    conn = pymssql.connect(server, user, password, database)
    cursor = conn.cursor()
    sql = "insert into table_for_test values(col1, col2, col3, col4, col5, col6, col7) values(%s, %s, %s, %s, %s, %s, %s)"
    cursor.executemany(sql, datalist)
    conn.commit()
except Exception as ex:
    conn.rollback()
    raise ex
finally:
    conn.close()