"Out Of Memory" exception while exporting from SQL Server to Excel, using EPPlus library

Question

I'm using the EPPlus .NET library in order to export data from SQL Server to an Excel file.

I'm using the SqlConnection class to read the data. For every row of the SqlDataReader cursor, I iterate through all the excel cells for the corresponding row, and enter the data from the reader.

The issue is that I'm getting an "out of memory" error when im using this function for large tables. I need a method to create some kind of a buffer inside the Read CURSOR.

A concise code example:

Dim sqlConnection As SqlConnection = New SqlConnection()
sqlConnection.ConnectionString = sqlConnectionString.ConnectionString 'connectionstring built before

Dim query As SqlCommand = New SqlCommand(query...)

Dim newFileStream As New FileStream("c:\junk\test.xlsx", System.IO.FileMode.Create,System.IO.FileAccess.ReadWrite)

Using excelApp As New ExcelPackage(newFileStream)
    sqlConnection.Open()
    Dim sqlReader As SqlDataReader = query.ExecuteReader()

    Dim numOfColumns As Byte = sqlReader.FieldCount()
    Dim rowNumber As Integer = 1

    While sqlReader.Read()
        Dim currentColumn As Byte

        For currentColumn = 1 To numOfColumns
            ws.Cells(rowNumber,currentColumn).Value = sqlReader.Item(currentColumn - 1)
        Next
     rowNumber += 1             
    End While

    excelApp.Save()
End Using

newFileStream.Close()

Answer 1

Since you will have to split the files anyway when you hit Excel's limits, here's some code that reads from the database in chunks into multiple Excel files:

static class Program
{
    private static string _dataSource;
    private static string _database;
    private static string _table;
    private static string _outputPath;
    private static int _batchSize;

    public static void Main()
    {
        try
        {
           _dataSource = ConfigurationManager.AppSettings["DataSource"];
           _database = ConfigurationManager.AppSettings["Database"];
           _table = ConfigurationManager.AppSettings["Table"];
           _outputPath = ConfigurationManager.AppSettings["OutputPath"];
           _batchSize = int.Parse(ConfigurationManager.AppSettings["BatchSize"]);

            CreateExcel(_dataSource, _database, _table, _outputPath, "SELECT * FROM " + _table);
        }
        catch (Exception e)
        {
            Console.WriteLine(e);
        }

        Console.WriteLine("All done!");
    }

    public static void CreateExcel(string dataSource, string databaseName, string tableName, string outputFilePath, string queryNoParameters)
    {
        var sqlConnectionString = new SqlConnectionStringBuilder
            {
                DataSource = dataSource,
                InitialCatalog = databaseName,
                IntegratedSecurity = true
            };

        using (var connection = new SqlConnection(sqlConnectionString.ConnectionString))
        {
            connection.Open();
            using (var command = new SqlCommand { Connection = connection, CommandType = CommandType.Text, CommandText = queryNoParameters })
            using (var sqlReader = command.ExecuteReader())
            {
                int i = 0;
                while (WriteExcelFile(tableName, GetFileInfo(databaseName, tableName, outputFilePath, i++),
                                      sqlReader, sqlReader.FieldCount, _batchSize))
                {
                    Console.WriteLine("Reading next batch...");
                }
            }                
        }
    }

    private static bool WriteExcelFile(string tableName, FileInfo fileInfo, IDataReader sqlReader, int numOfColumns, int count)
    {
        using (var excelPackage = new ExcelPackage(fileInfo))
        {
            ExcelWorksheet worksheet = excelPackage.Workbook.Worksheets.Add(tableName);

            Console.WriteLine("Populating header row...");
            for (var currentColumn = 1; currentColumn <= numOfColumns; currentColumn++)
            {
                worksheet.Cells[1, currentColumn].Value = sqlReader.GetName(currentColumn - 1);
                worksheet.Column(currentColumn).Style.Numberformat.Format =
                    TranslateSystemtypeToExceltype(sqlReader.GetFieldType(currentColumn - 1));
            }

            Console.WriteLine("Reading data rows...");
            int rowNumber = 2;
            while (rowNumber <= count + 1 && sqlReader.Read())
            {
                for (var currentColumn = 1; currentColumn <= numOfColumns; currentColumn++)
                    worksheet.Cells[rowNumber, currentColumn].Value = sqlReader[currentColumn - 1];
                rowNumber++;
            }

            if (rowNumber == 2) //nothing read
            {
                Console.WriteLine("Nothing to read, reached end of table!");
                return false;
            }

            Console.WriteLine("Saving Excel file...");
            excelPackage.Save();
            return rowNumber == count + 2; //in which case we want to read more
        }
    }

    private static FileInfo GetFileInfo(string databaseName, string tableName, string outputFilePath, int i)
    {
        return new FileInfo(Path.Combine(outputFilePath,
                                      Path.ChangeExtension(
                                          string.Format("{0}_{1}_{2}", databaseName, tableName.Replace('.', '-'), i), "xlsx")));
    }

    public static string TranslateSystemtypeToExceltype(Type sysType)
    {
        if (sysType == typeof(string))
            return "@";
        if (sysType == typeof(DateTime))
                return "dd/MM/YYYY";
        if (sysType == typeof(Decimal))
                return "0.000";
        if (sysType == typeof(bool))
                return "@";
        if (sysType == typeof(int))
                return "0";
        if (sysType == typeof(short))
                return "0";
        if (sysType == typeof(double))
                return "0.000";
        return "General";
    }
}