How to use Encoding.ASCII.GetBytes with large amounts of data? (throws OutOfMemoryException)

Question

I'm getting out of memory exception while converting string into byte array.

if (message.Contains("REQZ1S"))
{
    string strMsg = "REQZID;";
    try
    {
        var tmp = LoadCellService.readFromExcel(LoadCellModel.LoadCellRowList.Where(x => x.Stage == 1).ToList(), 1);
        LogHelper.StartTo(nameof(LoadCellSiemensOPCModel), $"tmp count: {tmp.Item1.Count}");
        if (tmp.Item1 != null)
        {
            tmp.Item1.ForEach(
                z => LoadCellModel.LoadCellRowList.Where(x => x.Stage == z.Stage && x.RowIndex == z.RowIndex).First().LoadCellRowColumnList = z.LoadCellRowColumnList
            );

            LoadCellModel.LoadCellRowList.ForEach(
                x => x.LoadCellRowColumnList.ForEach(y =>
                {
                    LogHelper.StartTo("temp", y.LoadCellRowColumnKey + ";" + y.LoadCellRowColumnValue);
                    if (y.LoadCellRowColumnKey == "Distance")
                    {
                        strMsg += strMsg + ";" + y.LoadCellRowColumnValue;
                    }
                })
            );

            LogHelper.KeyValue(nameof(LoadCellSiemensOPCModel), "message2", strMsg);
            byte[] msg = System.Text.Encoding.ASCII.GetBytes(strMsg + "\r");
            Stream.Write(msg, 0, msg.Length);
            LogHelper.KeyValue(nameof(LoadCellSiemensOPCModel), "message", strMsg);
            LogHelper.SiemensOPCTrace(nameof(LoadCellSiemensOPCModel), $"Write <<< {strMsg}");
        }
        else
        {
            byte[] msg = System.Text.Encoding.ASCII.GetBytes("REQZ1E;\r");
            Stream.Write(msg, 0, msg.Length);
            LogHelper.SiemensOPCTrace(nameof(LoadCellSiemensOPCModel), $"Write <<< REQZ1E;");
        }
        LogHelper.Done(nameof(LoadCellSiemensOPCModel), $"Write <<< REQZ1S;");
    }
    catch (Exception EX)
    {           
        LogHelper.KeyValue(nameof(LoadCellSiemensOPCModel), "message1", strMsg);
        LogHelper.Error("TEMP", EX);
    }
}

I suspect that need to add some proper logic more in this part:

 byte[] msg = System.Text.Encoding.ASCII.GetBytes(strMsg + "\r");
 Stream.Write(msg, 0, msg.Length); 

Answer 1

The problem isn't System.Text.Encoding.ASCII.GetBytes("REQZ1E;\r"); - the problem is here:

LoadCellModel.LoadCellRowList.ForEach(
    x => x.LoadCellRowColumnList.ForEach(y => {
    // ...
    strMsg += strMsg + ";" + y.LoadCellRowColumnValue;
    // ...
}

Using the += operator to concatenate strings causes an entire copying and reallocation of the entire string.

So for example, if you have a loop that iterates 100 times (10*x and 10*y for 100 total) and in each iteration it adds 50 characters to the string (so the final output length is 5000 characters), but it also copies everything all-over again, so once you get past, say, 1000 characters the computer now has to copy a whole kilobyte every time - and you got there by copying 999, 998, 997, 996, etc characters too - the runtime complexity for this is pretty awful (on the order of O(n^2)).

You can do this instead:

• (Outer if, try/catch statements and Logging calls ommited for brevity)
• I see you're writing to a Stream, so use a StreamWriter with the desired encoding set.
• I replaced your .ForEach calls with foreach statements.

var tmp = LoadCellService.readFromExcel( LoadCellModel.LoadCellRowList.Where(x => x.Stage == 1 ).ToList() , 1 );

if (tmp.Item1 != null)
{
    // This can be further optimized by loading `LoadCellModel.LoadCellRowList` into a dictionary by an appropriate key.
    foreach( var z in tmp.Item1 )
    {
        var columnList = LoadCellModel.LoadCellRowList
            .Where( x =>
                x.Stage    == z.Stage &&
                x.RowIndex == z.RowIndex
            )
            .First();

        columnList.LoadCellRowColumnList = z.LoadCellRowColumnList;
    }

    using( StreamWriter wtr = new StreamWriter( stream, Encoding.ASCII ) )
    {
        foreach( var x in LoadCellModel.LoadCellRowList )
        {
            foreach( var y in x.LoadCellRowColumnList.Where( yc => yc.LoadCellRowColumnKey == "Distance" ) )
            {
                wtr.Write( ';' );
                wtr.Write( y.LoadCellRowColumnValue );
            }
        }

        wtr.Flush();
    }
}
else
{
    byte[] msg = Encoding.ASCII.GetBytes("REQZ1E;\r");
    stream.Write(msg, 0, msg.Length);
}