Receiving an out of memory exception with memorystream

Question

I'm attempting to write an application that will work with the open source media management platform, Kaltura. Kaltura has provided some C# client libraries that talk to their web API and I have been able to talk to the server and upload videos successfully. The problem I am having is that once files reach a certain size, I receive an out of memory exception and the program crashes. I would like to attempt to fix this issue and submit the improved code back to the open source project, but being new to C#, I'm not exactly where to start. Is there a better way than memorystream to do what they're doing?

Thanks in advance.

//Problematic code

private void PostMultiPartWithFiles(HttpWebRequest request, KalturaParams kparams, KalturaFiles kfiles)
    {
        string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
        request.ContentType = "multipart/form-data; boundary=" + boundary;

        // use a memory stream because we don't know the content length of the request when we have multiple files
        MemoryStream memStream = new MemoryStream();


        byte[] buffer;
        int bytesRead = 0;

        StringBuilder sb = new StringBuilder();
        sb.Append("--" + boundary + "\r\n");
        foreach (KeyValuePair<string, string> param in kparams)
        {
            sb.Append("Content-Disposition: form-data; name=\"" + param.Key + "\"" + "\r\n");
            sb.Append("\r\n");
            sb.Append(param.Value);
            sb.Append("\r\n--" + boundary + "\r\n");
        }

        buffer = Encoding.UTF8.GetBytes(sb.ToString());
        memStream.Write(buffer, 0, buffer.Length);

        foreach (KeyValuePair<string, FileStream> file in kfiles)
        {
            sb = new StringBuilder();
            FileStream fileStream = file.Value;
            sb.Append("Content-Disposition: form-data; name=\"" + file.Key + "\"; filename=\"" + Path.GetFileName(fileStream.Name) + "\"" + "\r\n");
            sb.Append("Content-Type: application/octet-stream" + "\r\n");
            sb.Append("\r\n");

            // write the current string builder content
            buffer = Encoding.UTF8.GetBytes(sb.ToString());
            memStream.Write(buffer, 0, buffer.Length);

            // write the file content
            buffer = new Byte[checked((uint)Math.Min(4096, (int)fileStream.Length))];
            bytesRead = 0;
            while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) != 0)
                memStream.Write(buffer, 0, bytesRead);

            buffer = Encoding.UTF8.GetBytes("\r\n--" + boundary + "\r\n");
            memStream.Write(buffer, 0, buffer.Length);
        }

        request.ContentLength = memStream.Length;

        Stream requestStream = request.GetRequestStream();
        // write the memorty stream to the request stream
        memStream.Seek(0, SeekOrigin.Begin);
        buffer = new Byte[checked((uint)Math.Min(4096, (int)memStream.Length))];
        bytesRead = 0;
        while ((bytesRead = memStream.Read(buffer, 0, buffer.Length)) != 0)
            requestStream.Write(buffer, 0, bytesRead);

        requestStream.Close();
        memStream.Close();
    }

Answer 1

Here's a version more or less how I would write it. It only compiles, but I haven't tested it. Note the use of a StreamWriter and the direct use of the request stream...

public class SendStuff
    {
        private readonly HttpWebRequest _request;
        private readonly Dictionary<string, string> _kparams;
        private readonly Dictionary<string, FileStream> _kfiles;
        readonly string _boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");

        public SendStuff(
            HttpWebRequest request, 
            Dictionary<string, string> kparams, 
            Dictionary<string, FileStream> kfiles)
        {
            _request = request;
            _kparams = kparams;
            _kfiles = kfiles;
            _request.ContentType = "multipart/form-data; boundary=" + _boundary;
        }

        public void Do()
        {

            // Based on HTTP 1.1, if the server can determine the content length, it need not insist on
            // us sending one. If you are talking
            // to a "special" server, construct the headers beforehand, measure their length
            // and identify the file lengths of the files to be sent.

            using (var reqStream = _request.GetRequestStream())
            using (var writer = new StreamWriter(reqStream))
            {
                writer.NewLine = "\r\n";
                WriteBoundary(writer);
                WriteParams(writer);

                foreach (var file in _kfiles)
                {
                    writer.WriteLine("Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"", 
                        file.Key, 
                        Path.GetFileName(file.Value.Name));
                    writer.WriteLine("Content-Type: application/octet-stream");
                    writer.WriteLine();

                    WriteTheFileContent(reqStream, file.Value);

                    WriteBoundary(writer);
                }
            }
        }

        private static void WriteTheFileContent(Stream reqStream, Stream fileStream)
        {
            int bytesRead;
            var buffer = new byte[4096];
            while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) != 0)
                reqStream.Write(buffer, 0, bytesRead);
        }

        private void WriteParams(StreamWriter writer)
        {
            foreach (var param in _kparams)
            {
                writer.WriteLine("Content-Disposition: form-data; name=\"{0}\"", param.Key);
                writer.WriteLine();
                writer.WriteLine(param.Value);
                WriteBoundary(writer);
            }
        }

        private void WriteBoundary(TextWriter writer)
        {
            writer.WriteLine("\r\n--{0}\r\n", _boundary);
        }
    }