Garbage in headers when POST-ing with Indy 10.5.8

Question

I'm trying to send a file using POST in multipart/form data via Indy 10.5.8. I'm using Delphi XE2 and I've been trying to POST a file to a server. This is the firs time I've tried this and since my experience with Indy is quite limited, i took the following snippet of code:

unit MsMultiPartFormData;

interface

uses
  SysUtils, Classes;

const
  CONTENT_TYPE = 'multipart/form-data; boundary=';
  CRLF = #13#10;
  CONTENT_DISPOSITION = 'Content-Disposition: form-data; name="%s"';
  FILE_NAME_PLACE_HOLDER = '; filename="%s"';
  CONTENT_TYPE_PLACE_HOLDER = 'Content-Type: %s' + crlf + crlf;
  CONTENT_LENGTH = 'Content-Length: %d' + crlf;

type
  TMsMultiPartFormDataStream = class(TMemoryStream)
  private
    FBoundary: string;
    FRequestContentType: string;
    function GenerateUniqueBoundary: string;
  public
    procedure AddFormField(const FieldName, FieldValue: string);
    procedure AddFile(const FieldName, FileName, ContentType: string; FileData: TStream); overload;
    procedure AddFile(const FieldName, FileName, ContentType: string); overload;
    procedure PrepareStreamForDispatch;
    constructor Create;
    property Boundary: string read FBoundary;
    property RequestContentType: string read FRequestContentType;
  end;

implementation

{ TMsMultiPartFormDataStream }

constructor TMsMultiPartFormDataStream.Create;
begin
  inherited;
  FBoundary := GenerateUniqueBoundary;
  FRequestContentType := CONTENT_TYPE + FBoundary;
end;

procedure TMsMultiPartFormDataStream.AddFile(const FieldName, FileName,
  ContentType: string; FileData: TStream);
var
  sFormFieldInfo: string;
  Buffer: PChar;
  iSize: Int64;
begin
  iSize := FileData.Size;
  sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION +
    FILE_NAME_PLACE_HOLDER + CRLF + CONTENT_LENGTH +
      CONTENT_TYPE_PLACE_HOLDER, [FieldName, FileName, iSize, ContentType]);
  {so: boundary + crlf + content-disposition+file-name-place-holder}

  Write(Pointer(sFormFieldInfo)^, Length(sFormFieldInfo));
  FileData.Position := 0;
  GetMem(Buffer, iSize);
  try
    FileData.Read(Buffer^, iSize);
    Write(Buffer^, iSize);
  finally
    FreeMem(Buffer, iSize);
  end;
end;

procedure TMsMultiPartFormDataStream.AddFile(const FieldName, FileName,
  ContentType: string);
var
  FileStream: TFileStream;
begin
  FileStream := TFileStream.Create(FileName, fmOpenRead or fmShareDenyWrite);
  try
    AddFile(FieldName, FileName, ContentType, FileStream);
  finally
    FileStream.Free;
  end;
end;

procedure TMsMultiPartFormDataStream.AddFormField(const FieldName,
  FieldValue: string);
var
  sFormFieldInfo: string;
begin
  sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION + CRLF + CRLF +
    FieldValue, [FieldName]);
  Write(Pointer(sFormFieldInfo)^, Length(sFormFieldInfo));
end;

function TMsMultiPartFormDataStream.GenerateUniqueBoundary: string;
begin
  Result := '---------------------------' + FormatDateTime('mmddyyhhnnsszzz', Now);
end;

procedure TMsMultiPartFormDataStream.PrepareStreamForDispatch;
var
  sFormFieldInfo: string;
begin
  sFormFieldInfo := CRLF + '--' + Boundary + '--' + CRLF;
  Write(Pointer(sFormFieldInfo)^, Length(sFormFieldInfo));
  Position := 0;
end;

end.

I'm calling the code like that:

function PostFile(filename, apikey: string): boolean;
var
  ResponseStream: TMemoryStream;
  MultiPartFormDataStream: TMsMultiPartFormDataStream;
begin
//  Form5.IdHTTP1.HandleRedirects := true;
  Form5.idHTTP1.ReadTimeout := 0;
//  Form5.idHTTP1.IOHandler.LargeStream           := True;
  Result := false;
  MultiPartFormDataStream := TMsMultiPartFormDataStream.Create;
  ResponseStream := TMemoryStream.Create;
  try
    try
    Form5.IdHttp1.Request.ContentType := MultiPartFormDataStream.RequestContentType;
    MultiPartFormDataStream.AddFormField('apikey', apikey);
    MultiPartFormDataStream.AddFile('file', filename, 'multipart/form-data');

    MultiPartFormDataStream.PrepareStreamForDispatch;
    MultiPartFormDataStream.Position := 0;
    Form5.IdHTTP1.Post('http://www.updserver.tld/api//file/save', MultiPartFormDataStream, ResponseStream);
    MultiPartFormDataStream.SaveToFile(ExtractFilePath(Application.ExeName) + 'a.txt');
    Result := true;
    except
 on E:Exception do
   begin
    Form5.Close;
    ShowMessage('Upload failed! '+E.Message);
   end;

    end;
  finally
    MultiPartFormDataStream.Free;
    ResponseStream.Free;
  end;
end;

The file gets sent, but is rejected by the server. Closer inspection of the data sent reveals that the data gets somewhat corrupt (i suspect encoding issues) - what I see is:

POST /api/file/save HTTP/1.0
Connection: keep-alive
Content-Type: multipart/form-data; boundary=---------------------------071312151405662
Content-Length: 11040172
Host: www.updserver.tld
Accept: text/html, */*
Accept-Encoding: identity
User-Agent: Mozilla/3.0 (compatible; Indy Library)




.
.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.0.7.1.3.1.2.1.5.1.4.0.5.6.6.2.
.
.C.o.n.t.e.n.t.-.D.i.s.p.o.s.i.t.i.o.n.:. .f.o.r.m.-.d.a.t.a.;. .n
.
.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.0.7.1.3.1.2.1.5.1.4.0.5.6.6.2.
.
.C.o.n.t.e.n.t.-.D.i.s.p.o.s.i.t.i.o.n.:. .f.o.r.m.-.d.a.t.a.;. .n.a.m.e.=.".f.i.l.e.".;. .f.i.l.e.n.a.m.e.=.".C.:.\.U.s.e........................>.......................................................v.......:...;...<.......[.......v.......................t.......o.......z............
...
...

The regular headers, sent from a working Python client, look like this:

POST https://updserver.tld/api/file/save HTTP/1.0
content-type: multipart/form-data; boundary=---------------------------071312151405662
content-length: 6613364

---------------------------071312151405662
Content-Disposition: form-data; name="apikey"
ac36fae9a406596[rest-of-api-key-goes-here]17966c42b60c8c4cd
---------------------------071312151405662
Content-Disposition: form-data; name="file"; filename="C:\Users\User\Desktop\Project1.exe"
Content-Type: application/octet-stream

Any idea about what I'm doing wrong?

Thanks in advance.

Answer 1

The root of the problem is that your custom TStream code is not compatible with D2009+ versions of Delphi. Delphi's String and PChar types are not Ansi anymore, but the code assumes they still are. They are Unicode UTF-16 now. You are not accounting for that correctly, eg:

procedure TMsMultiPartFormDataStream.AddFile(const FieldName, FileName, ContentType: string; FileData: TStream);   
var   
  sFormFieldInfo: AnsiString;
  iSize: Int64;   
begin   
  iSize := FileData.Size;   
  // NOTE: this will only work for ASCII filenames!!!!
  //
  // Non-ASCII filenames will get converted to Ansi, which can cause data loss.
  // To send non-ASCII filenames correctly, you have to encode it to a charset
  // first, such as UTF-8, and then encode the resulting bytes using
  // MIME's RFC2047 encoding so the server can decode the filename properly
  // on its end...
  //
  sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION +   
    FILE_NAME_PLACE_HOLDER + CRLF + CONTENT_LENGTH +   
      CONTENT_TYPE_PLACE_HOLDER, [FieldName, FileName, iSize, ContentType]);   
  {so: boundary + crlf + content-disposition+file-name-place-holder}   

  Write(sFormFieldInfo[1], Length(sFormFieldInfo) * SizeOf(AnsiChar));   

  if iSize > 0 then
  begin
    FileData.Position := 0;   
    CopyFrom(FileData, iSize);   
  end;
end;   

procedure TMsMultiPartFormDataStream.AddFormField(const FieldName, FieldValue: string);   
var   
  sFormFieldInfo: AnsiString;   
begin   
  // NOTE: this will only work for ASCII text!!!!
  //
  // Non-ASCII text will get converted to Ansi, which can cause data loss.
  // To send non-ASCII text correctly, you have to encode it to a charset
  // first, such as UTF-8 and then encode the resulting bytes using
  // MIME's 'quoted-printable' or 'base64' enoding, and then include
  // appropriate 'charset' and Content-Transfer-Encoding' headers so the
  // server can decode the data properly on its end...
  //
  sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION + CRLF + CRLF +   
    FieldValue, [FieldName]);   
  Write(sFormFieldInfo[1], Length(sFormFieldInfo) * AnsiString(AnsiChar));   
end;   

procedure TMsMultiPartFormDataStream.PrepareStreamForDispatch;   
var   
  sFormFieldInfo: AnsiString;   
begin   
  sFormFieldInfo := CRLF + '--' + Boundary + '--' + CRLF;   
  Write(sFormFieldInfo[1], Length(sFormFieldInfo) * SizeOf(AnsiChar));   
  Position := 0;   
end;   

With that said, I strongly suggest you get rid of your custom TMsMultiPartFormDataStream class completely. All it is doing is mimicing an outdated version of Indy's own TIdMultipartFormDataStream class . Just use Indy's native TIdMultipartFormDataStream class as-is instead. It handles D2009+ Unicode for you, eg:

uses
  ..., IdMultipartFormData;

function PostFile(const filename, apikey: string): boolean; 
var 
  ResponseStream: TMemoryStream; 
  MultiPartFormDataStream: TIdMultiPartFormDataStream; 
begin 
  Result := False; 

  //Form5.IdHTTP1.HandleRedirects := true; 
  Form5.idHTTP1.ReadTimeout := 0; 
  //Form5.idHTTP1.IOHandler.LargeStream := True; 

  try
    ResponseStream := TMemoryStream.Create; 
    try
      MultiPartFormDataStream := TIdMultiPartFormDataStream.Create; 
      try 
        MultiPartFormDataStream.AddFormField('apikey', apikey); 
        MultiPartFormDataStream.AddFile('file', filename, 'application/octet-stream');      
        Form5.IdHTTP1.Post('http://www.updserver.tld/api/file/save', MultiPartFormDataStream, ResponseStream); 
        ResponseStream.SaveToFile(ExtractFilePath(Application.ExeName) + 'a.txt'); 
        Result := True; 
      finally 
        MultiPartFormDataStream.Free; 
      end;
    finally
      ResponseStream.Free; 
    end; 
  except 
    on E:Exception do 
    begin 
      Form5.Close; 
      ShowMessage('Upload failed! ' + E.Message); 
      end; 
    end; 
  end; 
end;