Search code examples
delphimp3

Odd characters in Unicode String


I'm running into something when it comes to getting an MP3 ID3 v2 implementation going. I have it working for most part except for this one issue, which probably isn't related to that at all. Anyhow, I'm using the below code to handle retrieving data of header tags that involve text.

What I'm running into is that (I guess?) I'm encountering Unicode characters in some different strings. I made the attempt to convert it below, and it works. But I'm getting $3F as a character ahead of the string and $3F$3F afterwards. Is there anything I can do to the code below to parse those out or would I have to do it myself? The files were encoded by ITunes if that helps any.

function Id3v2_string(currp: pointer; datasize: integer): string;
{ handles string processing for ID3v2 data }
  const
    IS_TEXT_UNICODE_UNICODE_MASK = $0F;
  var
    outstr: string;
    uscan: integer;
  begin
    outstr := '';
    SetLength(outstr, datasize);
    uscan := IS_TEXT_UNICODE_UNICODE_MASK;
    if IsTextUnicode(currp, datasize, @uscan) then
      outstr := WideCharToString(currp)
    else
      move(currp^, outstr[1], datasize);
    Result := outstr;
  end;

Note, I really am not interested in a media library for this since all I'm looking to do is edit ID3 tags and not play files - the implementation is already completed except for a few minor problems like this one.


Solution

  • Depending on which version of ID3 v2 is being used, text strings may or may not be preceeded with a byte to tell you the actual encoding of the string. Don't use IsTextUnicode() to guess what the encoding is (especially since it can report false results).

    In ID3 v2 up to v2.3, there is no encoding byte, text is either ISO-8859-1 or UCS-2, and UCS-2 strings always start with a BOM so you know the byte ordering. For example:

    // prior to Delphi 2009 - String is Ansi
    function Id3v2_string(currp: Pointer; datasize: Integer): String; 
    var
      W: WideString;
      I: Integer;
      Ch: WideChar;
    begin 
      Result := '';
      if (datasize >= SizeOf(Word)) and ((PWord(currp)^ = $FEFF) or (PWord(currp)^= $FFFE)) then begin
        // UCS-2 with BOM
        W := WideCharLenToString(PWideChar(Integer(currp) + SizeOf(Word)), (datasize - SizeOf(Word)) div SizeOf(WideChar)); 
        if PWord(currp)^ = $FFFE then begin
          // BE, convert to LE
          for I := 1 to Length(W) do begin
            Ch := W[I];
            W[I] := WideChar(((Word(Ch) and $FF) shl 8) or (Word(Ch) shr 8));
          end;
        end;
      end else begin
        // ISO-8859-1
        I := MultiByteToWideChar(28591, 0, PAnsiChar(currp), datasize, nil, 0);
        if I > 0 then begin
          SetLength(W, I);
          MultiByteToWideChar(28591, 0, PAnsiChar(currp), datasize, PWideChar(W), I);
        end;
      end;
      Result := TrimRight(W);
    end; 
    

    .

    // Delphi 2009+ - String is Unicode
    function Id3v2_string(currp: Pointer; datasize: Integer): String; 
    var
      Enc: TEncoding;
    
      function Convert(P: Pointer; Size: Integer): String;
      var
        Buf: TBytes;
      begin
        SetLength(Buf, Size);
        if Size > 0 then Move(P^, Buf[0], Size);
        Result := Enc.GetString(Buf);
      end;
    
    begin 
      Result := '';
      if (datasize >= SizeOf(Word)) and ((PWord(currp)^ = $FEFF) or (PWord(currp)^ = $FFFE)) then begin
        // UCS-2 with BOM
        if PWord(currp)^ = $FFFE then begin
          // BE
          Enc := TEncoding.BigEndianUnicode;
        end else begin
          // LE
          Enc := TEncoding.Unicode;
        end;
        Result := Convert(PWord(currp)+1, datasize - SizeOf(Word));
      end else begin
        // ISO-8859-1
        Enc := TEncoding.GetEncoding(28591);
        try
          Result := Convert(currp, datasize);
        finally
          Enc.Free;
        end;
      end;
    end; 
    

    ID3 v2.4 switches UCS-2 to UTF-16, and adds support for UTF-8 and UTF-16BE without a BOM, eg:

    // prior to Delphi 2009 - String is Ansi
    function Id3v2_string(currp: Pointer; datasize: Integer; Encoding: Byte): String; 
    var
      W: WideString;
      I: Integer;
      Ch: WideChar;
    begin 
      Result := '';
    
      case Encoding of
        $00: begin
          // ISO-8859-1
          I := MultiByteToWideChar(28591, 0, PAnsiChar(currp), datasize, nil, 0);
          if I > 0 then begin
            SetLength(W, I);
            MultiByteToWideChar(28591, 0, PAnsiChar(currp), datasize, PWideChar(W), I);
          end;
        end;
        $01: begin
          // UTF-16 with BOM
          SetString(W, PWideChar(Integer(currp) + SizeOf(Word)), (datasize - SizeOf(Word)) div SizeOf(WideChar));
          if PWord(currp)^ = $FFFE then begin
            // BE, convert to LE
            for I := 1 to Length(W) do begin
              Ch := W[I];
              W[I] := WideChar(((Word(Ch) and $FF) shl 8) or (Word(Ch) shr 8));
            end;
          end;
        end;
        $02: begin
          // UTF-16BE without BOM, convert to LE
          SetString(W, PWideChar(currp), datasize div SizeOf(WideChar));
          for I := 1 to Length(W) do begin
            Ch := W[I];
            W[I] := WideChar(((Word(Ch) and $FF) shl 8) or (Word(Ch) shr 8));
          end;
        end;
        $03: begin
          // UTF-8
          I := MultiByteToWideChar(65001, 0, PAnsiChar(currp), datasize, nil, 0);
          if I > 0 then begin
            SetLength(W, I);
            MultiByteToWideChar(65001, 0, PAnsiChar(currp), datasize, PWideChar(W), I);
          end;
        end;
      end;
      Result := TrimRight(W);
    end;
    

    .

    // Delphi 2009+ - String is Unicode
    function Id3v2_string(currp: Pointer; datasize: Integer; Encoding: Byte): String; 
    var
      Enc: TEncoding;
    
      function Convert(P: Pointer; Size: Integer): String;
      var
        Buf: TBytes;
      begin
        SetLength(Buf, Size);
        if Size > 0 then Move(P^, Buf[0], Size);
        Result := Enc.GetString(Buf);
      end;
    
    begin 
      Result := '';
    
      case Encoding of
        $00: begin
          // ISO-8859-1
          Enc := TEncoding.GetEncoding(28591);
          try
            Result := Convert(currp, datasize);
          finally
            Enc.Free;
          end;
        end;
        $01: begin
          // UTF-16 with BOM
          if PWord(currp)^ = $FFFE then begin
            // BE
            Enc := TEncoding.BigEndianUnicode;
          end else begin
            // LE
            Enc := TEncoding.Unicode;
          end;
          Result := Convert(PWord(currp)+1, datasize - SizeOf(Word));
        end;
        $02: begin
          // UTF-16BE without BOM
          Enc := TEncoding.BigEndianUnicode;
          Result := Convert(currp, datasize);
        end;
        $03: begin
          // UTF-8
          Enc := TEncoding.UTF8;
          Result := Convert(currp, datasize);
        end;
      end;
      Result := TrimRight(Result);
    end;