Search code examples
creverse-engineeringdecompilerida

HexRays - what is "__OFSUB__()" purpose?


In the following decompiled function using Ida pro's Hex rays:

int sub_409650()
{
  int v0; // ecx@1
  int result; // eax@1
  bool v2; // zf@1
  bool v3; // sf@1
  unsigned __int8 v4; // of@1
  unsigned __int16 v5; // cx@2
  unsigned int v6; // ecx@2

  v0 = gS1_dword_62EEA8 & 7;
  result = gS1_dword_62EEA8 - v0;
  v4 = __OFSUB__(gS1_dword_62EEA8 - v0, 16);
  v2 = gS1_dword_62EEA8 - v0 == 16;
  v3 = gS1_dword_62EEA8 - v0 - 16 < 0;
  gS1_dword_62EEA8 -= v0;
  gs2_dword_62EFB4 >>= v0;
  if ( (unsigned __int8)(v3 ^ v4) | v2 )
  {
    v5 = *dword_62EFB0;
    ++dword_62EFB0;
    v6 = (v5 << result) | gs2_dword_62EFB4;
    result += 16;
    gs2_dword_62EFB4 = v6;
    gS1_dword_62EEA8 = result;
  }
  return result;
}

It calls __OFSUB__ but what does this do? I figured it was something to do with overflow - but if that was true then why isn't the condition:

// Checking if subtracting v0 is 16 or negative?
if ( v3 | v2 )

Update: raw asm is (some stuff renamed now):

.text:00409650 sub_409650 proc near      
.text:00409650                 mov     eax, gBitCounter_62EEA8
.text:00409655                 push    esi
.text:00409656                 mov     esi, gFirstAudioFrameDWORD_dword_62EFB4
.text:0040965C                 mov     ecx, eax
.text:0040965E                 and     ecx, 7
.text:00409661                 shr     esi, cl
.text:00409663                 sub     eax, ecx
.text:00409665                 cmp     eax, 10h
.text:00409668                 mov     gBitCounter_62EEA8, eax
.text:0040966D                 mov     gFirstAudioFrameDWORD_dword_62EFB4, esi
.text:00409673                 jg      short loc_4096A5
.text:00409675                 mov     edx, gAudioFrameDataPtr
.text:0040967B                 xor     ecx, ecx
.text:0040967D                 mov     cx, [edx]
.text:00409680                 add     edx, 2
.text:00409683                 mov     esi, ecx
.text:00409685                 mov     ecx, eax
.text:00409687                 shl     esi, cl
.text:00409689                 mov     ecx, gFirstAudioFrameDWORD_dword_62EFB4
.text:0040968F                 mov     gAudioFrameDataPtr, edx
.text:00409695                 or      ecx, esi
.text:00409697                 add     eax, 10h
.text:0040969A                 mov     gFirstAudioFrameDWORD_dword_62EFB4, ecx
.text:004096A0                 mov     gBitCounter_62EEA8, eax
.text:004096A5
.text:004096A5 loc_4096A5:                             ; CODE XREF: sub_409650+23j
.text:004096A5                 pop     esi
.text:004096A6                 retn
.text:004096A6 sub_409650 endp

Solution

  • There's nothing much to say about it. HexRays did a pretty bad job decompiling this functions to something legible, but it did not make a mistake.

    A quick analysis of the given assembly:

    mov     eax, gBitCounter_62EEA8                 ; eax = gBitCounter_62EEA8
    push    esi
    mov     esi, gFirstAudioFrameDWORD_dword_62EFB4 ; esi = gFirstAudioFrameDWORD_dword_62EFB4
    mov     ecx, eax                                    
    and     ecx, 7                              
    
    ; esi = gFirstAudioFrameDWORD_dword_62EFB4 >> (gBitCounter_62EEA8 & 7)  
    ; i.e, esi got shiftted by the 3 LSB's of gBitCounter_62EEA8
    shr     esi, cl                                 
    
    ; eax = gBitCounter_62EEA8 - (gBitCounter_62EEA8 & 7)
    sub     eax, ecx    
    
    cmp     eax, 10h
    
    ; gBitCounter_62EEA8 -= gBitCounter_62EEA8 & 7
    mov     gBitCounter_62EEA8, eax                 
    
    ; gFirstAudioFrameDWORD_dword_62EFB4 >>= gBitCounter_62EEA8 & 7
    mov     gFirstAudioFrameDWORD_dword_62EFB4, esi 
    
    ; if gBitCounter_62EEA8 > 0x10 { return; }
    jg      short loc_4096A5            
    
    ; else... continue work
    ....
    
    loc_4096A5:                             ; CODE XREF: sub_409650+23j
    pop     esi
    retn
    

    A better decompilation would've been:

    gBitCounter_62EEA8 -= gBitCounter_62EEA8 & 7
    gFirstAudioFrameDWORD_dword_62EFB4 >>= gBitCounter_62EEA8 & 7
    
    if (gBitCounter_62EEA8 > 0x10)
    {
        return;
    }
    else
    {
        // rest of code
    }
    

    However, you may notice that HexRays reversed the condition. It has generated this condition:

    if ( (unsigned __int8)(
        (gBitCounter_62EEA8 - 16 < 0)  ^ 
        (__OFSUB__(gBitCounter_62EEA8 - (gBitCounter_62EEA8 & 7), 16))) // when this is actually the gBitCounter_62EEA8 var before modifications
            | 
            (gBitCounter_62EEA8 == 16) )
    

    According to Intel's reference, jg is taken if ZF = 0 and SF = OF.

    The condition closely represents it:

    • Either Counter == 16 - meaning that ZF = 0 so the jg is not taken
    • Either Counter < 16 XOR __OFSUB__(gBitCounter_62EEA8 - (gBitCounter_62EEA8 & 7), 16), which means:

      1. Counter < 16 (let's call it A) - which means SF=1
      2. __OFSUB__(gBitCounter_62EEA8 - (gBitCounter_62EEA8 & 7), 16) (let's call is B) which means OF=1

      If A=true and B=false it means SF=1,OF=0 => SF!=OF. If A=false and B=true it means SF=0,OF=1 = > SF!=OF, which means that if A^B then SF!=OF, which means that if A^B => jg not taken.

    And generally, if the jg is not taken, then "rest of code" is to be executed.

    I hope that helped your understanding of HexRays behavior. It's decompilation was correct but very redundant (it didn't clear away a lot of garbage that it could) and it couldn't predict the appropriate way to determine the conditions (it took the more "difficult" path)