Search code examples
cassemblyreverse-engineeringdecompilingida

HexRays - output has values read that are never written?


HexRays generates the following code:

void __cdecl Psx_gte_ncs_446930()
{
  double vx_scaled; // st7
  double vy_scaled; // st6
  double vz_scaled; // st5
  double light_matrix_2; // st7
  double light_colour_matrix_3; // st7
  char v5; // cl
  char never_written_v7; // c0
  char v9; // c0
  char never_written_v11; // c0
  double light_matrix_0; // [esp+0h] [ebp-24h]
  double b_value; // [esp+0h] [ebp-24h]
  double light_matrix_1; // [esp+8h] [ebp-1Ch]
  double light_colour_matrix_2; // [esp+10h] [ebp-14h]
  double g_value; // [esp+10h] [ebp-14h]
  double light_colour_matrix_1; // [esp+18h] [ebp-Ch]
  double r_value; // [esp+18h] [ebp-Ch]

  ++gGteData_722688.gte_ncs_count_72268C;       // Normal color single
  vx_scaled = (double)gGte_VXY0_993EC0.regs.VX * 0.000244140625;
  vy_scaled = (double)gGte_VXY0_993EC0.regs.VY * 0.000244140625;
  vz_scaled = (double)gGte_VXY0_993EC0.regs.VZ * 0.000244140625;
  light_matrix_0 = ((double)gGte_light_source_matrix_993E60.m[0][2] * vz_scaled
                  + (double)gGte_light_source_matrix_993E60.m[0][1] * vy_scaled
                  + (double)gGte_light_source_matrix_993E60.m[0][0] * vx_scaled)
                 * 0.000244140625;
  light_matrix_1 = ((double)gGte_light_source_matrix_993E60.m[1][2] * vz_scaled
                  + (double)gGte_light_source_matrix_993E60.m[1][1] * vy_scaled
                  + (double)gGte_light_source_matrix_993E60.m[1][0] * vx_scaled)
                 * 0.000244140625;
  light_matrix_2 = ((double)gGte_light_source_matrix_993E60.m[2][2] * vz_scaled
                  + (double)gGte_light_source_matrix_993E60.m[2][1] * vy_scaled
                  + (double)gGte_light_source_matrix_993E60.m[2][0] * vx_scaled)
                 * 0.000244140625;
  if ( light_matrix_0 < 0.0 )
    light_matrix_0 = 0.0;
  if ( light_matrix_1 < 0.0 )
    light_matrix_1 = 0.0;
  if ( light_matrix_2 < 0.0 )
    light_matrix_2 = 0.0;
  if ( light_matrix_0 > 7.999 )
    light_matrix_0 = 7.999;
  if ( light_matrix_1 > 7.999 )
    light_matrix_1 = 7.999;
  if ( light_matrix_2 > 7.999 )
    light_matrix_2 = 7.999;
  light_colour_matrix_1 = ((double)gGte_light_colour_matrix_source_993E80.m[0][2] * light_matrix_2
                         + (double)gGte_light_colour_matrix_source_993E80.m[0][1] * light_matrix_1
                         + (double)gGte_light_colour_matrix_source_993E80.m[0][0] * light_matrix_0
                         + (double)gGte_background_colour_993E74.x)
                        * 0.000244140625;
  light_colour_matrix_2 = ((double)gGte_light_colour_matrix_source_993E80.m[1][2] * light_matrix_2
                         + (double)gGte_light_colour_matrix_source_993E80.m[1][1] * light_matrix_1
                         + (double)gGte_light_colour_matrix_source_993E80.m[1][0] * light_matrix_0
                         + (double)gGte_background_colour_993E74.y)
                        * 0.000244140625;
  light_colour_matrix_3 = (light_matrix_2 * (double)gGte_light_colour_matrix_source_993E80.m[2][2]
                         + (double)gGte_light_colour_matrix_source_993E80.m[2][1] * light_matrix_1
                         + (double)gGte_light_colour_matrix_source_993E80.m[2][0] * light_matrix_0
                         + (double)gGte_background_colour_993E74.z)
                        * 0.000244140625;
  if ( light_colour_matrix_1 < 0.0 )
    light_colour_matrix_1 = 0.0;
  if ( light_colour_matrix_2 < 0.0 )
    light_colour_matrix_2 = 0.0;
  if ( light_colour_matrix_3 < 0.0 )
    light_colour_matrix_3 = 0.0;
  if ( light_colour_matrix_1 > 7.999 )
    light_colour_matrix_1 = 7.999;
  if ( light_colour_matrix_2 > 7.999 )
    light_colour_matrix_2 = 7.999;
  if ( light_colour_matrix_3 > 7.999 )
    light_colour_matrix_3 = 7.999;
  gGte_RGB0_993F10.cd = gGte_RGB1_993F14.cd;
  r_value = (double)gGte_r_993ED8 * light_colour_matrix_1 * 0.00390625;
  gGte_RGB1_993F14.cd = gGte_RGB2_993F18.cd;
  gGte_RGB2_993F18.cd = gGte_OTZ_993EDB;
  gGte_RGB0_993F10.r = gGte_RGB1_993F14.r;
  gGte_RGB1_993F14.r = gGte_RGB2_993F18.r;
  v5 = 0;
  g_value = (double)gGte_g_993ED9 * light_colour_matrix_2 * 0.00390625;
  b_value = (double)gGte_b_993EDA * light_colour_matrix_3 * 0.00390625;
  if ( never_written_v7 )
  {
    gGte_RGB2_993F18.r = 0;
  }
  else if ( r_value * 256.0 <= 255.0 )
  {
    gGte_RGB2_993F18.r = (signed int)(r_value * 256.0);
  }
  else
  {
    gGte_RGB2_993F18.r = -1;
  }
  gGte_RGB0_993F10.g = gGte_RGB1_993F14.g;
  gGte_RGB1_993F14.g = gGte_RGB2_993F18.g;
  if ( v9 )
  {
    gGte_RGB2_993F18.g = 0;
  }
  else if ( g_value * 256.0 <= 255.0 )
  {
    gGte_RGB2_993F18.g = (signed int)(g_value * 256.0);
  }
  else
  {
    gGte_RGB2_993F18.g = -1;
  }
  gGte_RGB0_993F10.b = gGte_RGB1_993F14.b;
  gGte_RGB1_993F14.b = gGte_RGB2_993F18.b;
  if ( never_written_v11 )
    goto LABEL_39;
  if ( b_value * 256.0 <= 255.0 )
  {
    v5 = (signed int)(b_value * 256.0);
LABEL_39:
    gGte_RGB2_993F18.b = v5;
    goto LABEL_40;
  }
  gGte_RGB2_993F18.b = -1;
LABEL_40:
  gGte_MAC1_993F24.MAC_32 = (signed int)(r_value * 4096.0);
  gGte_MAC2_993F28.MAC_32 = (signed int)(g_value * 4096.0);
  gGte_MAC3_993F2C.MAC_32 = (signed int)(b_value * 4096.0);
}

Which comes from the following asm:

.text:00446930
.text:00446930 ; Attributes: bp-based frame
.text:00446930
.text:00446930 ; void __cdecl Psx_gte_ncs_446930()
.text:00446930 Psx_gte_ncs_446930 proc near            ; CODE XREF: sub_443D4F+238↑p
.text:00446930
.text:00446930 light_matrix_0  = qword ptr -24h
.text:00446930 light_matrix_1  = qword ptr -1Ch
.text:00446930 light_colour_matrix_2= qword ptr -14h
.text:00446930 light_colour_matrix_1= qword ptr -0Ch
.text:00446930 var_4           = dword ptr -4
.text:00446930
.text:00446930                 push    ebp
.text:00446931                 mov     ebp, esp
.text:00446933                 sub     esp, 24h
.text:00446936                 inc     gGteData_722688.gte_ncs_count_72268C
.text:0044693C                 movsx   eax, word ptr gGte_VXY0_993EC0
.text:00446943                 movsx   ecx, word ptr gGte_VXY0_993EC0+2
.text:0044694A                 mov     [ebp+var_4], eax
.text:0044694D                 fild    [ebp+var_4]
.text:00446950                 movsx   edx, word ptr gGte_VXY0_993EC0+4
.text:00446957                 fmul    ds:dbl_64BF78
.text:0044695D                 mov     [ebp+var_4], ecx
.text:00446960                 fild    [ebp+var_4]
.text:00446963                 movsx   eax, gGte_light_source_matrix_993E60.m+4
.text:0044696A                 fmul    ds:dbl_64BF78
.text:00446970                 mov     [ebp+var_4], edx
.text:00446973                 fild    [ebp+var_4]
.text:00446976                 movsx   ecx, gGte_light_source_matrix_993E60.m+2
.text:0044697D                 fmul    ds:dbl_64BF78
.text:00446983                 mov     [ebp+var_4], eax
.text:00446986                 fild    [ebp+var_4]
.text:00446989                 movsx   edx, gGte_light_source_matrix_993E60.m
.text:00446990                 fmul    st, st(1)
.text:00446992                 mov     [ebp+var_4], ecx
.text:00446995                 fild    [ebp+var_4]
.text:00446998                 mov     [ebp+var_4], edx
.text:0044699B                 movsx   eax, gGte_light_source_matrix_993E60.m+0Ah
.text:004469A2                 fmul    st, st(3)
.text:004469A4                 movsx   ecx, gGte_light_source_matrix_993E60.m+8
.text:004469AB                 faddp   st(1), st
.text:004469AD                 fild    [ebp+var_4]
.text:004469B0                 mov     [ebp+var_4], eax
.text:004469B3                 movsx   edx, gGte_light_source_matrix_993E60.m+6
.text:004469BA                 fmul    st, st(4)
.text:004469BC                 movsx   eax, gGte_light_source_matrix_993E60.m+10h
.text:004469C3                 faddp   st(1), st
.text:004469C5                 fmul    ds:dbl_64BF78
.text:004469CB                 fstp    [ebp+light_matrix_0]
.text:004469CE                 fild    [ebp+var_4]
.text:004469D1                 mov     [ebp+var_4], ecx
.text:004469D4                 movsx   ecx, gGte_light_source_matrix_993E60.m+0Eh
.text:004469DB                 fmul    st, st(1)
.text:004469DD                 fild    [ebp+var_4]
.text:004469E0                 mov     [ebp+var_4], edx
.text:004469E3                 movsx   edx, gGte_light_source_matrix_993E60.m+0Ch
.text:004469EA                 fmul    st, st(3)
.text:004469EC                 faddp   st(1), st
.text:004469EE                 fild    [ebp+var_4]
.text:004469F1                 mov     [ebp+var_4], eax
.text:004469F4                 fmul    st, st(4)
.text:004469F6                 faddp   st(1), st
.text:004469F8                 fmul    ds:dbl_64BF78
.text:004469FE                 fstp    [ebp+light_matrix_1]
.text:00446A01                 fild    [ebp+var_4]
.text:00446A04                 mov     [ebp+var_4], ecx
.text:00446A07                 fmul    st, st(1)
.text:00446A09                 fild    [ebp+var_4]
.text:00446A0C                 mov     [ebp+var_4], edx
.text:00446A0F                 fmul    st, st(3)
.text:00446A11                 faddp   st(1), st
.text:00446A13                 fild    [ebp+var_4]
.text:00446A16                 fmul    st, st(4)
.text:00446A18                 faddp   st(1), st
.text:00446A1A                 fmul    ds:dbl_64BF78
.text:00446A20                 fstp    st(3)
.text:00446A22                 fstp    st
.text:00446A24                 fstp    st
.text:00446A26                 fld     [ebp+light_matrix_0]
.text:00446A29                 fcomp   ds:dbl_64A328
.text:00446A2F                 fnstsw  ax
.text:00446A31                 test    ah, 1
.text:00446A34                 jz      short loc_446A44
.text:00446A36                 mov     dword ptr [ebp+light_matrix_0], 0
.text:00446A3D                 mov     dword ptr [ebp+light_matrix_0+4], 0
.text:00446A44
.text:00446A44 loc_446A44:                             ; CODE XREF: Psx_gte_ncs_446930+104↑j
.text:00446A44                 fld     [ebp+light_matrix_1]
.text:00446A47                 fcomp   ds:dbl_64A328
.text:00446A4D                 fnstsw  ax
.text:00446A4F                 test    ah, 1
.text:00446A52                 jz      short loc_446A62
.text:00446A54                 mov     dword ptr [ebp+light_matrix_1], 0
.text:00446A5B                 mov     dword ptr [ebp+light_matrix_1+4], 0
.text:00446A62
.text:00446A62 loc_446A62:                             ; CODE XREF: Psx_gte_ncs_446930+122↑j
.text:00446A62                 fcom    ds:dbl_64A328
.text:00446A68                 fnstsw  ax
.text:00446A6A                 test    ah, 1
.text:00446A6D                 jz      short loc_446A77
.text:00446A6F                 fstp    st
.text:00446A71                 fld     ds:dbl_64A328
.text:00446A77
.text:00446A77 loc_446A77:                             ; CODE XREF: Psx_gte_ncs_446930+13D↑j
.text:00446A77                 fld     [ebp+light_matrix_0]
.text:00446A7A                 fcomp   ds:dbl_64BFB0
.text:00446A80                 fnstsw  ax
.text:00446A82                 test    ah, 41h
.text:00446A85                 jnz     short loc_446A95
.text:00446A87                 mov     dword ptr [ebp+light_matrix_0], 0DB22D0E5h
.text:00446A8E                 mov     dword ptr [ebp+light_matrix_0+4], 401FFEF9h
.text:00446A95
.text:00446A95 loc_446A95:                             ; CODE XREF: Psx_gte_ncs_446930+155↑j
.text:00446A95                 fld     [ebp+light_matrix_1]
.text:00446A98                 fcomp   ds:dbl_64BFB0
.text:00446A9E                 fnstsw  ax
.text:00446AA0                 test    ah, 41h
.text:00446AA3                 jnz     short loc_446AB3
.text:00446AA5                 mov     dword ptr [ebp+light_matrix_1], 0DB22D0E5h
.text:00446AAC                 mov     dword ptr [ebp+light_matrix_1+4], 401FFEF9h
.text:00446AB3
.text:00446AB3 loc_446AB3:                             ; CODE XREF: Psx_gte_ncs_446930+173↑j
.text:00446AB3                 fcom    ds:dbl_64BFB0
.text:00446AB9                 fnstsw  ax
.text:00446ABB                 test    ah, 41h
.text:00446ABE                 jnz     short loc_446AC8
.text:00446AC0                 fstp    st
.text:00446AC2                 fld     ds:dbl_64BFB0
.text:00446AC8
.text:00446AC8 loc_446AC8:                             ; CODE XREF: Psx_gte_ncs_446930+18E↑j
.text:00446AC8                 movsx   eax, gGte_light_colour_matrix_source_993E80.m+4
.text:00446ACF                 movsx   ecx, gGte_light_colour_matrix_source_993E80.m+2
.text:00446AD6                 mov     [ebp+var_4], eax
.text:00446AD9                 fild    [ebp+var_4]
.text:00446ADC                 movsx   edx, gGte_light_colour_matrix_source_993E80.m
.text:00446AE3                 fmul    st, st(1)
.text:00446AE5                 mov     [ebp+var_4], ecx
.text:00446AE8                 fild    [ebp+var_4]
.text:00446AEB                 mov     [ebp+var_4], edx
.text:00446AEE                 movsx   eax, gGte_light_colour_matrix_source_993E80.m+0Ah
.text:00446AF5                 fmul    [ebp+light_matrix_1]
.text:00446AF8                 movsx   ecx, gGte_light_colour_matrix_source_993E80.m+8
.text:00446AFF                 faddp   st(1), st
.text:00446B01                 fild    [ebp+var_4]
.text:00446B04                 mov     [ebp+var_4], eax
.text:00446B07                 movsx   edx, gGte_light_colour_matrix_source_993E80.m+6
.text:00446B0E                 fmul    [ebp+light_matrix_0]
.text:00446B11                 movsx   eax, gGte_light_colour_matrix_source_993E80.m+10h
.text:00446B18                 faddp   st(1), st
.text:00446B1A                 fild    gGte_background_colour_993E74.x
.text:00446B20                 faddp   st(1), st
.text:00446B22                 fmul    ds:dbl_64BF78
.text:00446B28                 fstp    [ebp+light_colour_matrix_1]
.text:00446B2B                 fild    [ebp+var_4]
.text:00446B2E                 mov     [ebp+var_4], ecx
.text:00446B31                 movsx   ecx, gGte_light_colour_matrix_source_993E80.m+0Eh
.text:00446B38                 fmul    st, st(1)
.text:00446B3A                 fild    [ebp+var_4]
.text:00446B3D                 mov     [ebp+var_4], edx
.text:00446B40                 movsx   edx, gGte_light_colour_matrix_source_993E80.m+0Ch
.text:00446B47                 fmul    [ebp+light_matrix_1]
.text:00446B4A                 faddp   st(1), st
.text:00446B4C                 fild    [ebp+var_4]
.text:00446B4F                 mov     [ebp+var_4], eax
.text:00446B52                 fmul    [ebp+light_matrix_0]
.text:00446B55                 faddp   st(1), st
.text:00446B57                 fild    gGte_background_colour_993E74.y
.text:00446B5D                 faddp   st(1), st
.text:00446B5F                 fmul    ds:dbl_64BF78
.text:00446B65                 fstp    [ebp+light_colour_matrix_2]
.text:00446B68                 fild    [ebp+var_4]
.text:00446B6B                 mov     [ebp+var_4], ecx
.text:00446B6E                 fmulp   st(1), st
.text:00446B70                 fild    [ebp+var_4]
.text:00446B73                 mov     [ebp+var_4], edx
.text:00446B76                 fmul    [ebp+light_matrix_1]
.text:00446B79                 faddp   st(1), st
.text:00446B7B                 fild    [ebp+var_4]
.text:00446B7E                 fmul    [ebp+light_matrix_0]
.text:00446B81                 faddp   st(1), st
.text:00446B83                 fild    gGte_background_colour_993E74.z
.text:00446B89                 faddp   st(1), st
.text:00446B8B                 fmul    ds:dbl_64BF78
.text:00446B91                 fld     [ebp+light_colour_matrix_1]
.text:00446B94                 fcomp   ds:dbl_64A328
.text:00446B9A                 fnstsw  ax
.text:00446B9C                 test    ah, 1
.text:00446B9F                 jz      short loc_446BAF
.text:00446BA1                 mov     dword ptr [ebp+light_colour_matrix_1], 0
.text:00446BA8                 mov     dword ptr [ebp+light_colour_matrix_1+4], 0
.text:00446BAF
.text:00446BAF loc_446BAF:                             ; CODE XREF: Psx_gte_ncs_446930+26F↑j
.text:00446BAF                 fld     [ebp+light_colour_matrix_2]
.text:00446BB2                 fcomp   ds:dbl_64A328
.text:00446BB8                 fnstsw  ax
.text:00446BBA                 test    ah, 1
.text:00446BBD                 jz      short loc_446BCD
.text:00446BBF                 mov     dword ptr [ebp+light_colour_matrix_2], 0
.text:00446BC6                 mov     dword ptr [ebp+light_colour_matrix_2+4], 0
.text:00446BCD
.text:00446BCD loc_446BCD:                             ; CODE XREF: Psx_gte_ncs_446930+28D↑j
.text:00446BCD                 fcom    ds:dbl_64A328
.text:00446BD3                 fnstsw  ax
.text:00446BD5                 test    ah, 1
.text:00446BD8                 jz      short loc_446BE2
.text:00446BDA                 fstp    st
.text:00446BDC                 fld     ds:dbl_64A328
.text:00446BE2
.text:00446BE2 loc_446BE2:                             ; CODE XREF: Psx_gte_ncs_446930+2A8↑j
.text:00446BE2                 fld     [ebp+light_colour_matrix_1]
.text:00446BE5                 fcomp   ds:dbl_64BFB0
.text:00446BEB                 fnstsw  ax
.text:00446BED                 test    ah, 41h
.text:00446BF0                 jnz     short loc_446C00
.text:00446BF2                 mov     dword ptr [ebp+light_colour_matrix_1], 0DB22D0E5h
.text:00446BF9                 mov     dword ptr [ebp+light_colour_matrix_1+4], 401FFEF9h
.text:00446C00
.text:00446C00 loc_446C00:                             ; CODE XREF: Psx_gte_ncs_446930+2C0↑j
.text:00446C00                 fld     [ebp+light_colour_matrix_2]
.text:00446C03                 fcomp   ds:dbl_64BFB0
.text:00446C09                 fnstsw  ax
.text:00446C0B                 test    ah, 41h
.text:00446C0E                 jnz     short loc_446C1E
.text:00446C10                 mov     dword ptr [ebp+light_colour_matrix_2], 0DB22D0E5h
.text:00446C17                 mov     dword ptr [ebp+light_colour_matrix_2+4], 401FFEF9h
.text:00446C1E
.text:00446C1E loc_446C1E:                             ; CODE XREF: Psx_gte_ncs_446930+2DE↑j
.text:00446C1E                 fcom    ds:dbl_64BFB0
.text:00446C24                 fnstsw  ax
.text:00446C26                 test    ah, 41h
.text:00446C29                 jnz     short loc_446C33
.text:00446C2B                 fstp    st
.text:00446C2D                 fld     ds:dbl_64BFB0
.text:00446C33
.text:00446C33 loc_446C33:                             ; CODE XREF: Psx_gte_ncs_446930+2F9↑j
.text:00446C33                 mov     eax, dword ptr gGte_r_993ED8
.text:00446C38                 mov     edx, 0FFh
.text:00446C3D                 and     eax, edx
.text:00446C3F                 xor     ecx, ecx
.text:00446C41                 mov     [ebp+var_4], eax
.text:00446C44                 mov     cl, gGte_g_993ED9
.text:00446C4A                 fild    [ebp+var_4]
.text:00446C4D                 mov     [ebp+var_4], ecx
.text:00446C50                 xor     eax, eax
.text:00446C52                 mov     al, gGte_b_993EDA
.text:00446C57                 mov     cl, gGte_RGB1_993F14.cd
.text:00446C5D                 fmul    [ebp+light_colour_matrix_1]
.text:00446C60                 mov     gGte_RGB0_993F10.cd, cl
.text:00446C66                 mov     cl, byte ptr gGte_OTZ_993EDB
.text:00446C6C                 fmul    ds:dbl_64BFA8
.text:00446C72                 fstp    [ebp+light_colour_matrix_1]
.text:00446C75                 fild    [ebp+var_4]
.text:00446C78                 mov     [ebp+var_4], eax
.text:00446C7B                 mov     al, gGte_RGB2_993F18.cd
.text:00446C80                 mov     gGte_RGB1_993F14.cd, al
.text:00446C85                 mov     al, gGte_RGB1_993F14.r
.text:00446C8A                 fmul    [ebp+light_colour_matrix_2]
.text:00446C8D                 mov     gGte_RGB2_993F18.cd, cl
.text:00446C93                 mov     cl, gGte_RGB2_993F18.r
.text:00446C99                 mov     gGte_RGB0_993F10.r, al
.text:00446C9E                 mov     gGte_RGB1_993F14.r, cl
.text:00446CA4                 fmul    ds:dbl_64BFA8
.text:00446CAA                 xor     cl, cl
.text:00446CAC                 fstp    [ebp+light_colour_matrix_2]
.text:00446CAF                 fild    [ebp+var_4]
.text:00446CB2                 fmul    st, st(1)
.text:00446CB4                 fmul    ds:dbl_64BFA8
.text:00446CBA                 fstp    [ebp+light_matrix_0]
.text:00446CBD                 fstp    st
.text:00446CBF                 fld     [ebp+light_colour_matrix_1]
.text:00446CC2                 fmul    ds:dbl_64BF90
.text:00446CC8                 fcom    ds:dbl_64A328
.text:00446CCE                 fst     [ebp+light_matrix_1]
.text:00446CD1                 fnstsw  ax
.text:00446CD3                 test    ah, 1
.text:00446CD6                 jz      short loc_446CE2
.text:00446CD8                 fstp    st
.text:00446CDA                 mov     gGte_RGB2_993F18.r, cl
.text:00446CE0                 jmp     short loc_446D05

.text:00446CE2
.text:00446CE2 loc_446CE2:                             ; CODE XREF: Psx_gte_ncs_446930+3A6↑j
.text:00446CE2                 fcomp   ds:dbl_64BF88
.text:00446CE8                 fnstsw  ax
.text:00446CEA                 test    ah, 41h
.text:00446CED                 jnz     short loc_446CF7
.text:00446CEF                 mov     gGte_RGB2_993F18.r, dl
.text:00446CF5                 jmp     short loc_446D05

.text:00446CF7
.text:00446CF7 loc_446CF7:                             ; CODE XREF: Psx_gte_ncs_446930+3BD↑j
.text:00446CF7                 fld     [ebp+light_matrix_1]
.text:00446CFA                 fistp   [ebp+var_4]
.text:00446CFD                 mov     al, byte ptr [ebp+var_4]
.text:00446D00                 mov     gGte_RGB2_993F18.r, al
.text:00446D05
.text:00446D05 loc_446D05:                             ; CODE XREF: Psx_gte_ncs_446930+3B0↑j
.text:00446D05                                         ; Psx_gte_ncs_446930+3C5↑j
.text:00446D05                 fld     [ebp+light_colour_matrix_2]
.text:00446D08                 mov     al, gGte_RGB1_993F14.g
.text:00446D0D                 fmul    ds:dbl_64BF90
.text:00446D13                 mov     gGte_RGB0_993F10.g, al
.text:00446D18                 mov     al, gGte_RGB2_993F18.g
.text:00446D1D                 mov     gGte_RGB1_993F14.g, al
.text:00446D22                 fcom    ds:dbl_64A328
.text:00446D28                 fst     [ebp+light_matrix_1]
.text:00446D2B                 fnstsw  ax
.text:00446D2D                 test    ah, 1
.text:00446D30                 jz      short loc_446D3C
.text:00446D32                 fstp    st
.text:00446D34                 mov     gGte_RGB2_993F18.g, cl
.text:00446D3A                 jmp     short loc_446D5F

.text:00446D3C
.text:00446D3C loc_446D3C:                             ; CODE XREF: Psx_gte_ncs_446930+400↑j
.text:00446D3C                 fcomp   ds:dbl_64BF88
.text:00446D42                 fnstsw  ax
.text:00446D44                 test    ah, 41h
.text:00446D47                 jnz     short loc_446D51
.text:00446D49                 mov     gGte_RGB2_993F18.g, dl
.text:00446D4F                 jmp     short loc_446D5F

.text:00446D51
.text:00446D51 loc_446D51:                             ; CODE XREF: Psx_gte_ncs_446930+417↑j
.text:00446D51                 fld     [ebp+light_matrix_1]
.text:00446D54                 fistp   [ebp+var_4]
.text:00446D57                 mov     al, byte ptr [ebp+var_4]
.text:00446D5A                 mov     gGte_RGB2_993F18.g, al
.text:00446D5F
.text:00446D5F loc_446D5F:                             ; CODE XREF: Psx_gte_ncs_446930+40A↑j
.text:00446D5F                                         ; Psx_gte_ncs_446930+41F↑j
.text:00446D5F                 fld     [ebp+light_matrix_0]
.text:00446D62                 mov     al, gGte_RGB1_993F14.b
.text:00446D67                 fmul    ds:dbl_64BF90
.text:00446D6D                 mov     gGte_RGB0_993F10.b, al
.text:00446D72                 mov     al, gGte_RGB2_993F18.b
.text:00446D77                 mov     gGte_RGB1_993F14.b, al
.text:00446D7C                 fcom    ds:dbl_64A328
.text:00446D82                 fst     [ebp+light_matrix_1]
.text:00446D85                 fnstsw  ax
.text:00446D87                 test    ah, 1
.text:00446D8A                 jz      short loc_446D90
.text:00446D8C                 fstp    st
.text:00446D8E                 jmp     short loc_446DAE

.text:00446D90
.text:00446D90 loc_446D90:                             ; CODE XREF: Psx_gte_ncs_446930+45A↑j
.text:00446D90                 fcomp   ds:dbl_64BF88
.text:00446D96                 fnstsw  ax
.text:00446D98                 test    ah, 41h
.text:00446D9B                 jnz     short loc_446DA5
.text:00446D9D                 mov     gGte_RGB2_993F18.b, dl
.text:00446DA3                 jmp     short loc_446DB4

.text:00446DA5
.text:00446DA5 loc_446DA5:                             ; CODE XREF: Psx_gte_ncs_446930+46B↑j
.text:00446DA5                 fld     [ebp+light_matrix_1]
.text:00446DA8                 fistp   [ebp+var_4]
.text:00446DAB                 mov     cl, byte ptr [ebp+var_4]
.text:00446DAE
.text:00446DAE loc_446DAE:                             ; CODE XREF: Psx_gte_ncs_446930+45E↑j
.text:00446DAE                 mov     gGte_RGB2_993F18.b, cl
.text:00446DB4
.text:00446DB4 loc_446DB4:                             ; CODE XREF: Psx_gte_ncs_446930+473↑j
.text:00446DB4                 fld     [ebp+light_colour_matrix_1]
.text:00446DB7                 fmul    ds:dbl_64BF70
.text:00446DBD                 fstp    [ebp+light_matrix_1]
.text:00446DC0                 fld     [ebp+light_matrix_1]
.text:00446DC3                 fistp   [ebp+var_4]
.text:00446DC6                 fld     [ebp+light_colour_matrix_2]
.text:00446DC9                 fmul    ds:dbl_64BF70
.text:00446DCF                 mov     edx, [ebp+var_4]
.text:00446DD2                 mov     dword ptr gGte_MAC1_993F24, edx
.text:00446DD8                 fstp    [ebp+light_matrix_1]
.text:00446DDB                 fld     [ebp+light_matrix_1]
.text:00446DDE                 fistp   [ebp+var_4]
.text:00446DE1                 fld     [ebp+light_matrix_0]
.text:00446DE4                 fmul    ds:dbl_64BF70
.text:00446DEA                 mov     eax, [ebp+var_4]
.text:00446DED                 mov     dword ptr gGte_MAC2_993F28, eax
.text:00446DF2                 fstp    [ebp+light_matrix_0]
.text:00446DF5                 fld     [ebp+light_matrix_0]
.text:00446DF8                 fistp   [ebp+var_4]
.text:00446DFB                 mov     ecx, [ebp+var_4]
.text:00446DFE                 mov     dword ptr gGte_MAC3_993F2C, ecx
.text:00446E04                 mov     esp, ebp
.text:00446E06                 pop     ebp
.text:00446E07                 retn
.text:00446E07 Psx_gte_ncs_446930 endp

Now what I can't figure out is how should the code around:

.text:00446CC2                 
fmul    ds:dbl_64BF90
fcom    ds:dbl_64A328
fst     [ebp+light_matrix_1]
fnstsw  ax
test    ah, 1
jz      short loc_446CE2
fstp    st
mov     gGte_RGB2_993F18.r, cl
jmp     short loc_446D05

Should be decompiled. Since

  b_value = (double)gGte_b_993EDA * light_colour_matrix_3 * 0.00390625;
  if ( never_written_v7 )
  {
    gGte_RGB2_993F18.r = 0;
  }

Can't be correct if never_written_v7 is never written.


Solution

  • After much digging I managed to scale down the issue to this:

    Original source code:

    #include <stdio.h>
    
    struct Reg_RGB
    {
        char r, g, b, cd;
    };
    Reg_RGB gGte_RGB2_993F18;
    
    short int field_0_v = 123;
    unsigned char gGte_r_993ED8 = 99;
    unsigned char gGte_g_993ED9 = 15;
    
    void Test2()
    {
      double g_value = (double)gGte_g_993ED9 * field_0_v;
      double r_value = (double)gGte_r_993ED8 * field_0_v;
    
      if ( r_value * 256.0 < 0.0)
      {
        gGte_RGB2_993F18.r = 0;
      }
      else if ( r_value * 256.0 <= 255.0 )
      {
        gGte_RGB2_993F18.r = (signed int)(r_value * 256.0);
      }
    
      if (g_value * 256.0 < 255.0)
      {
        printf("Hello1\n");
      }
    }
    

    Which VC5 generates the following asm for:

    .text:00401000 ; =============== S U B R O U T I N E =======================================
    .text:00401000
    .text:00401000
    .text:00401000 ; void __cdecl Test2()
    .text:00401000                 public ?Test2@@YAXXZ
    .text:00401000 ?Test2@@YAXXZ   proc near               ; DATA XREF: .rdata:off_408488↓o
    .text:00401000
    .text:00401000 var_C           = dword ptr -0Ch
    .text:00401000 g_value         = qword ptr -8
    .text:00401000
    .text:00401000                 sub     esp, 0Ch
    .text:00401003                 movsx   eax, field_0_v  ; 123
    .text:0040100A                 mov     ecx, gGte_r_993ED8 ; 99
    .text:00401010                 mov     edx, gGte_g_993ED9 ; 15
    .text:00401016                 mov     [esp+0Ch+var_C], eax
    .text:0040101A                 and     ecx, 0FFh
    .text:00401020                 fild    [esp+0Ch+var_C]
    .text:00401024                 mov     [esp+0Ch+var_C], ecx
    .text:00401028                 and     edx, 0FFh
    .text:0040102E                 fild    [esp+0Ch+var_C]
    .text:00401032                 fxch    st(1)
    .text:00401034                 fstp    [esp+0Ch+g_value]
    .text:00401038                 fmul    [esp+0Ch+g_value]
    .text:0040103C                 mov     [esp+0Ch+var_C], edx
    .text:00401040                 fild    [esp+0Ch+var_C]
    .text:00401044                 fxch    st(1)
    .text:00401046                 fmul    ds:dbl_408000   ; 256.0
    .text:0040104C                 fxch    st(1)
    .text:0040104E                 fmul    [esp+0Ch+g_value]
    .text:00401052                 fld     st(1)
    .text:00401054                 fcomp   ds:dbl_408008   ; 0.0
    .text:0040105A                 fstp    [esp+0Ch+g_value] ; any other instruction between fcom/fcomp and fnstsw will break HexRays decompiler
    .text:0040105E                 fnstsw  ax
    .text:00401060                 test    ah, 1
    .text:00401063                 jz      short loc_401070
    .text:00401065                 fstp    st
    .text:00401067                 mov     gGte_RGB2_993F18, 0
    .text:0040106E                 jmp     short loc_40108B
    .text:00401070 ; ---------------------------------------------------------------------------
    .text:00401070
    .text:00401070 loc_401070:                             ; CODE XREF: Test2(void)+63↑j
    .text:00401070                 fcom    ds:dbl_408010
    .text:00401076                 fnstsw  ax
    .text:00401078                 test    ah, 41h
    .text:0040107B                 jz      short loc_401089
    .text:0040107D                 call    __ftol
    .text:00401082                 mov     gGte_RGB2_993F18, al
    .text:00401087                 jmp     short loc_40108B
    .text:00401089 ; ---------------------------------------------------------------------------
    .text:00401089
    .text:00401089 loc_401089:                             ; CODE XREF: Test2(void)+7B↑j
    .text:00401089                 fstp    st
    .text:0040108B
    .text:0040108B loc_40108B:                             ; CODE XREF: Test2(void)+6E↑j
    .text:0040108B                                         ; Test2(void)+87↑j
    .text:0040108B                 fld     [esp+0Ch+g_value]
    .text:0040108F                 fmul    ds:dbl_408000
    .text:00401095                 fcomp   ds:dbl_408010
    .text:0040109B                 fnstsw  ax
    .text:0040109D                 test    ah, 1
    .text:004010A0                 jz      short loc_4010AF
    .text:004010A2                 push    offset aHello1  ; "Hello1\n"
    .text:004010A7                 call    _printf
    .text:004010AC                 add     esp, 4
    .text:004010AF
    .text:004010AF loc_4010AF:                             ; CODE XREF: Test2(void)+A0↑j
    .text:004010AF                 add     esp, 0Ch
    .text:004010B2                 retn
    .text:004010B2 ?Test2@@YAXXZ   endp
    

    And HexRays decompiler generates:

    void __cdecl Test2()
    {
      double temp; // ST08_8
      double r_value; // st7
      char never_written; // c0
      double g_value; // [esp+4h] [ebp-8h]
    
      temp = (double)field_0_v;
      r_value = (double)(unsigned __int8)gGte_r_993ED8 * temp * 256.0;
      g_value = (double)(unsigned __int8)gGte_g_993ED9 * temp;
      if ( never_written )
      {
        gGte_RGB2_993F18 = 0;
      }
      else if ( r_value <= 255.0 )
      {
        gGte_RGB2_993F18 = (signed __int64)r_value;
      }
      if ( g_value * 256.0 < 255.0 )
        printf("Hello1\n");
    }
    

    Here we can see that the decompiler has again generated incorrect output because char never_written; // c0 should never exist and it should actually be using r_value instead.

    This limitation or bug in the decompiler seems to trigger when other floating point instructions are executed after fcom fcomp and before fnstsw. In this case it appears that fstp is what causes the broken output.

    Re-arranging the code so that the sequnce is fcom followed by fnstsw resolves the issue.