Search code examples
c++assemblyx86encryptionxor

Assembly decrypting/reversing? Is XOR an issue?


I have a code function that I have attempted to reverse the effects of with no luck. The original function I have is:

          ror al,1                      // rotates the al part of the eax register (the Ekey) bitwise by 1 bit, as 1 mod 8 = 1 (al = 2D)
      ror al,1                      // moves the rightmost bit from al (the end of the Ekey) and shifts everything along
      ror al,1                      // rotates al bitwise by 1 bit, as 1 mod 8 = 1 (al = 4B)
      ror al,1                      // rotates the end 8 bits of the Ekey bitwise by 1 bit, as 1 mod 8 = 1 (al = A5)
      push ecx                      // preserves the value of the encrypted character by pushing it on the stack, the stack pointer decrements by 4 to allow this
      not eax                       // completes the ones' complement on the Ekey, toggling the bits
      mov edx,eax                   // copies the current value of the Ekey register and places it in edx, for holding
      pop eax                       // restores original register value from stack
      xor eax,edx                   // completes a bitwise exclusive or on the Ekey, with the previous value of the Ekey that was stored in edx
      ror al,1                      // rotates the last 8 bits of the Ekey bitwise by 1 bit, as 1 mod 8 = 1
      ror al,1                      // rotates al bitwise by 1 bit, as 1 mod 8 = 1
      not eax                       // completes the ones' complement on the Ekey value, 'flipping' eax entirely
      add eax,0x20                  // adds the hex value of 20 (32 in base 10) to the current value in the Ekey

I must reverse the EFFECT of the above code only, not each specific line. I have tried various things... attempt 1 (which is wrong):

      sub eax, 0x20
      not eax
      rol al, 2
      xor ecx, eax
      push eax
      mov eax, edx
      not eax
      pop ecx
      rol al, 4

My second attempt is below:

      sub eax, 0x20
      not eax
      rol al, 2 
      not eax
      xor ecx, eax

What is going wrong with this... can an xor's effect be reversed?


Solution

  • I left your function as it was but simplified the decryption:

    unsigned int encrypt(unsigned int input, unsigned int key)
    {
        _asm
        {
            mov ecx, input
            mov eax, key
            push ecx                      ; preserves the value of the encrypted character by pushing it on the stack, the stack pointer decrements by 4 to allow this
            not eax                       ; completes the ones' complement on the Ekey, toggling the bits
            mov edx,eax                   ; copies the current value of the Ekey register and places it in edx, for holding
            pop eax                       ; restores original register value from stack
            xor eax,edx                   ; completes a bitwise exclusive or on the Ekey, with the previous value of the Ekey that was stored in edx
            ror al,1                      ; rotates the last 8 bits of the Ekey bitwise by 1 bit, as 1 mod 8 = 1
            ror al,1                      ; rotates al bitwise by 1 bit, as 1 mod 8 = 1
            not eax                       ; completes the ones' complement on the Ekey value, 'flipping' eax entirely   
        }
    }
    
    unsigned int decrypt(unsigned int input, unsigned int key)
    {
        _asm
        {
            mov eax, input 
            not eax
            rol al,1
            rol al,1
            mov edx, key
            not edx
            xor eax, edx
        }
    }
    
    int main()
    {
        unsigned int data = 0xB84A35F2;
        unsigned int encrypted  = 0;
        unsigned int decrypted = 0;
        unsigned int key = 0x3DB76E8C2;
    
        encrypted = encrypt(data, key);
        decrypted = decrypt(encrypted, key);
        std::cout << "Original Data: " << data << "\nEncrypted Data: " << encrypted << "\nDecrypted Data: " << decrypted << "\n";
        system("PAUSE");
        return 0;
    }