I'm working on the picoCTF reverse engineering challenge GDB Test Drive, where the goal is to retrieve the encrypted flag using the decryption function inside the binary.
But instead of using GDB, Ghidra, or any other reverse engineering tools, I decided to challenge myself by analyzing only the disassembly output (objdump). I carefully went through each instruction and wrote the decryption function in C on my own, without using any decompiler.
I was able to reverse-engineer the function correctly and got the flag. However, when I tried to write an encryption function that does the reverse of the decryption, it didn’t work fully. Some characters are missing or incorrect, so the encryption is only partially correct.
rsp - 0x50 (80) -> 80 / 8 -> 10 QWORD
rbp <+16> <return_address>
rbp <+8> <old_stack_frame>
rbp <+0>
QWORD - 10 <-8> <stack_canary>
QWORD - 09 <-16> '\0'
QWORD - 08 <-24> <0x666665356664674e>
QWORD - 07 <-32> <0x3035436047623066>
QWORD - 06 <-40> <0x3562334638386243>
QWORD - 05 <-48> <0x413a34407225754c>
QWORD - 04 <-56>
QWORD - 03 <-64>
QWORD - 02 <-72>
QWORD - 01 <-80> char** argv
rsp - 0x30 (48) -> 48 / 8 -> 6 QWORD
rbp <+16> <return address>
rbp <+8> <old_stack_frame_base>
rbp <+0>
QWORD -06 <-8> <length>
QWORD -05 <-16> <copy>
QWORD -04 <-24> <i = 0>
QWORD -03 <-32>
QWORD -02 <-40> <Argument-2 / encrypted_text>
QWORD -01 <-48> <Argument-1 / shift>
I named Argument-1 as shift just to give it a descriptive name, but I am not sure exactly what it represents. However, it was not used anywhere in the function body.
Disassembler dump:-
0000000000001209 <rotate_encrypt>:
1209: f3 0f 1e fa endbr64
120d: 55 push rbp
120e: 48 89 e5 mov rbp,rsp
1211: 48 83 ec 30 sub rsp,0x30
1215: 48 89 7d d8 mov QWORD PTR [rbp-0x28],rdi
1219: 48 89 75 d0 mov QWORD PTR [rbp-0x30],rsi
121d: 48 8b 45 d0 mov rax,QWORD PTR [rbp-0x30]
1221: 48 89 c7 mov rdi,rax
1224: e8 d7 fe ff ff call 1100 <strdup@plt>
1229: 48 89 45 f0 mov QWORD PTR [rbp-0x10],rax
122d: 48 8b 45 f0 mov rax,QWORD PTR [rbp-0x10]
1231: 48 89 c7 mov rdi,rax
1234: e8 97 fe ff ff call 10d0 <strlen@plt>
1239: 48 89 45 f8 mov QWORD PTR [rbp-0x8],rax
123d: 48 c7 45 e8 00 00 00 mov QWORD PTR [rbp-0x18],0x0
1244: 00
1245: eb 70 jmp 12b7 <rotate_encrypt+0xae>
1247: 48 8b 55 f0 mov rdx,QWORD PTR [rbp-0x10]
124b: 48 8b 45 e8 mov rax,QWORD PTR [rbp-0x18]
124f: 48 01 d0 add rax,rdx
1252: 0f b6 00 movzx eax,BYTE PTR [rax]
1255: 3c 20 cmp al,0x20
1257: 7e 58 jle 12b1 <rotate_encrypt+0xa8>
1259: 48 8b 55 f0 mov rdx,QWORD PTR [rbp-0x10]
125d: 48 8b 45 e8 mov rax,QWORD PTR [rbp-0x18]
1261: 48 01 d0 add rax,rdx
1264: 0f b6 00 movzx eax,BYTE PTR [rax]
1267: 3c 7f cmp al,0x7f
1269: 74 46 je 12b1 <rotate_encrypt+0xa8>
126b: 48 8b 55 f0 mov rdx,QWORD PTR [rbp-0x10]
126f: 48 8b 45 e8 mov rax,QWORD PTR [rbp-0x18]
1273: 48 01 d0 add rax,rdx
1276: 0f b6 00 movzx eax,BYTE PTR [rax]
1279: 0f be c0 movsx eax,al
127c: 83 c0 2f add eax,0x2f
127f: 89 45 e4 mov DWORD PTR [rbp-0x1c],eax
1282: 83 7d e4 7e cmp DWORD PTR [rbp-0x1c],0x7e
1286: 7e 17 jle 129f <rotate_encrypt+0x96>
1288: 8b 45 e4 mov eax,DWORD PTR [rbp-0x1c]
128b: 8d 48 a2 lea ecx,[rax-0x5e]
128e: 48 8b 55 f0 mov rdx,QWORD PTR [rbp-0x10]
1292: 48 8b 45 e8 mov rax,QWORD PTR [rbp-0x18]
1296: 48 01 d0 add rax,rdx
1299: 89 ca mov edx,ecx
129b: 88 10 mov BYTE PTR [rax],dl
129d: eb 13 jmp 12b2 <rotate_encrypt+0xa9>
129f: 48 8b 55 f0 mov rdx,QWORD PTR [rbp-0x10]
12a3: 48 8b 45 e8 mov rax,QWORD PTR [rbp-0x18]
12a7: 48 01 d0 add rax,rdx
12aa: 8b 55 e4 mov edx,DWORD PTR [rbp-0x1c]
12ad: 88 10 mov BYTE PTR [rax],dl
12af: eb 01 jmp 12b2 <rotate_encrypt+0xa9>
12b1: 90 nop
12b2: 48 83 45 e8 01 add QWORD PTR [rbp-0x18],0x1
12b7: 48 8b 45 e8 mov rax,QWORD PTR [rbp-0x18]
12bb: 48 3b 45 f8 cmp rax,QWORD PTR [rbp-0x8]
12bf: 72 86 jb 1247 <rotate_encrypt+0x3e>
12c1: 48 8b 45 f0 mov rax,QWORD PTR [rbp-0x10]
12c5: c9 leave
12c6: c3 ret
00000000000012c7 <main>:
12c7: f3 0f 1e fa endbr64
12cb: 55 push rbp
12cc: 48 89 e5 mov rbp,rsp
12cf: 48 83 ec 50 sub rsp,0x50
12d3: 89 7d bc mov DWORD PTR [rbp-0x44],edi
12d6: 48 89 75 b0 mov QWORD PTR [rbp-0x50],rsi
12da: 64 48 8b 04 25 28 00 mov rax,QWORD PTR fs:0x28
12e1: 00 00
12e3: 48 89 45 f8 mov QWORD PTR [rbp-0x8],rax
12e7: 31 c0 xor eax,eax
12e9: 48 b8 41 3a 34 40 72 movabs rax,0x4c75257240343a41
12f0: 25 75 4c
12f3: 48 ba 35 62 33 46 38 movabs rdx,0x4362383846336235
12fa: 38 62 43
12fd: 48 89 45 d0 mov QWORD PTR [rbp-0x30],rax
1301: 48 89 55 d8 mov QWORD PTR [rbp-0x28],rdx
1305: 48 b8 30 35 43 60 47 movabs rax,0x6630624760433530
130c: 62 30 66
130f: 48 ba 66 66 65 35 66 movabs rdx,0x4e67646635656666
1316: 64 67 4e
1319: 48 89 45 e0 mov QWORD PTR [rbp-0x20],rax
131d: 48 89 55 e8 mov QWORD PTR [rbp-0x18],rdx
1321: c6 45 f0 00 mov BYTE PTR [rbp-0x10],0x0
1325: bf a0 86 01 00 mov edi,0x186a0
132a: e8 e1 fd ff ff call 1110 <sleep@plt>
132f: 48 8d 45 d0 lea rax,[rbp-0x30]
1333: 48 89 c6 mov rsi,rax
1336: bf 00 00 00 00 mov edi,0x0
133b: e8 c9 fe ff ff call 1209 <rotate_encrypt>
1340: 48 89 45 c8 mov QWORD PTR [rbp-0x38],rax
1344: 48 8b 15 c5 2c 00 00 mov rdx,QWORD PTR [rip+0x2cc5] # 4010 <stdout@GLIBC_2.2.5>
134b: 48 8b 45 c8 mov rax,QWORD PTR [rbp-0x38]
134f: 48 89 d6 mov rsi,rdx
1352: 48 89 c7 mov rdi,rax
1355: e8 96 fd ff ff call 10f0 <fputs@plt>
135a: bf 0a 00 00 00 mov edi,0xa
135f: e8 5c fd ff ff call 10c0 <putchar@plt>
1364: 48 8b 45 c8 mov rax,QWORD PTR [rbp-0x38]
1368: 48 89 c7 mov rdi,rax
136b: e8 40 fd ff ff call 10b0 <free@plt>
1370: b8 00 00 00 00 mov eax,0x0
1375: 48 8b 4d f8 mov rcx,QWORD PTR [rbp-0x8]
1379: 64 48 33 0c 25 28 00 xor rcx,QWORD PTR fs:0x28
1380: 00 00
1382: 74 05 je 1389 <main+0xc2>
1384: e8 57 fd ff ff call 10e0 <__stack_chk_fail@plt>
1389: c9 leave
138a: c3 ret
138b: 0f 1f 44 00 00 nop DWORD PTR [rax+rax*1+0x0]
This the C code i written based on the assembly instructions
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
char* rotate_encrypt(long long int shift, int encrypted_text) {
char* copy = strdup(encrypted_text);
size_t length = strlen(copy);
size_t i;
for (i = 0; i < length; i++) {
if (copy[i] > 0x20) {
if (copy[i] != 0x7F) {
int rbp_28 = copy[i] + 0x2F;
if (rbp_28 > 0x7E) {
copy[i] = (rbp_28) - 0x5E;
} else {
copy[i] = (rbp_28);
}
}
}
return copy;
}
}
int main() {
char encrypted_text[] = "A:4@r%uL5b3F88bC05C`Gb0fffe5fdgN";
sleep(100000); // 27.7 Hours pause for people who try to run and get the flag
char* result = rotate_encrypt(0, encrypted_text);
fputs(result, stdout);
putchar('\n');
free(result);
return 0;
}
Here is my encryption function, which I tried to implement based on my understanding of the decryption logic:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
char* rotate_encrypt2(char* encrypted_text)
{
char* copy = strdup(encrypted_text);
size_t length = strlen(copy);
size_t i;
for(i = 0; i < length; i++)
{
if(copy[i] > 0x20)
{
if(copy[i] != 0x7F)
{
int no_name = copy[i] - 0x2F;
if(no_name > 0x7E)
{
copy[i] = (no_name) + 0x5E;
}
else
{
printf("%c ->",copy[i]);
copy[i] = (no_name);
printf("%c | %d\n",copy[i],copy[i]);
}
}
}
printf("after %zuth iter %s\n",i,copy);
}
return copy;
}
int main()
{
char encrypted_text[32];
printf("Enter the encrypted_text: ");
scanf("%s",encrypted_text);
char* flag = rotate_encrypt2(encrypted_text);
printf("Flag:%s\n",flag);
free(flag);
return 0;
}
It is working fine, but the problem is that it's only partially correct.
Encrypted flag in binary: A:4@r%uL5b3F88bC05C`Gb0fffe5fdgN
Decrypted flag: picoCTF{d3bugg3r_dr1v3_7776d758}, using the C code I wrote.
I tried encrypting this using the function I wrote to check whether my function rotate_encrypt2 is correct or not. However, the output I got was only partially correct.
Expected output: A:4@r%uL5b3F88bC05C`Gb0fffe5fdgN
Actual output: A:4@%L53F88C055G0 N
I even tried to dig deeper to understand what I was missing in the encryption logic.
I noticed that all the missing characters are non-printable due to their ASCII codes, and the difference between these non-printable and actual characters for every character is 0x5E.
So, I decided to change this condition:
if(no_name > 0x7E) → if(no_name < 0x7E).
Now, I can see that the output contains missing characters, but the remaining characters are altered.
Output: ����r�u��b����b����`�b�fffe�fdg�
Can anyone please help me write the encryption function?
Your rotate_encrypt2 function is wrong. First, rotate_encrypt is its own inverse for values under 128, so merely calling it on the encrypted or decrypted text will produce the decrypted or encrypted text, respectively. Second, for character value 81, rotate_encrypt produces 34, but, for character value 34, rotate_encrypt2 produces −13 or 243, depending on whether char is signed or unsigned. (The int no_name takes the value −13 and is later stored in a char.)
For values under 128, the behavior of rotate_encrypt is:
The behavior of rotate_encrypt2 must be the inverse of that. That behavior is its own inverse.