Assembly YASM Distance Squared Between 2 Points

Question

I am honestly so lost in my Assembly class. Would anyone be willing to walk me through how to go about this problem? I am to code in EBE. This is the problem:

Write an assembly language program to compute the distance squared between 2 points in the plane identified as 2 integer coordinates each, stored in memory.

I am completely unsure if I am going about this right, but so far this is what I have:

        segment .data
a       dq      5         ; one point
b       dq      10         ; another point
        segment .text
        global  main
main:
        mov     rax, [a]    ; move a into rax
        imul    rax, rax    ; a squared
        mov     rdx, [b]    ; move b into rdx
        imul    rdx, rdx    ; b squared
        sub     rax, rcx    ; is rax 0?
        xor     rax, rax
        ret

Answer 1

The formula would be like:

distance squared = (a.x - b.x)**2 + (a.y - b.y)**2

For example (untested):

        segment .data
a:                         ; one point
.x:     dq      5
.y:     dq      7

b:                         ; another point
.x:     dq      10
.y:     dq      12

        segment .text
        global  main
main:
        mov rax,[a.x]      ;rax = a.x
        mov rbx,[a.y]      ;rbx = a.y
        sub rax,[b.x]      ;rax = (a.x - b.x)
        sub rbx,[b.y]      ;rbx = (a.y - b.y)
        imul rax,rax       ;rax = (a.x - b.x)**2
        imul rbx,rbx       ;rbx = (a.y - b.y)**2
        add rax,rbx        ;rax = (a.x - b.x)**2 + (a.y - b.y)**2 = distance squared
        ret