how to properly use posix_memalign

I'am struggling to find out how to proper use the pread and pwrite. In this case, I am trying to read only 256 bytes using pread. However, that whenever I try to read less than 512 bytes pread will not return anything. I believe that this problem has to be with the SECTOR argument that I am assigning to posix_memalign...

Is there some obvious info that I have to be aware of?

#define BUF_SIZE 256
#define SECTOR 512
#define FILE_SIZE 1024 * 1024 * 1024 //1G


int main( int argc, char **argv ){

    int fd, nr;
    char fl_nm[]={"/dev/nvme0n1p1"};

    char* aligned_buf_w = NULL;
    char* aligned_buf_r = NULL;

    void* ad = NULL;
    if (posix_memalign(&ad, SECTOR, BUF_SIZE)) {
        perror("posix_memalign failed"); exit (EXIT_FAILURE);
    }

    aligned_buf_w = (char *)(ad);


    ad = NULL;
    if (posix_memalign(&ad, SECTOR, BUF_SIZE)) {
        perror("posix_memalign failed"); exit (EXIT_FAILURE);
    }
    aligned_buf_r = (char *)(ad);

    memset(aligned_buf_w, '*', BUF_SIZE * sizeof(char));

    printf("BEFORE READ BEGIN\n");
    printf("\t aligned_buf_w::%ld\n",strlen(aligned_buf_w));
    printf("\t aligned_buf_r::%ld\n",strlen(aligned_buf_r));
    printf("BEFORE READ END\n");


    fd = open(fl_nm, O_RDWR | O_DIRECT);
    pwrite(fd, aligned_buf_w, BUF_SIZE, 0);


    //write error checking
    if(nr == -1){
        perror("[error in write 2]\n");
    }


    nr = pread(fd, aligned_buf_r, BUF_SIZE, 0);
    //read error checking
    if(nr == -1){
        perror("[error in read 2]\n");
    }


    printf("AFTER READ BEGIN\n");
    printf("\taligned_buf_r::%ld \n",strlen(aligned_buf_r));
    printf("AFTER READ END\n");


    //error checking for close process
    if(close(fd) == -1){
        perror("[error in close]\n");
    }else{
        printf("[succeeded in close]\n");
    }


    return  0;
}

Here is the output when I read and write 512 bytes

BEFORE READ BEGIN
         aligned_buf_w::512
         aligned_buf_r::0
BEFORE READ END
AFTER READ BEGIN
        aligned_buf_r::512 
AFTER READ END
[succeeded in close]

and here is the result when I try to read 256 bytes

BEFORE READ BEGIN
         aligned_buf_w::256
         aligned_buf_r::0
BEFORE READ END
[error in read 2]
: Invalid argument
AFTER READ BEGIN
        aligned_buf_r::0 
AFTER READ END
[succeeded in close]

Solution

While using O_DIRECT "the kernel will do DMA directly from/to the physical memory pointed by the userspace buffer passed as parameter" - https://www.ukuug.org/events/linux2001/papers/html/AArcangeli-o_direct.html - so you have to observe some restrictions - http://man7.org/linux/man-pages/man8/raw.8.html

All I/Os must be correctly aligned in memory and on disk: they must start at a sector offset on disk, they must be an exact number of sectors long, and the data buffer in virtual memory must also be aligned to a multiple of the sector size. The sector size is 512 bytes for most devices.

With buffered IO you do not care of that. The following sample illustrates that while reading a HDD (/dev/sda9) :

#define _GNU_SOURCE
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#define SECTOR 512

int main( int argc, char **argv ){

int fd, nr, BUF_SIZE;
char fl_nm[]={"/dev/sda9"};
char* buf = NULL;

if (argc>1) {
    BUF_SIZE = atoi(argv[1]);

    // BUFFERED IO
    printf("Buffered IO -------\n");
    if ((buf = (char*)malloc(BUF_SIZE)) == NULL) perror("[malloc]");
    else {
        if ((fd = open(fl_nm, O_RDONLY)) == -1) perror("[open]");

        if((nr = pread(fd, buf, BUF_SIZE, 4096)) == -1) perror("[pread]");
        else
            printf("%i bytes read %.2x %.2x ...\n",nr,buf[0],buf[1]);

        free(buf);

        if(close(fd) == -1) perror("[close]");
    }

    // DIRECT IO
    printf("Direct IO ---------\n");
    if (posix_memalign((void *)&buf, SECTOR, BUF_SIZE)) {
        perror("posix_memalign failed");
    }
    else {
        if ((fd = open(fl_nm, O_RDONLY | O_DIRECT)) == -1) perror("[open]");

        /* buf size , buf alignment and offset has to observe hardware restrictions */
        if((nr = pread(fd, buf, BUF_SIZE, 4096)) == -1) perror("[pread]");
        else
            printf("%i bytes read %.2x %.2x ...\n",nr,buf[0],buf[1]);

        free(buf);

        if(close(fd) == -1) perror("[close]");
    }
}

return  0;
}

You can verify the following behaviour :

$ sudo ./testodirect 512
Buffered IO -------
512 bytes read 01 04 ...
Direct IO ---------
512 bytes read 01 04 ...
$ sudo ./testodirect 4
Buffered IO -------
4 bytes read 01 04 ...
Direct IO ---------
[pread]: Invalid argument

By the way O_DIRECT is not in flavour of everybody https://yarchive.net/comp/linux/o_direct.html