Hope the included C code is not too long. I've tried to pare it down to the bare minimum to display the problem I'm having. I'm using asynchronous I/O to write a small amount of data. The main code calls aio_write and then waits for the I/O callback to set a global variable before returning. I know this is not the usual way of doing things, but I am trying to test something related to the much larger code that is giving me the real problem.
This code compiles and runs just fine under Cray or GNU.
Under Intel, it compiles and runs just fine with optimization "-O1". But an Intel build with any higher optimization level hangs in the wait loop. The callback does set the global variable but the main code never sees it happen.
Under GNU, for instance, the output is something like,
> head -n 32 /dev/urandom > random_stuff.dat # put random data into test file
> ./write_only 10 # run the code; read and write ten doubles from the input file
reader read 5.991429e-16
writer wrote 80 bytes
writer received 5.991429e-16
writer thinks no_more_writes is 1
ran in 7.650000e-04 sec
With Intel, however, (compiled at "-O2" or higher), I get
reader read 5.991429e-16
writer wrote 80 bytes
writer received 5.991429e-16
writer thinks no_more_writes is 1
main code still waiting
main code still waiting
main code still waiting
main code still waiting
....
I am rather new to asynchronous I/O. I know there are issues with global variables, like race conditions. But that one issue should not arise in this simple code. Any idea what is wrong, and why it is wrong only for an optimized Intel build? (BTW, if I replace the global variable wait with the usual wait based on aio_error(), everything works fine for all compilers, all optimization levels. I think, though, for the larger, real code I am working on, I need something more like what I am doing here.)
Here's the code:
#include <stdio.h>
#include <stdlib.h>
#include <aio.h>
#include <fcntl.h>
#include <unistd.h>
#include<strings.h>
/* for EINPROGRESS value: */
#include <errno.h>
/* for timing */
#include <sys/time.h>
void writer(sigval_t sigval);
int chunksize, numelems;
double *alldata;
int no_more_writes=0;
int main(int argc, char **argv) {
struct aiocb aiowriter;
int fdin, fdout;
struct timeval start_time, end_time;
long int time_usec;
double time_sec;
char *infile = "random_stuff.dat";
if (argc < 2) {
printf("./write_only numelems\n");
return 1;
}
numelems = atoi(argv[1]);
chunksize = numelems * sizeof(double);
// allocate alldata
alldata = (double*)calloc(numelems, sizeof(double));
// open up input file and read
fdin = open(infile, O_RDONLY);
read(fdin, alldata, chunksize);
close(fdin);
printf("reader read %e \n", alldata[0]);
// writing
// open up output file
fdout = open("write_out.dat", O_WRONLY | O_CREAT, S_IRUSR | S_IRGRP);
// Check is opening of output file was successful:
if (fdout == -1) {
printf("cannot open 'write_out.dat'\n");
if (errno == EACCES || errno == EEXIST) {
printf("file aleady exists\n");
} else {
printf("errno is %d\n", errno);
}
return errno;
}
// bzero out aiowriter
bzero((char*)&aiowriter, sizeof(struct aiocb));
// set writer file desc. and other related info
aiowriter.aio_fildes = fdout;
aiowriter.aio_offset = 0;
aiowriter.aio_nbytes = chunksize;
// set writer buffer
aiowriter.aio_buf = (char*)alldata;
// set writer callback info
aiowriter.aio_sigevent.sigev_notify = SIGEV_THREAD;
aiowriter.aio_sigevent.sigev_notify_function = writer;
aiowriter.aio_sigevent.sigev_notify_attributes = NULL;
aiowriter.aio_sigevent.sigev_value.sival_ptr = &aiowriter;
// timing
gettimeofday(&start_time, NULL);
// the write
aio_write(&aiowriter);
int debugindex=0;
// wait for final writing to finish
while (no_more_writes != 1) {
usleep(100);
if (debugindex > 10000) {
printf("main code still waiting\n");
debugindex=1;
}
debugindex++;
}
// alternative wait loop:
//while (aio_error(&aiowriter) == EINPROGRESS) {}
// timing
gettimeofday(&end_time, NULL);
time_usec = 1000000*(end_time.tv_sec - start_time.tv_sec) +
(end_time.tv_usec - start_time.tv_usec);
time_sec = time_usec / 1000000.;
printf("ran in %e sec\n", time_sec);
free(alldata);
}
// write function
void writer(sigval_t sigval) {
int write_error;
// inputs: the AIO writing object
struct aiocb *writing;
writing = (struct aiocb *)sigval.sival_ptr;
// check amount of data written
write_error = aio_return(writing);
printf("writer wrote %d bytes\n", write_error);
printf("writer received %e\n", alldata[0]);
close(writing->aio_fildes);
no_more_writes = 1;
printf("writer thinks no_more_writes is %d\n",
no_more_writes);
return;
}
You need to set the global variable to volatile otherwise the compiler can optimize redundant loads, because it will not know that the variable changed from the outside.
volatile int no_more_writes = 0;