I'm a C++ beginner learning I/O multiplexing.
Here is my code:
test.cpp (doesn't use epoll() and works well):
#include <iostream>
#include <cstdio>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>
int main() {
std::cout << "Hello" << std::endl;
char buffer[1024];
buffer[0] = 'f';
fprintf(stdout, "%s", buffer);
std::cout << "Hello" << std::endl;
int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
// bind & listen
struct sockaddr_in serverAddr;
memset(&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
serverAddr.sin_port = htons(80);
int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
if (bindResult < 0) {
fprintf(stderr, "Fail to bind\n");
return 1;
}
int listenResult = listen(serverFd, 1024);
if (listenResult < 0) {
fprintf(stderr, "Fail to listen\n");
return 1;
}
struct sockaddr clientAddr;
unsigned int clientlen = sizeof(clientAddr);
int acceptFd = accept(serverFd, &clientAddr, &clientlen);
if (acceptFd < 0) {
fprintf(stderr, "Fail to create client connection file descriptor\n");
return 1;
}
int fd = acceptFd;
ssize_t received = recv(fd, &buffer, 1024, 0);
if (received < 0) {
fprintf(stderr, "Fail to received bytess from client\n");
if (errno == EINTR) {
fprintf(stderr, "Reason: EINTR\n");
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
} else {
fprintf(stderr, "Reason: %d\n", errno);
close(fd);
return 1;
}
} else if (received == 0) {
close(fd);
} else {
buffer[received] = '\0';
fprintf(stdout, "%s", buffer);
}
}
test_2.cpp (does use epoll() and doesn't work well):
#include <iostream>
#include <sys/epoll.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <signal.h>
#include <string.h>
int main() {
// TODO: too much error message to handle, so it's necessary to deal with it (maybe macros can)
std::cout << "Hello" << std::endl;
// process ignore SIGPIPE which is caused by send(), or process will exit, which is hard to find out
signal(SIGPIPE, SIG_IGN);
// here needs a socket fd or other fd
// well, AF_INET is okay;socket(PF_INET, SOCK_SEQPACKET, 0) is sctp, tcp cannot use SOCK_SEQPACKET :(
// when using tcp, watch out **record boundaries**
int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (serverFd < 0) {
fprintf(stderr, "Fail to create socket file descriptor\n");
return 1;
}
// nonblock
// nonblock
int flags = fcntl(serverFd, F_GETFL, 0);
if (flags < 0) {
fprintf(stderr, "Fail to get flags\n");
return 1;
}
int setFlagResult = fcntl(serverFd, F_SETFL, flags | O_NONBLOCK);
if (setFlagResult < 0) {
fprintf(stderr, "Fail to set flags\n");
return 1;
}
// bind & listen
struct sockaddr_in serverAddr;
memset(&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
serverAddr.sin_port = htons(80);
int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
if (bindResult < 0) {
fprintf(stderr, "Fail to bind\n");
return 1;
}
int listenResult = listen(serverFd, 1024);
if (listenResult < 0) {
fprintf(stderr, "Fail to listen\n");
return 1;
}
// epoll fd
int epollFd = epoll_create(1);
if (epollFd < 0) {
fprintf(stderr, "Fail to create epoll file descriptor\n");
return 1;
}
// event
struct epoll_event event, events[1024];
event.events = EPOLLIN;
event.data.fd = serverFd;
// ctl
int ctlResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, serverFd, &event);
if (ctlResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
// wait
while (1) {
int event_count = epoll_wait(epollFd, events, 1024, -1);
for (int i = 0; i < event_count; i++) {
struct epoll_event event = events[i];
// accept
if (event.data.fd == serverFd) {
unsigned int clientlen = sizeof(clientAddr);
int acceptFd = accept(serverFd, (struct sockaddr *) &clientAddr, &clientlen);
if (acceptFd < 0) {
fprintf(stderr, "Fail to create client connection file descriptor\n");
fprintf(stderr, "Fail Reason: %d\n", errno);
return 1;
}
// nonblock
int flags = fcntl(acceptFd, F_GETFL, 0);
if (flags < 0) {
fprintf(stderr, "Fail to get flags\n");
return 1;
}
int setFlagResult = fcntl(acceptFd, F_SETFL, flags | O_NONBLOCK);
if (setFlagResult < 0) {
fprintf(stderr, "Fail to set flags\n");
return 1;
}
struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd;
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
// client recv
} else if (event.events & EPOLLIN) {
int fd = event.data.fd;
char buffer[1024+1];
ssize_t received = recv(fd, &buffer, 1024, 0);
if (received < 0) {
fprintf(stderr, "Fail to received bytess from client\n");
if (errno == EINTR) {
fprintf(stderr, "Reason: EINTR\n");
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
} else {
fprintf(stderr, "Reason: %d\n", errno);
close(fd);
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
return 1;
}
} else if (received == 0) {
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
close(fd);
} else {
buffer[received] = '\0';
fprintf(stdout, "%s", buffer);
// if you want to send something...
event.events |= EPOLLOUT;
// here is some data that event can hold
event.data.u32 = (uint32_t) 1;
// you can now send data or just put event in epoll, which is maybe easier
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
}
// client send
} else if (event.events & EPOLLOUT) {
int fd = event.data.fd;
char buffer[] = "I see you";
ssize_t sendResult = send(fd, &buffer, 1024, 0);
if (sendResult < 0) {
fprintf(stderr, "Fail to received bytess from client\n");
if (errno == EINTR) {
fprintf(stderr, "Reason: EINTR\n");
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
} else {
if (errno == EPIPE) {
fprintf(stderr, "Reason: EPIPE\n");
} else {
fprintf(stderr, "Reason: %d\n", errno);
}
close(fd);
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
return 1;
}
} else if (sendResult == 0) {
event.events = EPOLLIN;
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctl\n");
return 1;
}
} else {
// if you want to recv something...
// event.events |= EPOLLIN;
// // you can now send data or just put event in epoll, which is maybe easier
// int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
// if (ctlClientResult < 0) {
// fprintf(stderr, "Fail to run epoll_ctl\n");
// return 1;
// }
}
}
}
}
return 0;
}
When I try to make a TCP socket connection (such as curl -v "http://host:80/", which can make test2.cpp run in line 81), acceptFd is < 0 and errno is 11 according to line 84, which means "Resource deadlock avoided".
Why? There isn't any thread-related code, is it?
Error code 11 is EAGAIN, which is a very common error code to encounter when dealing with non-blocking socket I/O. It means the requested operation has nothing to do, try it again later.
In the case of accept(), it means:
EAGAINorEWOULDBLOCKThe socket is marked nonblocking and no connections are present to be accepted. POSIX.1-2001 and POSIX.1-2008 allow either error to be returned for this case, and do not require these constants to have the same value, so a portable application should check for both possibilities.
This means you are calling accept() at the wrong time, when the listening socket does not have any clients to accept. Double check your epoll() usage, it likely has a logic bug in it that is causing you to call accept() prematurely.
For instance, after accept() has successfully accepted a client, you are assigning the listening socket instead of the client socket to event.data.fd when calling epoll_ctl(EPOLL_CTL_ADD):
struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd; // <-- HERE, SHOULD BE acceptFd INSTEAD!
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);
As such, when the client has data waiting to be read, your loop will end up calling accept() on the listening socket instead of calling recv() on the client socket.
Also, you are not checking the reported event.events field for socket errors. If an error occurs on a client socket, the reported events may include the EPOLLERR and/or EPOLLHUP flags. You should check for those flags and if present then close the client socket, before checking for the EPOLLIN flag to call recv().
Also, note that in your example, it is not necessary to call epoll_ctl(EPOLL_CTL_DEL) before calling close() on a socket. Closing a socket file descriptor will automatically remove it from the epoll instance that is monitoring it.