clinuxsocketsbsd
Sockets behavior differently between BSD (Mac OS X and OpenBSD) and Linux (Ubuntu)
I wrote a man-in-the-middle/proxy server initially on my mac. Essentially the proxy creates a socket and waits for a connection, then connects to another application. This works flawlessly on in OS X and in OpenBSD; however, when porting the proxy to Ubuntu, it doesn't work as intended.
There is two instances of this proxy running, listening on two separate ports. When this is ran on Ubuntu, all the traffic goes through a single port. I also run into a problem when setting the socket to nonblocking (through fcntl) that sometimes it fails with "Invalid Argument"
I am using sys/socket.
Any pitfalls during this port that I am missing?
EDIT:
I believe there are two problems. One being the Invalid Argument, the other being that traffic is being pushed to different ports.
Service 1 binds to Proxy instance 1 which then binds back to the appropriate service on the black box, which kicks off service 2. However, for some reason on Ubuntu it connects to the Instance 1 which is listening on the incorrect port.
EDIT Solution to Invalid Argument for fcntl:
Found out why i was getting the invalid argument, sadly i'm still having the other issue. fcntl(fd, cmd, arg)
cmd - F_SETFL(long)
I was passing in a pointer to an int instead of the long primitive.
EDIT:
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <netdb.h>
#include <string.h>
#include <signal.h>
#include <assert.h>
#include <syslog.h>
#include <sys/types.h>
#include <sys/select.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <sys/param.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <netinet/in.h>
#include <arpa/ftp.h>
#include <arpa/inet.h>
#include <arpa/telnet.h>
void cleanup(int sig)
{
syslog(LOG_INFO, "Cleaning up...");
exit(0);
}
void sigreap(int sig)
{
int status;
pid_t p;
while ((p = waitpid(-1, &status, WNOHANG)) > 0) {
syslog(LOG_INFO, "sigreap: pid=%d, status=%d\n", (int) p, status);
}
/* doh! */
signal(SIGCHLD, sigreap);
}
void set_nonblock(int fd)
{
long fl;
int x;
fl = fcntl(fd, F_GETFL);
if (fl < 0) {
syslog(LOG_ERR, "fcntl F_GETFL: FD %d: %s", fd, strerror(errno));
exit(1);
}
fl |= O_NONBLOCK;
x = fcntl(fd, F_SETFL, fl);
if (x < 0) {
syslog(LOG_ERR, "fcntl F_SETFL: FD %d: %s", fd, strerror(errno));
exit(1);
}
}
int create_server_sock(char *addr, int port)
{
int addrlen, s, on = 1, x;
static struct sockaddr_in client_addr;
s = socket(AF_INET, SOCK_STREAM, 0);
if (s < 0)
perror("socket"), exit(1);
addrlen = sizeof(client_addr);
memset(&client_addr, '\0', addrlen);
client_addr.sin_family = AF_INET;
client_addr.sin_addr.s_addr = INADDR_ANY; //inet_addr(addr);
client_addr.sin_port = htons(port);
setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, 4);
x = bind(s, (struct sockaddr *) &client_addr, addrlen);
if (x < 0)
perror("bind"), exit(1);
x = listen(s, 5);
if (x < 0)
perror("listen"), exit(1);
return s;
}
int open_remote_host(char *host, int port)
{
struct sockaddr_in rem_addr;
int len, s, x;
struct hostent *H;
int on = 1;
H = gethostbyname(host);
if (!H)
return (-2);
len = sizeof(rem_addr);
s = socket(AF_INET, SOCK_STREAM, 0);
if (s < 0)
return s;
setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, 4);
len = sizeof(rem_addr);
memset(&rem_addr, '\0', len);
rem_addr.sin_family = AF_INET;
memcpy(&rem_addr.sin_addr, H->h_addr, H->h_length);
rem_addr.sin_port = htons(port);
x = connect(s, (struct sockaddr *) &rem_addr, len);
if (x < 0) {
close(s);
return x;
}
set_nonblock(s);
return s;
}
int get_hinfo_from_sockaddr(struct sockaddr_in addr, int len, char *fqdn)
{
struct hostent *hostinfo;
hostinfo = gethostbyaddr((char *) &addr.sin_addr.s_addr, len, AF_INET);
if (!hostinfo) {
sprintf(fqdn, "%s", inet_ntoa(addr.sin_addr));
return 0;
}
if (hostinfo && fqdn)
sprintf(fqdn, "%s [%s]", hostinfo->h_name, inet_ntoa(addr.sin_addr));
return 0;
}
int wait_for_connection(int s)
{
int newsock;
socklen_t len;
static struct sockaddr_in peer;
len = sizeof(struct sockaddr);
newsock = accept(s, (struct sockaddr *) &peer, &len);
/* dump_sockaddr (peer, len); */
if (newsock < 0) {
if (errno != EINTR)
perror("accept");
}
get_hinfo_from_sockaddr(peer, len, client_hostname);
set_nonblock(newsock);
return (newsock);
}
static int print_bytes(char * buf, ssize_t length)
{
int i = 0, ascii_off = 0, hex_off = 0;
char * hex_bytes = (char *) calloc(32*2,1);
char * ascii_bytes = (char *) calloc(32*2,1);
for( i = 0; i < length; i++)
{
hex_off += sprintf(hex_bytes+hex_off,"%02X ",(unsigned char)buf[i]);
if(buf[i] >= '!' && buf[i] <= '~')
ascii_off += sprintf(ascii_bytes+ascii_off,"%c ",buf[i]);
else
ascii_off += sprintf(ascii_bytes+ascii_off,". ");
if( ((i+1) % 16 == 0) || i == length-1 )
{
fprintf(stderr,"%-48s\t%s\n",hex_bytes,ascii_bytes);
free(hex_bytes);
free(ascii_bytes);
hex_bytes = (char *) calloc(32*2,1);
ascii_bytes = (char *) calloc(32*2,1);
ascii_off = 0;
hex_off = 0;
if(i != length-1)
fprintf(stderr,"\t");
}
}
free(hex_bytes);
free(ascii_bytes);
return 0;
}
int mywrite(int fd, char *buf, int *len)
{
int x = write(fd, buf, *len);
print_bytes(buf,*len);
if (x < 0)
return x;
if (x == 0)
return x;
if (x != *len)
memmove(buf, buf+x, (*len)-x);
*len -= x;
return x;
}
void service_client(int fd1, int fd2, int injfd)
{
int maxfd;
cfd = fd1;
sfd = fd2;
char *sbuf;
char *cbuf;
int x, n;
int cbo = 0;
int sbo = 0;
int ibo = 0;
fd_set R;
int max_clients = 30;
int i = 0,s = 0, addrlen;
struct sockaddr_in address;
sbuf = malloc(BUF_SIZE);
cbuf = malloc(BUF_SIZE);
cntrlbuf = calloc(1,BUF_SIZE);
char * injbuf = malloc(BUF_SIZE);
maxfd = cfd > sfd ? cfd : sfd;
maxfd = injfd > maxfd ? injfd : maxfd;
maxfd++;
maxfd++;
struct inj_con * ptr;
while (1) {
struct timeval to;
if (cbo) {
process_packet(cbuf,&cbo,sfd);
}
if (sbo) {
process_packet(sbuf,&sbo,cfd);
}
if (ibo) {
process_packet(injbuf,&ibo,cfd);
}
if (cntrlo) {
fprintf(stderr,"\033[33;1mControl->(%d), len = 0x%x (%d):\033[0m\n\t",cfd,cntrlo,cntrlo);
if (mywrite(cfd, cntrlbuf, &cntrlo) < 0 && errno != EWOULDBLOCK) {
syslog(LOG_ERR, "write %d: %s", cfd, strerror(errno));
exit(1);
}
}
FD_ZERO(&R);
if (cbo < BUF_SIZE)
FD_SET(cfd, &R);
if (sbo < BUF_SIZE)
FD_SET(sfd, &R);
if (ibo < BUF_SIZE)
FD_SET(injfd, &R);
to.tv_sec = 0;
to.tv_usec = 1000;
x = select(max_clients+3, &R, 0, 0, &to);
if (x > 0 || cntrl_q->item_count > 0) {
if (FD_ISSET(injfd, &R)) {
int new_socket;
if((new_socket = accept(injfd, (struct sockaddr *) &address, (socklen_t *) &addrlen)) < 0)
{
perror("accept");
exit(1);
}
// Truncated
//
}
char * temp_pkt;
if (FD_ISSET(cfd, &R)) {
temp_pkt = (char *) calloc(BUF_SIZE,1);
n = read(cfd, temp_pkt, BUF_SIZE);
syslog(LOG_INFO, "read %d bytes from CLIENT (%d)", n, cfd);
if (n > 0) {
push_msg(s_q,temp_pkt,n);
} else {
free(temp_pkt);
close(cfd);
close(sfd);
close_injection_sockets();
close(injfd);
_exit(0);
}
}
if (FD_ISSET(sfd, &R)) {
temp_pkt = (char *) calloc(BUF_SIZE,1);
n = read(sfd, temp_pkt, BUF_SIZE);
syslog(LOG_INFO, "read %d bytes from SERVER (%d)\n", n, sfd);
if (n > 0) {
push_msg(c_q,temp_pkt,n);
} else {
free(temp_pkt);
close(sfd);
close(cfd);
close_injection_sockets();
close(injfd);
_exit(0);
}
}
if(cntrlo == 0 && cntrl_q->front != NULL)
{
struct msg * tmp = next_msg(cntrl_q);
if(tmp != NULL)
{
memcpy(cntrlbuf,tmp->msg,tmp->len);
cntrlo += tmp->len;
free(tmp->msg);
free(tmp);
}
}
if(sbo == 0 && c_q->front != NULL)
{
struct msg * tmp = next_msg(c_q);
if(tmp != NULL)
{
memcpy(sbuf,tmp->msg,tmp->len);
sbo += tmp->len;
free(tmp->msg);
free(tmp);
}
}
if(cbo == 0 && s_q->front != NULL)
{
struct msg * tmp = next_msg(s_q);
if(tmp != NULL)
{
memcpy(cbuf,tmp->msg,tmp->len);
cbo += tmp->len;
free(tmp->msg);
free(tmp);
}
}
if(ibo == 0 && inj_q->front != NULL)
{
struct msg * tmp = next_msg(inj_q);
if(tmp != NULL)
{
memcpy(injbuf,tmp->msg,tmp->len);
ibo += tmp->len;
free(tmp->msg);
free(tmp);
}
}
} else if (x < 0 && errno != EINTR) {
close(sfd);
close(cfd);
_exit(0);
}
}
}
static int create_injection_sock(int injectionport)
{
struct sockaddr_in serv_addr;
int portno = injectionport;
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd <0)
{
perror("ERROR: opening socket");
exit(1);
}
bzero((char *) &serv_addr, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(portno);
if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0)
{
perror("ERROR: on bind");
exit(1);
}
if (listen(sockfd,5) < 0 )
{
perror("listen injection");
exit(1);
}
return sockfd;
}
int main(int argc, char *argv[])
{
if (!(5 == argc || 6 == argc)) {
fprintf(stderr, "usage: %s laddr lport rhost rport [injectionport]\n", argv[0]);
exit(1);
}
char *localaddr = strdup(argv[1]);
int localport = atoi(argv[2]);
char *remoteaddr = strdup(argv[3]);
int remoteport = atoi(argv[4]);
int injectionport;
if(argc == 6)
injectionport = atoi(argv[5]);
int client, server;
int master_sock;
int injection_sock = -1;
cntrl_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
inj_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
s_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
c_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
identities = (struct item_queue *) calloc(1,sizeof(struct item_queue));
assert(localaddr);
assert(localport > 0);
assert(remoteaddr);
assert(remoteport > 0);
if(argc == 6)
assert(injectionport > 0);
openlog(argv[0], LOG_PID, LOG_LOCAL4);
signal(SIGINT, cleanup);
signal(SIGCHLD, sigreap);
if(argc == 6)
injection_sock = create_injection_sock(injectionport);
master_sock = create_server_sock(localaddr, localport);
for (;;) {
if ((client = wait_for_connection(master_sock)) < 0)
continue;
if ((server = open_remote_host(remoteaddr, remoteport)) < 0)
continue;
if (!fork()) {
service_client(client, server, injection_sock);
}
close(client);
close(server);
}
}
Solution
Turns out that SO_REUSEADDR socket option was the issue. I removed me setting that socket option and everything worked out.
This Lead me to the solution.
- Is this declaration UB?
- Calling an external C function (in a shared lib) from Perl with Inline/C does not work
- Eclipse C/C++ how to find variable belongs to which struct quickly
- Is this truly best way to delete last element in C?
- Why do we need address virtualization in an operating system?
- Can a real floating-point type alias a complex floating-point type in C?
- Shadowing an iterator inside a for loop has undefined (?) behaviour in C
- Can a C program execute successfully if main() is defined as a macro?
- Ambiguity in scope of for loop declaration versus body
- Understanding the difference in timing of two functions that increment each element of an integer array
- Why MSVC generates warning C4127 when constant is used in "while" - C
- Executing a user-space function from the kernel space
- Why must the variable used to hold getchar's return value be declared as int?
- mpirun -np 4 ./a.out doesn't use all my cores (ubuntu 24.04LTS)
- Sleep for N seconds and wait for keypress
- Dereference twice in gdb
- Is it safe to cast a struct pointer to a different struct pointer having a prefix of elements?
- Can a C compiler legally reject a program if its call stack depth exceeds a fixed limit at compile time?
- return type defaults to 'int' [-Wimplicit-int]
- What is the scope of `fesetround()`?
- Which specific optimization flag causes libm functions to be treated as pure?
- How to render text in SDL2?
- Why would you use 'extern "C++"'?
- Strange Behavior Compiler Ignoring NULL Check Unless I Print Something in the if Statement
- Fast inverse square root using fixed point instead of floating point
- What is the const qualifier attached to in C: the memory area or the pointer?
- GCC options for strictest C code?
- How to do an explicit fall-through in C
- How do compilers treat CONST qualifier when the pointer points to a memory location obtained with malloc()?
- C: cmocka headers - how to unittest?