Search code examples
c++openssldtls

Random lack of connection and message delay in DTLS with OpenSSL

Trying to write a server for DTLS that will currently just output the text that it receives. The working client is taken from https://github.com/stepheny/openssl-dtls-custom-bio and it sends and receives to its own server just fine.

However, when it sends to this server something strange is happening. Firstly the connection happens only sometimes, there seems to be no way to determine if the connection will start or not. Secondly, and that is even stranger the data is "delayed". One needs to send 6 messages for 1 message to arrive.

So this is the situation:

  1. Start the server.
  2. Start the client.
  3. Hope for connection.
  4. If connected type 5 messages in client to send to server, they are sent, but the server keeps having an error decoding them.
  5. Once you send the 6th message you can note that the 1st message arrives on server.
  6. Once you send the 7th, you will get the 2nd. Etc.

It should be noted that we are not talking about a time delay, there is no way to simply read 5 empty messages at the start of the server, the queue is empty. Only once the 6th message is sent is the queue populated with the 1st real message.

Code:

//server.cpp
#include "DTLSConnection.hpp"
#include <iostream>
#include <chrono>
#include <thread>

int main(int argc, char *argv[])
{
    try
    {
        
        DTLSConnection con("192.168.31.177:1235");
        std::cout << "Connection created" << std::endl;
        
        ssize_t ret;
        for(;;)
        {
            ret = con.recv([](Client* c) {
                try{
                    std::cout << c->SSL_read_alt() << std::endl;
                    std::cout << "I am in onmessage" << std::endl;
                }
                catch(std::string &e)
                {
                    std::cerr << "EXCEPTION: " << e << std::endl;
                }
            });
            std::cout << "Returned value is " << ret << std::endl;
            std::this_thread::sleep_for(std::chrono::seconds(1));
        }
    }
    catch(std::string &e)
    {
        std::cerr << "EXCEPTION: " << e << std::endl;
    }
    return 0;
}

// CustomBIO.hpp
#include <memory>
#include <deque>
#include <vector>
#include <unordered_map>

#include <cstdio> // temporary

#include <cstring>
#include <cassert>
#include <openssl/ssl.h>
#include <signal.h>

const char *sdump_addr(const struct sockaddr *sa)
{
    static char buf[1024];

    switch (sa->sa_family)
    {
        case AF_INET:
            memmove(buf, "INET: ", 6);
            inet_ntop(AF_INET, &((struct sockaddr_in *)sa)->sin_addr, buf+6, sizeof(buf)-6);
            sprintf(buf+strlen(buf), ":%d", ntohs(((struct sockaddr_in *)sa)->sin_port));
            break;
        case AF_INET6:
            memmove(buf, "INET6: [", 8);
            inet_ntop(AF_INET6, &((struct sockaddr_in6 *)sa)->sin6_addr, buf+8, sizeof(buf)-8);
            sprintf(buf+strlen(buf), "]:%d", ntohs(((struct sockaddr_in6 *)sa)->sin6_port));
            break;
        default:
            memmove(buf, "unknown", 8);
            break;
    }

    return buf;
}

struct Packet
{
    size_t capacity = 0;
    size_t len = 0;
    char * data = nullptr;
    
    Packet() = default;
    Packet(size_t cap)
    {
        init(cap);
    }
    Packet(char *b, size_t cap)
    {
        capacity=cap;
        len=capacity;
        data = new char[capacity];
        memcpy(data, b, capacity);
    }
    Packet(char* b, char* e)
    {
        capacity=e-b;
        len=capacity;
        data = new char[capacity];
        memcpy(data, b, capacity);
    }
    char* begin()
    {
        return data;
    }
    char* end()
    {
        return data ? data+len : nullptr;
    }
    void swap(Packet& that)
    {
        std::swap(this->capacity, that.capacity);
        std::swap(this->len, that.len);
        std::swap(this->data, that.data);
    }
    void init(size_t cap)
    {
        data = new char[cap];
        len = 0;
        capacity = cap;
    }
    void free()
    {
        if(!data) return;
        delete data;
        len = 0;
        capacity = 0;
        data = nullptr;
    }
};

// used for both reading and writing
struct CustomBIO
{
    using dataBuffer = Packet;
    
    int sockfd;
    
    sockaddr_storage thisAddr{};
    socklen_t thisAddr_len{sizeof(sockaddr_storage)};
    
    sockaddr_storage thatAddr{};
    socklen_t thatAddr_len{sizeof(sockaddr_storage)};
    
    template<typename T>
    T* getThat()
    {
        return reinterpret_cast<T*>(&thatAddr);
    }
    
    std::deque<dataBuffer> receivingQueue{};
    bool peekmode{false};
};

inline CustomBIO* BIO_get_CBIO(BIO* b)
{
    return reinterpret_cast<CustomBIO *>(BIO_get_data(b));
}

extern "C"
{
    int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written);
    int BIO_s_custom_write(BIO *b, const char *data, int dlen);
    int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes);
    int BIO_s_custom_read(BIO *b, char *data, int dlen);
    int BIO_s_custom_gets(BIO *b, char *data, int size);
    int BIO_s_custom_puts(BIO *b, const char *data);
    long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs);
    int BIO_s_custom_create(BIO *b);
    int BIO_s_custom_destroy(BIO *b);
    // long BIO_s_custom_callback_ctrl(BIO *, int, BIO_info_cb *);
    
    BIO_METHOD *BIO_s_custom();
    void BIO_s_custom_meth_free();
    
    
    int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written)
    {
        fprintf(stderr, "BIO_s_custom_write_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *written[%ld])\n", (long unsigned int)b, (long unsigned int)data, dlen, *written);
        fflush(stderr);
        
        return -1;
    }
    
    int BIO_s_custom_write(BIO *b, const char *data, int dlen)
    {
        int ret;
        CustomBIO *cbio;
        
        ret = -1;
        fprintf(stderr, "BIO_s_custom_write(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (unsigned long)b, (unsigned long)data, (long)dlen);
        fflush(stderr);
        cbio = BIO_get_CBIO(b);
        
        // dump_addr((struct sockaddr *)&cbio->txaddr, ">> ");
        // dump_hex((unsigned const char *)data, dlen, "    ");
        ret = sendto(cbio->sockfd, data, dlen, 0, cbio->getThat<const sockaddr>(), cbio->thatAddr_len);
        if (ret >= 0)
        {
            fprintf(stderr, "  %d bytes sent\n", ret);
        }
        else
        {
            fprintf(stderr, "  ret: %d errno: [%d] %s\n", ret, errno, strerror(errno));
            fprintf(stderr, "  socket: %d\n", cbio->sockfd);
            fprintf(stderr, "  thatAddrLen: %d\n", cbio->thatAddr_len);
            fprintf(stderr, "  thatAddr: %s\n", sdump_addr(cbio->getThat<sockaddr>()));
        }

        return ret;
    }

    int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes)
    {
        fprintf(stderr, "BIO_s_custom_read_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *readbytes[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen, *readbytes);
        fflush(stderr);
        
        return -1;
    }

    int BIO_s_custom_read(BIO *b, char *data, int dlen)
    {
        int ret;
        CustomBIO *cbio;
        
        ret = -1;
        fprintf(stderr, "BIO_s_custom_read(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen);
        fprintf(stderr, "  probe peekmode %d\n", ((CustomBIO *)BIO_get_data(b))->peekmode);
        fflush(stderr);
        
        cbio = BIO_get_CBIO(b);
        if(!cbio->receivingQueue.empty())
        {
            if(cbio->receivingQueue.front().len > (size_t)dlen)
            {
                fprintf(stderr, "if(cbio->receivingQueue.front().len > (size_t)dlen)");
                memmove(data, cbio->receivingQueue.front().data, dlen);
                
                ret = dlen;
                if(!cbio->peekmode)
                {
                    CustomBIO::dataBuffer rest{cbio->receivingQueue.front().begin()+ret, cbio->receivingQueue.front().end()};
                    cbio->receivingQueue.front().swap(rest);
                }
            }
            else
            {
                Packet &pac = cbio->receivingQueue.front();
                ret = pac.len;
                memmove(data, pac.data, ret);
                
                if(!cbio->peekmode)
                {
                    pac.free();
                    cbio->receivingQueue.pop_front();
                }
            }
            fprintf(stderr, "   %d bytes read from queue\n", ret);
            fflush(stderr);
        }
        else
        {
            fprintf(stderr, "   The queue is empty\n");
            /*ret = recvfrom(cbio->sockfd, data, dlen, 0, cbio->getThat<sockaddr>(), &cbio->thatAddr_len); // not right
            if(ret>0 && cbio->peekmode)
            {
                // todo
            }*/
        }
        
        return ret;
    }
    
    int BIO_s_custom_gets(BIO *b, char *data, int size)
    {
        fprintf(stderr, "BIO_s_custom_gets(BIO[0x%016lX], data[0x%016lX], size[%d]\n", (long unsigned int)b, (long unsigned int)data, size);
        if(size <= 1)
        {
            return 0;
        }
        else
        {
            size = BIO_s_custom_read(b, data, size-1);
            data[size] = '\0';
            return size;
        }
    }
    
    int BIO_s_custom_puts(BIO *b, const char *buf)
    {
        fprintf(stderr, "BIO_s_custom_puts(BIO[0x%016lX], buf[0x%016lX]\n", (long unsigned int)b, (long unsigned int)buf);
        size_t size = std::strlen(buf);
        return size > 0 ? BIO_s_custom_write(b, buf, size) : 0;
    }
    
    long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs)
    {
        long ret = 0;
        
        fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
        if(pargs)
        {
            for(int i=0; ; ++i)
            {
                fprintf(stderr, "[%d]=%X ", i, (int)((unsigned char*)pargs)[i]);
                if(((unsigned char*)pargs)[i] == 0) break;
            }
        }
        fprintf(stderr, "\n");
        fflush(stderr);
        
        switch(cmd)
        {
            case BIO_CTRL_FLUSH: // 11
            case BIO_CTRL_DGRAM_SET_CONNECTED: // 32
            case BIO_CTRL_DGRAM_SET_PEER: // 44
            case BIO_CTRL_DGRAM_GET_PEER: // 46
                ret = 1;
                break;
            case BIO_CTRL_WPENDING: // 13
                ret = 0;
                break;
            case BIO_CTRL_DGRAM_QUERY_MTU: // 40
            case BIO_CTRL_DGRAM_GET_FALLBACK_MTU: // 47
                ret = 1500;
                // ret = 9000; // jumbo?
                break;
            case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD: // 49
                ret = 96; // random guess
                break;
            case BIO_CTRL_DGRAM_SET_PEEK_MODE: // 71
                BIO_get_CBIO(b)->peekmode = (larg != 0);
                ret = 1;
                break;
            case BIO_CTRL_PUSH: // 6
            case BIO_CTRL_POP: // 7
            case BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT: // 45
                ret = 0;
                break;
            default:
                fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
                fprintf(stderr, "  unknown cmd: %d\n", cmd);
                fflush(stderr);
                ret = 0;
                break;
        }
        
        return ret;
    }
    
    int BIO_s_custom_create(BIO *b)
    {
        fprintf(stderr, "BIO_s_custom_create(BIO[0x%016lX])\n", (long unsigned int)b);
        fflush(stderr);
        
        return 1;
    }
    
    int BIO_s_custom_destroy(BIO *b)
    {
        fprintf(stderr, "BIO_s_custom_destroy(BIO[0x%016lX])\n", (long unsigned int)b);
        fflush(stderr);
        
        return 1;
    }
    
    BIO_METHOD *_BIO_s_custom = nullptr;
    BIO_METHOD *BIO_s_custom()
    {
        if (!_BIO_s_custom)
        {
            _BIO_s_custom = BIO_meth_new(BIO_get_new_index()|BIO_TYPE_SOURCE_SINK, "BIO_s_custom");
            
            //BIO_meth_set_callback_ctrl(_BIO_s_custom, BIO_s_custom_callback_ctrl);
            BIO_meth_set_create(_BIO_s_custom, BIO_s_custom_create);
            BIO_meth_set_ctrl(_BIO_s_custom, BIO_s_custom_ctrl);
            BIO_meth_set_destroy(_BIO_s_custom, BIO_s_custom_destroy);
            BIO_meth_set_gets(_BIO_s_custom, BIO_s_custom_gets);
            BIO_meth_set_puts(_BIO_s_custom, BIO_s_custom_puts);
            BIO_meth_set_read_ex(_BIO_s_custom, BIO_s_custom_read_ex);
            BIO_meth_set_read(_BIO_s_custom, BIO_s_custom_read);
            BIO_meth_set_write_ex(_BIO_s_custom, BIO_s_custom_write_ex);
            BIO_meth_set_write(_BIO_s_custom, BIO_s_custom_write);
        }
        
        return _BIO_s_custom;
    }
    
    void BIO_s_custom_meth_free()
    {
        if (_BIO_s_custom)
            BIO_meth_free(_BIO_s_custom);

        _BIO_s_custom = NULL;
    }
}

// DTLSConnection.hpp
#include <string>
#include <list>
#include <functional>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <sys/epoll.h>

#include <cerrno>

#include <iostream> // temp

#include "CustomBIO.hpp"

constexpr int TIME_OUT = 10000; // ms
char cookie_str[] = "BISCUIT!"; // how to change this
//нужен способ, чтобы клиент был извествен (knownclients) но еще не подключен (не прошел ssl_accept())
// (void) BIO_dgram_get_peer(SSL_get_rbio(ssl), &peer);
// see https://github.com/nplab/DTLS-Examples/blob/master/src/dtls_udp_echo.c
int generate_cookie([[maybe_unused]] SSL *ssl, unsigned char *cookie, unsigned int *cookie_len)
{
    memmove(cookie, cookie_str, sizeof(cookie_str)-1);
    *cookie_len = sizeof(cookie_str)-1;

    return 1;
}

int verify_cookie([[maybe_unused]] SSL *ssl, const unsigned char *cookie, unsigned int cookie_len)
{
    return sizeof(cookie_str)-1==cookie_len && memcmp(cookie, cookie_str, sizeof(cookie_str)-1)==0;
}

void throw_SSL_error_if_error(SSL* ssl, int ret, std::string str)
{
    if(ret>0) return; // SSL_ERROR_NONE
    
    str += " ret="+std::to_string(ret)+' ';
    auto sslError = SSL_get_error(ssl, ret);
    if(sslError == SSL_ERROR_SYSCALL){
        throw std::string{str+"SSL_ERROR_SYSCALL + error "}+std::to_string(errno);
    }
}

namespace std
{
    template<> struct hash<sockaddr>
    {
        size_t operator()(sockaddr const& val) const noexcept
        {
            size_t res = 0;
            for(unsigned long h : val.sa_data)
            {
                    res = (res << 1) ^ h;
            }
            return res;
        }
    };
}
bool operator==(const sockaddr& l, const sockaddr& r)
{
    if(l.sa_family != r.sa_family) return false;
    for(int i=0; i<14; ++i)
    {
        if(l.sa_data[i] != r.sa_data[i]) return false;
    }
    return true;
}

class DTLSConnection;

class SSLSetterUpper
{
    friend DTLSConnection;
    SSL_CTX *ctx;
    SSLSetterUpper()
    {
        int ret; // because it is C;
        
        SSL_load_error_strings();
        SSL_library_init();
        
        const SSL_METHOD *mtd = DTLS_server_method();
        ctx = SSL_CTX_new(mtd);
        SSL_CTX_set_min_proto_version(ctx, DTLS1_2_VERSION);
        SSL_CTX_use_certificate_chain_file(ctx, "server-cert.pem");
        SSL_CTX_use_PrivateKey_file(ctx, "server-key.pem", SSL_FILETYPE_PEM);
        ret = SSL_CTX_load_verify_locations(ctx, "root-ca.pem", nullptr);
        if(ret != 1)
        {
            throw std::string{"SSL_CTX_load_verify_locations failed"};
        }
        ret = SSL_CTX_set_default_verify_file(ctx);
        if(ret != 1)
        {
            throw std::string{"SSL_CTX_set_default_verify_file failed"};
        }
        SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
        
        SSL_CTX_set_cookie_generate_cb(ctx, generate_cookie);
        SSL_CTX_set_cookie_verify_cb(ctx, verify_cookie);
    }
    
    SSL* generateSSL() const
    {
        return SSL_new(ctx);
    }
};

struct Client
{
    CustomBIO cbio;
    SSL *ssl;
    
    explicit Client(SSL *ssl)
        :cbio{}, ssl{ssl}
    {
        cbio.peekmode = false;
        
        BIO *bio = BIO_new(BIO_s_custom());
        BIO_set_data(bio, (void *)&cbio);
        BIO_set_init(bio, 1);
        SSL_set_bio(ssl, bio, bio);
    }
    bool on_init() const
    {
        int ret = DTLSv1_listen(this->ssl, nullptr);
        //throw_SSL_error_if_error(ssl, ret, "DTLSv1_listen failed");
        std::cout << "DTLSv1_listen " << ret << std::endl;
        return (ret==1);
    }
    bool on_connect() const
    {
        int ret = SSL_accept(ssl);
        if(ret!=1) return false;
        std::cout << "ssl = " << SSL_state_string_long(ssl) << std::endl;
        std::cout << "SSL_accept successful!" << std::endl;
        return true;
    }
    std::string SSL_read_alt() const
    {
        Packet p(2000);
        std::cout << "sizeA = " << cbio.receivingQueue.size() << std::endl;
        int ret = ::SSL_read(ssl,p.data,p.capacity);
        std::cout << "ret = " << ret << std::endl;
        std::cout << "sizeB = " << cbio.receivingQueue.size() << std::endl;
        std::cout << "pdata0 " << p.data[0] << std::endl;
        //std::cout << SSL_get_error(ssl, ret) << std::endl;
        throw_SSL_error_if_error(ssl, ret, "sslread");
        p.len = ret;
        std::cerr << "plen" << std::endl;
        std::cerr << p.len << std::endl;
        std::string result(p.data,p.len);
        if(result[0] == '\0') std::cout << "res = " << result << std::endl;
        p.free();
        return result;
    }
};

class DTLSConnection
{
    static const SSLSetterUpper sslSetup;
    
    int epoll_fd;
    
    std::unordered_map<sockaddr, std::shared_ptr<Client>> knownClients;
    std::unordered_map<sockaddr, std::shared_ptr<Client>> connectedClients;
    std::shared_ptr<Client> incomingClient;
public:
    // 127.0.0.1:1234 or [::1]:1234
    explicit DTLSConnection(std::string thisAddress)
    {
        std::list<sockaddr_storage> addresses;
        
        if (thisAddress[0]=='[')
        {
            auto pos = thisAddress.find(']', 1);
            if (pos == std::string::npos)
            {
                throw std::string{"invalid target"};
            }
            int port = std::stoi(thisAddress.substr(pos+2));
            if (port<1||port>65535)
            {
                throw std::string{"invalid port"};
            }
            
            addresses.emplace_back();
            auto* thisAddr = (sockaddr_in6 *)&(addresses.back());
            thisAddr->sin6_family = AF_INET6;

            if ( ! inet_pton(AF_INET6, thisAddress.substr(1, pos).c_str(), &thisAddr->sin6_addr) )
            {
                throw std::string{"invalid ipv6 address"};
            }
            thisAddr->sin6_port = htons(port);
        }
        else
        {
            auto pos = thisAddress.find(':');
            if (pos == std::string::npos)
            {
                throw std::string{"invalid target"};
            }
            int port = std::stoi(thisAddress.substr(pos+1));
            if (port<1||port>65535)
            {
                throw std::string{"invalid port"};
            }

            addresses.emplace_back();
            auto * thisAddr = (sockaddr_in *)&(addresses.back());
            thisAddr->sin_family = AF_INET;

            if ( ! inet_pton(AF_INET, thisAddress.substr(0, pos).c_str(), &thisAddr->sin_addr) )
            {
                throw std::string{"invalid ipv4 address"};
            }
            thisAddr->sin_port = htons(port);
        }
        
        epoll_fd = epoll_create1(EPOLL_CLOEXEC);
        
        for(auto &address : addresses)
        {
            epoll_event epe {};
            epe.data.fd = socket(address.ss_family, SOCK_DGRAM/*|SOCK_NONBLOCK*/|SOCK_CLOEXEC, 0);
            if(bind(epe.data.fd, (const sockaddr*)&address, sizeof(address)) != 0)
            {
                throw std::string{"failed to bind"};
            }
            epe.events = EPOLLIN|EPOLLET;
            epoll_ctl(epoll_fd, EPOLL_CTL_ADD, epe.data.fd, &epe);
        }
        
        //signal(SIGINT, signal_handler); // do i need this?
        
        incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
    }
    
    ssize_t recv(const std::function<void(Client *)>& onmessage)
    {
        std::cout << "recv(onmessage)" << std::endl;
        epoll_event epe{};
        
        int ret;
        
        ret = epoll_wait(epoll_fd, &epe, 1, TIME_OUT);
        if (ret==-1)
        {
            throw std::string{"epoll_wait failed"};
        }
        if(ret==0)
        {
            return ret; // wait longer
        }
        
        Packet packet{2000};
        
        ret = recvfrom(epe.data.fd, packet.data, packet.capacity, 0, incomingClient->cbio.getThat<sockaddr>(), &incomingClient->cbio.thatAddr_len);
        packet.len = ret;
        
        if(ret==0) return ret;
        if(ret<0)
        {
            switch(errno)
            {
                case EAGAIN:
                //case EWOULDBLOCK:
                    return ret;
                case EBADF:
                    throw std::string{"EBADF"};
                case ECONNREFUSED:
                    throw std::string{"ECONNREFUSED"};
                case EFAULT:
                    throw std::string{"EFAULT"};
                case EINTR:
                    throw std::string{"EINTR"};
                case EINVAL:
                    throw std::string{"EINVAL"};
                case ENOMEM:
                    throw std::string{"ENOMEM"};
                case ENOTCONN:
                    throw std::string{"ENOTCONN"};
                case ENOTSOCK:
                    throw std::string{"ENOTSOCK"};
                default:
                    throw std::string{"Unknwon errno with negative return from recvfrom: "}+std::to_string(errno);
            }
        }
        auto known = knownClients.find(*incomingClient->cbio.getThat<sockaddr>());
        auto connected = connectedClients.find(*incomingClient->cbio.getThat<sockaddr>());
        
        std::cout << "START" << std::endl;
        for(auto &pair : knownClients)
        {
            std::cout << sdump_addr(&pair.first) << std::endl;
        }
        std::cout << "END" << std::endl;
        if(known == knownClients.end())
        {

            std::cout << "inetaddr = " << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << std::endl;
            ret = 0;
            incomingClient->cbio.receivingQueue.push_back(std::move(packet));
            
            incomingClient->cbio.sockfd = epe.data.fd;
            if( incomingClient->on_init() )
            {
                std::cout << "inc = " << incomingClient->on_connect() << std::endl; //скорее всего, всегда будет ложь
                std::cout << "on_init if" << std::endl;
                knownClients[*incomingClient->cbio.getThat<sockaddr>()] = incomingClient;
                
                incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
            }
        }
        else if(connected == connectedClients.end())
        {
            std::cout << "elseif" << std::endl;
            ret = 0;
            auto cli = known->second;
            cli->cbio.receivingQueue.push_back(std::move(packet));
            
            if( cli->on_connect() )
            {
                std::cout << "cli->cbio.receivingQueue.size()" << cli->cbio.receivingQueue.size() << std::endl;
                connectedClients[*cli->cbio.getThat<sockaddr>()] = cli;
                SSL_write(cli->ssl, "hello", 6);
            }
        }
        else
        {
            std::cout << "else" << std::endl;
            std::cout << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << " has been found as connected" << std::endl;
            connected->second->cbio.receivingQueue.push_back(std::move(packet));
            onmessage(connected->second.get());
        }

        return ret;
    }
};
const SSLSetterUpper DTLSConnection::sslSetup{};

The output from running the server (CustomBIO output truncated) is:

Connection created
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
  probe peekmode 0
   211 bytes read from queue
  36 bytes sent

  probe peekmode 0
   The queue is empty
DTLSv1_listen -1
Returned value is 0
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
  probe peekmode 0
   219 bytes read from queue

DTLSv1_listen 1
  1180 bytes sent
  probe peekmode 0
   The queue is empty
inc = 0
on_init if
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
  86 bytes sent
  823 bytes sent
  167 bytes sent
  79 bytes sent
  25 bytes sent
  probe peekmode 0
   219 bytes read from queue
  probe peekmode 0
   The queue is empty
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
  probe peekmode 0
   1088 bytes read from queue
  618 bytes sent
ssl = SSL negotiation finished successfully
SSL_accept successful!
cli->cbio.receivingQueue.size()0
  43 bytes sent
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
  probe peekmode 0
   824 bytes read from queue
  probe peekmode 0
   The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 824
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
  probe peekmode 0
   58 bytes read from queue
  probe peekmode 0
   The queue is empty
ret = -1
sizeB = 0
pdata0 `
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 58
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
  probe peekmode 0
   131 bytes read from queue
  probe peekmode 0
   The queue is empty
ret = -1
sizeB = 0
pdata0  
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 131
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
  probe peekmode 0
   14 bytes read from queue
  probe peekmode 0
   The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 14
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
   The queue is empty
ret = -1
sizeB = 0
pdata0 
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 61
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
  probe peekmode 0
   55 bytes read from queue
ret = 18
sizeB = 0
pdata0 d
plen
18
dfksaiopfjiaosjfio
I am in onmessage
Returned value is 55
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
  probe peekmode 0
   46 bytes read from queue
ret = 9
sizeB = 0
pdata0 s
plen
9
sdasdasda
I am in onmessage
Returned value is 46
recv(onmessage)
Returned value is 0
recv(onmessage)

Where you see EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0 that means that SSL_read has returned -1. Sorry for somewhat dirty code.

Solution

  • In case somebody else will have a similar issue. The problem was that the wait between calling server's recv function was 1 second. In that time client thought that server has not responded and began doing weird things. Lowering the delay solved the problem.