Communication with Openssl epoll server to multiple clients simultaneously in C/C++

Question

I have an SSL server (code listed below) connecting to multiple SSL client. I am using a single context and the initialization is

SSL_CTX *ctx;
SSL *ssl[MAXEVENTS];
SSL_library_init();
ctx = InitServerCTX();        // initialize SSL
...
...

Then I have the following piece of code

ssl[i] = SSL_new(ctx);           // get new SSL state with context
SSL_set_fd(ssl[i], infd);        // set connection socket to SSL state

Then I perform SSL_accept(ssl[i]).

All this is being performed using epoll (edge trigger mode). I have modified the example in https://banu.com/blog/2/how-to-use-epoll-a-complete-example-in-c/ to use SSL by referring to https://www.cs.utah.edu/~swalton/listings/articles/ssl_server.c as a reference

The logic for this is

 events = new epoll_event[MAXEVENTS * sizeof event];

 // The event loop
 while (true)
 {
    int n, i;

    n = epoll_wait (efd, events, MAXEVENTS, -1);
    for (i = 0; i < n; i++)
    {
        if ((events[i].events & EPOLLERR) ||
                (events[i].events & EPOLLHUP) ||
                (!(events[i].events & EPOLLIN)))
        {
            // An error has occured on this fd, or the socket is not
            // ready for reading (why were we notified then?)
            fprintf (stderr, "epoll error\n");
            close (events[i].data.fd);
            continue;
        } else if (sfd == events[i].data.fd) {
            // We have a notification on the listening socket, which
            // means one or more incoming connections.
            while (1)
            {

                struct sockaddr in_addr;
                socklen_t in_len;
                int infd;
                char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];

                in_len = sizeof in_addr;
                infd = accept (sfd, &in_addr, &in_len);
                if (infd == -1)
                {
                    if ((errno == EAGAIN) ||(errno == EWOULDBLOCK)) {
                        // We have processed all incoming
                        // connections.
                        break;
                    } else {
                        perror ("accept");
                        break;
                    }
                }

                s = getnameinfo (&in_addr, in_len,
                                 hbuf, sizeof hbuf,
                                 sbuf, sizeof sbuf,
                                 NI_NUMERICHOST | NI_NUMERICSERV);
                if (s == 0) {
                    printf("Accepted connection on descriptor %d "
                           "(host=%s, port=%s)\n", infd, hbuf, sbuf);
                }


    ssl[i] = SSL_new(ctx);           // get new SSL state with context
    SSL_set_fd(ssl[i], infd);        // set connection socket to SSL state

    int ret;
    if ( (ret=SSL_accept(ssl[i])) == FAIL ) {    // do SSL-protocol accept
    ERR_print_errors_fp(stderr);
    printf("Performing exchange Error 1.\n");
    int error = SSL_get_error(ssl[i], 0);

    //TODO A retry timer or retry counter. Cannot keep retrying perpetually.
    if (ret <=0 && (error == SSL_ERROR_WANT_READ)) {
        //Need to wait until socket is readable. Take action?
        //LOG the reason here
        perror ("Need to wait until socket is readable.");
    } else if (ret <=0 && (error == SSL_ERROR_WANT_WRITE)) {
        //Need to wait until socket is writable. Take action?
        //LOG the reason here
        perror ("Need to wait until socket is writable.");
    } else {
        //LOG the reason here
        perror ("Need to wait until socket is ready.");
    }
    shutdown (infd, 2);
    SSL_free (ssl[i]);
    continue;
    }
    // Make the incoming socket non-blocking and add it to the
    // list of fds to monitor.
    s = SocketNonBlocking (infd);
    if (s == -1) {
    abort ();
    }
    event.data.fd = infd;
    event.events = EPOLLIN | EPOLLET | EPOLLHUP;
    s = epoll_ctl (efd, EPOLL_CTL_ADD, infd, &event);
    if (s == -1) {
    perror ("epoll_ctl");
    abort ();
    }
}
continue;

Now,

    while (1)
    {
        ssize_t count;
        char buf[1024];
        char reply[1024];

        printf("Performing exchange.\n");
        const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";

        ShowCerts(ssl[i]);        // get any certificates
        count = SSL_read(ssl[i], buf, sizeof(buf)); // get request
        int32_t ssl_error = SSL_get_error (ssl[i], count);
        switch (ssl_error) {
         case SSL_ERROR_NONE: 
                    printf("SSL_ERROR_NONE\n");
                    break;
         case SSL_ERROR_WANT_READ:
                    printf("SSL_ERROR_WANT_READ\n");
                    break;
         case SSL_ERROR_WANT_WRITE:
                    printf("SSL_ERROR_WANT_WRITE\n");
                    break;
         case SSL_ERROR_ZERO_RETURN:
                    printf("SSL_ERROR_ZERO_RETURN\n");  
                    break;
         default:
                    break;
         }

        if (( count > 0 ) )
        {
            buf[count] = 0;
            printf("count > 0 Client msg: \"%s\"\n", buf);
            sprintf(reply, HTMLecho, buf);   // construct reply
            SSL_write(ssl[i], reply, strlen(reply)); // send reply
        } else if ((count < 0)  ){
            printf("count < 0 \n");
            if (errno != EAGAIN)
            {
                printf("count < 0 errno != EAGAIN \n");
                perror ("read");
                done = 1;
            }
            break;
        } else if (count==0){
            ERR_print_errors_fp(stderr);
            epoll_ctl(efd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
            printf("count == 0 Client Disconnected.\n");
            done = 1;
            break;
        }

    }
    if (done)
    {
        printf("Freeing data.\n");
        int sd = SSL_get_fd(ssl[i]);
        SSL_free(ssl[i]);         // release SSL state
        close(sd);          // close connection
        //close (events[i].data.fd);
    }
   }

This works fine for one server - one client. But the moment I try to connect two clients, the client that connected last is the only one that receives the data. The client that was connected before just keeps hanging without any activity.

UPDATE I found that there is some indexing issue going on here. The value of the variable i from the epoll example is not corresponding to what I think it should be corresponding. I tried connecting two clients and I had thought initially the i should have incremented for second client but it is not the case. It still stays 0.

Answer 1

Ok I solved the issue. My problem stemmed from incorrect indexing. I was relying on the variable i that did not behave the way I expected. (see update in my question)

First I declare std::map<int,SSL*> sslPairMap;

Then I insert the successful fd and SSL accept into a std::map in C++. In C, one can use struct based pairing. There is one example here https://github.com/dCache/dcap/blob/b432bd322f0c1cf3e5c6a561845899eec3acad1e/plugins/ssl/sslTunnel.c

                //(c) 2014 enthusiasticgeek for stack overflow

                sslPairMap.insert(std::pair<int,SSL*>(infd, ssl));

                // Make the incoming socket non-blocking and add it to the
                // list of fds to monitor.
                s = AibSocketNonBlocking (infd);
                if (s == -1) {
                    abort ();
                }
                aibevent.data.fd = infd;
                aibevent.events = EPOLLIN | EPOLLET | EPOLLHUP;
                s = epoll_ctl (efd, EPOLL_CTL_ADD, infd, &aibevent);
                if (s == -1) {
                    perror ("epoll_ctl");
                    abort ();
                }

After this I simply retrieve the SSL* from the map which ensures I do not change index inadvertently. std::map saves the day

           //(c) 2014 enthusiasticgeek for stack overflow
           while (1)
            {
                ssize_t count;
                char buf[1024];
                char reply[1024];

                printf("Performing exchange where i = %d.\n",i);
                const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";

                ShowCerts(sslPairMap[aibevents[i].data.fd]);        // get any certificate

                count = SSL_read(sslPairMap[aibevents[i].data.fd], buf, sizeof(buf)); // get request
                ssl_error = SSL_get_error (sslPairMap[aibevents[i].data.fd], count);
                switch (ssl_error) {
                case SSL_ERROR_NONE: 
                            printf("SSL_ERROR_NONE\n");
                            break;
                case SSL_ERROR_WANT_READ:
                            printf("SSL_ERROR_WANT_READ\n");
                            break;
                case SSL_ERROR_WANT_WRITE:
                            printf("SSL_ERROR_WANT_WRITE\n");
                            break;
                case SSL_ERROR_ZERO_RETURN:
                            printf("SSL_ERROR_ZERO_RETURN\n");  
                            break;
                default:
                            break;
                 }

                if (( count > 0 ) )
                {
                    buf[count] = 0;
                    printf("count > 0 Client msg: \"%s\"\n", buf);
                    sprintf(reply, HTMLecho, buf);   // construct reply
                    SSL_write(sslPairMap[aibevents[i].data.fd], reply, strlen(reply)); // send reply
                    break;
                } else if ((count < 0)  ){
                    printf("count < 0 \n");
                    if (errno != EAGAIN)
                    {
                        printf("count < 0 errno != EAGAIN \n");
                        perror ("read");
                        done = 1;
                    }
                    break;
                } else if (count==0){
                    ERR_print_errors_fp(stderr);
                    epoll_ctl(efd, EPOLL_CTL_DEL, aibevents[i].data.fd, NULL);
                    printf("count == 0 Client Disconnected.\n");
                    done = 1;
                    break;
                }

            }
            if (done)
            {
                printf("Freeing data.\n");
                int sd = SSL_get_fd(sslPairMap[aibevents[i].data.fd]);
                if(ssl_error == SSL_ERROR_NONE){
                   SSL_shutdown(sslPairMap[aibevents[i].data.fd]);
                }
                SSL_free(sslPairMap[aibevents[i].data.fd]);         // release SSL state
                close(sd);          // close connection
                //close (aibevents[i].data.fd);
                erase_from_map(sslPairMap, aibevents[i].data.fd);
            }
        }

Answer 2

If anyone comes across this, another way to go about it is to store the SSL* pointer in the event data itself:

events[i].data.u64 = (long long)ssl;

And when you need to read/write from it:

auto ssl = (SSL*)events[i].data.u64;
SSL_read(ssl, someBuffer, sizeof(someBuffer));

For example.