Libevent code not connecting or timing out on OSX Mountain Lion

Question

Edit: Here is the tcpdump output when running the program on OSX:

Matthew-Mitchell:calm-ocean-4924 matt$ sudo tcpdump -X -i lo0 'port 45564'
Password:
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on lo0, link-type NULL (BSD loopback), capture size 65535 bytes
22:52:41.620969 IP localhost.53685 > localhost.45564: Flags [S], seq 3787197032, win 65535, options [mss 16344,nop,wscale 4,nop,nop,TS val 465807577 ecr 0,sackOK,eol], length 0
    0x0000:  4500 0040 ba9e 4000 4006 0000 7f00 0001  E..@..@.@.......
    0x0010:  7f00 0001 d1b5 b1fc e1bc 0a68 0000 0000  ...........h....
    0x0020:  b002 ffff fe34 0000 0204 3fd8 0103 0304  .....4....?.....
    0x0030:  0101 080a 1bc3 a8d9 0000 0000 0402 0000  ................
22:52:41.621062 IP localhost.45564 > localhost.53685: Flags [S.], seq 1362049502, ack 3787197033, win 65535, options [mss 16344,nop,wscale 4,nop,nop,TS val 465807577 ecr 465807577,sackOK,eol], length 0
    0x0000:  4500 0040 8dac 4000 4006 0000 7f00 0001  E..@..@.@.......
    0x0010:  7f00 0001 b1fc d1b5 512f 39de e1bc 0a69  ........Q/9....i
    0x0020:  b012 ffff fe34 0000 0204 3fd8 0103 0304  .....4....?.....
    0x0030:  0101 080a 1bc3 a8d9 1bc3 a8d9 0402 0000  ................
22:52:41.621075 IP localhost.53685 > localhost.45564: Flags [.], ack 1, win 9186, options [nop,nop,TS val 465807577 ecr 465807577], length 0
    0x0000:  4500 0034 fdb1 4000 4006 0000 7f00 0001  E..4..@.@.......
    0x0010:  7f00 0001 d1b5 b1fc e1bc 0a69 512f 39df  ...........iQ/9.
    0x0020:  8010 23e2 fe28 0000 0101 080a 1bc3 a8d9  ..#..(..........
    0x0030:  1bc3 a8d9                                ....
22:52:41.621086 IP localhost.45564 > localhost.53685: Flags [.], ack 1, win 9186, options [nop,nop,TS val 465807577 ecr 465807577], length 0
    0x0000:  4500 0034 53bf 4000 4006 0000 7f00 0001  E..4S.@.@.......
    0x0010:  7f00 0001 b1fc d1b5 512f 39df e1bc 0a69  ........Q/9....i
    0x0020:  8010 23e2 fe28 0000 0101 080a 1bc3 a8d9  ..#..(..........
    0x0030:  1bc3 a8d9 

I have a rather large library with some networking code using libevent. For a long time this has worked but recently with some newer code that uses the networking code it rarely works because connections cannot be made. Also the connection timeout event is not fired, despite the connection never completing (which it should). I tried to separate code to give an example of this not working. The following code does not work on OSX Mountain Lion (At the least):

#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <event2/event.h>
#include <stdbool.h>
#include <string.h>
#include <errno.h>

#ifdef SO_NOSIGPIPE
#define CB_NOSIGPIPE true
#else
#define CB_NOSIGPIPE false
#define SO_NOSIGPIPE 0
#endif

void loopErr(void * vself);
void loopErr(void * vself){
    printf("Loop error\n");
    exit(EXIT_FAILURE);
}

void timeout(void * vself, void * foo);
void timeout(void * vself, void * foo){
    printf("Timeout\n");
    exit(EXIT_FAILURE);
}

void didConnect(void * vself, void * foo);
void didConnect(void * vself, void * foo){
    printf("Did connect\n");
}

bool CBSocketAccept(int sock, int * connectionSock);
bool CBSocketAccept(int sock, int * connectionSock){
    *connectionSock = accept(sock, NULL, NULL);
    if (*connectionSock == -1)
        return false;
    // Make socket non-blocking
    evutil_make_socket_nonblocking(*connectionSock);
    // Stop SIGPIPE
    int i = 1;
    if (CB_NOSIGPIPE)
        setsockopt(*connectionSock, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof(i));
    return true;
}

void acceptConn(void * vself, int socket);
void acceptConn(void * vself, int socket){
    int connSock;
    if (! CBSocketAccept(socket, &connSock)){
        printf("Unable to accept a connection.\n");
        exit(EXIT_FAILURE);
    }
    printf("Did accept\n");
}

typedef struct{
    struct event_base * base;
    void (*onError)(void *);
    void (*onTimeOut)(void *, void *); /**< Callback for timeouts */
    void * communicator;
    pthread_t loopThread; /**< The thread for the event loop. */
    void  (*userCallback)(void *);
    void * userArg;
}CBEventLoop;

union CBOnEvent{
    void (*i)(void *, int);
    void (*ptr)(void *, void *);
};

typedef struct{
    CBEventLoop * loop; /**< For getting timeout CBLogError */
    struct event * event; /**< libevent event. */
    union CBOnEvent onEvent;
    void * peer;
}CBEvent;

void * CBStartEventLoop(void * vloop);
void * CBStartEventLoop(void * vloop){
    CBEventLoop * loop = vloop;
    // Start event loop
    printf("Starting network event loop.\n");
    if(event_base_dispatch(loop->base) == -1){
        // Error
        loop->onError(loop->communicator);
        return NULL;
    }
    // Break from loop. Free everything.
    event_base_free(loop->base);
    free(loop);
    return NULL;
}
void event_base_add_virtual(struct event_base *); 
bool CBNewEventLoop(CBEventLoop ** loop, void (*onError)(void *), void (*onDidTimeout)(void *, void *), void * communicator);
bool CBNewEventLoop(CBEventLoop ** loop, void (*onError)(void *), void (*onDidTimeout)(void *, void *), void * communicator){
    struct event_base * base = event_base_new();
    // Create dummy event to maintain loop.
    event_base_add_virtual(base);
    // Create arguments for the loop
    *loop = malloc(sizeof(**loop));
    (*loop)->base = base;
    (*loop)->onError = onError;
    (*loop)->onTimeOut = onDidTimeout;
    (*loop)->communicator = communicator;
    // Create thread
    pthread_create(&(*loop)->loopThread, NULL, CBStartEventLoop, *loop);
    return loop;
}

bool CBSocketBind(int * sock, bool IPv6, uint16_t port);
bool CBSocketBind(int * sock, bool IPv6, uint16_t port){
    struct addrinfo hints, *res, *ptr;
    // Set hints for the computer's addresses.
    memset(&hints, 0, sizeof(hints));
    hints.ai_flags = AI_PASSIVE;
    hints.ai_family = IPv6 ? AF_INET6 : AF_INET;
    hints.ai_socktype = SOCK_STREAM;
    // Get host for listening
    char portStr[6];
    sprintf(portStr, "%u", port);
    if (getaddrinfo(NULL, portStr, &hints, &res))
        return false;
    // Attempt to bind to one of the addresses.
    for(ptr = res; ptr != NULL; ptr = ptr->ai_next) {
        if ((*sock = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol)) == -1)
            continue;
        // Prevent EADDRINUSE
        int opt = 1;
        setsockopt(*sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
        if (bind(*sock, ptr->ai_addr, ptr->ai_addrlen) == -1) {
            printf("Bind gave the error %s for address on port %u.\n", strerror(errno), port);
            evutil_closesocket(*sock);
            continue;
        }
        break; // Success.
    }
    freeaddrinfo(res);
    if (ptr == NULL) // Failure
        return false;
    // Prevent SIGPIPE
    int i = 1;
    setsockopt(*sock, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof(i));
    // Make socket non-blocking
    evutil_make_socket_nonblocking(*sock);
    return true;
}

void CBCanAccept(evutil_socket_t sock, short eventNum, void * arg);
void CBCanAccept(evutil_socket_t sock, short eventNum, void * arg){
    CBEvent * event = arg;
    event->onEvent.i(event->loop->communicator, sock);
}

bool CBSocketCanAcceptEvent(CBEvent ** event, CBEventLoop * loop, int sock, void (*onCanAccept)(void *, int));
bool CBSocketCanAcceptEvent(CBEvent ** event, CBEventLoop * loop, int sock, void (*onCanAccept)(void *, int)){
    *event = malloc(sizeof(**event));
    (*event)->loop = loop;
    (*event)->onEvent.i = onCanAccept;
    (*event)->event = event_new((*event)->loop->base, sock, EV_READ|EV_PERSIST, CBCanAccept, *event);
    return true;
}

bool CBSocketAddEvent(CBEvent * event, uint32_t timeout);
bool CBSocketAddEvent(CBEvent * event, uint32_t timeout){
    int res;
    if (timeout) {
        uint32_t secs = timeout / 1000;
        struct timeval time = {secs, (timeout - secs*1000) * 1000};
        res = event_add(event->event, &time);
    }else
        res = event_add(event->event, NULL);
    return ! res;
}

bool CBNewSocket(int * sock, bool IPv6);
bool CBNewSocket(int * sock, bool IPv6){
    // You need to use PF_INET for IPv4 mapped IPv6 addresses despite using the IPv6 format.
    *sock = socket(IPv6 ? PF_INET6 : PF_INET, SOCK_STREAM, 0);
    if (*sock == -1)
        return false;
    // Stop SIGPIPE annoying us.
    int i = 1;
    if (CB_NOSIGPIPE)
        setsockopt(*sock, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof(i));
    // Make address reusable
    setsockopt(*sock, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i));
    // Make socket non-blocking
    evutil_make_socket_nonblocking(*sock);
    return true;
}

bool CBSocketConnect(int sock, uint8_t * IP, bool IPv6, uint16_t port);
bool CBSocketConnect(int sock, uint8_t * IP, bool IPv6, uint16_t port){
    // Create sockaddr_in6 information for a IPv6 address
    int res;
    if (IPv6) {
        struct sockaddr_in6 address;
        memset(&address, 0, sizeof(address)); // Clear structure.
        address.sin6_family = AF_INET6;
        memcpy(&address.sin6_addr, IP, 16); // Move IP address into place.
        address.sin6_port = htons(port); // Port number to network order
        res = connect(sock, (struct sockaddr *)&address, sizeof(address));
    }else{
        struct sockaddr_in address;
        memset(&address, 0, sizeof(address)); // Clear structure.
        address.sin_family = AF_INET;
        memcpy(&address.sin_addr, IP + 12, 4); // Move IP address into place. Last 4 bytes for IPv4.
        address.sin_port = htons(port); // Port number to network order
        res = connect(sock, (struct sockaddr *)&address, sizeof(address));
    }
    if (res < 0 && errno == EINPROGRESS)
        return true;
    return false;
}

void CBDidConnect(evutil_socket_t socketID, short eventNum, void * arg);
void CBDidConnect(evutil_socket_t socketID, short eventNum, void * arg){
    CBEvent * event = arg;
    if (eventNum & EV_TIMEOUT) {
        // Timeout for the connection
        event->loop->onTimeOut(event->loop->communicator, event->peer);
    }else{
        int optval = -1;
        socklen_t optlen = sizeof(optval);
        getsockopt(socketID, SOL_SOCKET, SO_ERROR, &optval, &optlen);
        if (optval){
            // Act as timeout
            printf("Connection error: %s\n", strerror(optval));
            event->loop->onTimeOut(event->loop->communicator, event->peer);
        }else
            // Connection successful
            event->onEvent.ptr(event->loop->communicator, event->peer);
    }
}

bool CBSocketDidConnectEvent(CBEvent ** event, CBEventLoop * loop, int sock, void (*onDidConnect)(void *, void *), void * peer);
bool CBSocketDidConnectEvent(CBEvent ** event, CBEventLoop * loop, int sock, void (*onDidConnect)(void *, void *), void * peer){
    *event = malloc(sizeof(**event));
    (*event)->loop = loop;
    (*event)->onEvent.ptr = onDidConnect;
    (*event)->peer = peer;
    (*event)->event = event_new((*event)->loop->base, sock, EV_TIMEOUT|EV_WRITE, CBDidConnect, *event);
    return true;
}

int main(int argc, const char * argv[]){
    int listeningSocket, connSocket;
    CBEvent *acceptEvent, *connectEvent;
    CBEventLoop *listeningEventLoop, *connEventLoop;
    if (!CBNewEventLoop(&listeningEventLoop, loopErr, timeout, NULL)
        || !CBNewEventLoop(&connEventLoop, loopErr, timeout, NULL)){
        printf("Unable to start event loops.");
        return EXIT_FAILURE;
    }
    if (!CBSocketBind(&listeningSocket, false, 45564)){
        printf("Unable to bind a socket for listening");
        return EXIT_FAILURE;
    }
    if (!CBSocketCanAcceptEvent(&acceptEvent, listeningEventLoop, listeningSocket, acceptConn)) {
        printf("Unable to create an accept event");
        return EXIT_FAILURE;
    }
    if(!CBSocketAddEvent(acceptEvent, 0)){
        printf("Unable to add an accept event\n");
        return EXIT_FAILURE;
    }
    if (listen(listeningSocket, 1) == -1){
        printf("Unable to start listening\n");
        return EXIT_FAILURE;
    }
    // Make socket
    if (!CBNewSocket(&connSocket, false)){
        printf("Socket create fail\n");
        return EXIT_FAILURE;
    }
    // Add event for connection
    if (!CBSocketDidConnectEvent(&connectEvent, connEventLoop, connSocket, didConnect, NULL)) {
        printf("Couldn't create did connect event\n");
        return EXIT_FAILURE;
    }
    if (!CBSocketAddEvent(connectEvent, 1000)) {
        printf("Couldn't add connect event\n");
        return EXIT_FAILURE;
    }
    // Connect
    if (!CBSocketConnect(connSocket, (uint8_t []){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 127, 0, 0, 1}, false, 45564)){
        printf("Couldn't connect\n");
        return EXIT_FAILURE;
    }
    pthread_exit(NULL);
    return 0;
}

When I run this I get:

Starting network event loop.
Starting network event loop.

And nothing else. No connection to accept, no timeout, nothing. However I can confirm that this does work on Linux Mint 13. I also tried a different port number but no luck.

Edit: It appears that I'm getting a similar problem with libev. I've just tried reimplementing the code with libev and I have the same problem. I haven't tried changing the small example above to libev though.

Answer 1

There are several problems with this code.

First, CBNewEventLoop always returns true because it returns (bool)loop (which is the address of one of the stack variables in main). Also, the only error you seem would detect by returning (bool)(*loop) would be a malloc failure, but if malloc fails you need to check for its result before writing to *loop.

Second and more fundamentally, you're running an event loop in one thread and calling methods on it from the main thread. Event-driven systems are not generally thread-safe. You should structure this code as a series of callbacks to be run on the event loop's thread.

Finally, it's pretty rare to need more than one event loop at a time. You should have one event loop for the process and use it for both listening and connecting.