Shared socket descriptor invalid over message queue when socket opened after queue

Question

I am trying to share a socket descriptor with another local process on the same Linux machine. These processes are "unrelated" i.e., they are not parent/child related and are not forked. They are independent. Ultimately, the flow I want is something like:

| [Process1]
|  -> master_socket = socket()
|  -> setsockopt(master_socket...)
|  -> fcntl(master_socket...)
|  -> bind(master_socket...)
|  -> listen(master_socket...)
|  -> Share master_socket with Process2
|
| [Process2]
| -> Receive and store master_socket
| -> Use select() on master_socket
| -> Use accept() on master_socket to receive connections...

Based on a few related threads, it seems as though this is possible using a Unix domain socket which will track that the socket handle was sent from Process1 to Process2 in the kernel, giving it permission (e.g., here, here, and here).

What I am trying to determine is whether or not the descriptor is possible to share over a POSIX message queue. Oddly enough, if I create the socket BEFORE opening the queue, it seems to work OK. However, if I create the socket AFTER opening the queue, the descriptor read on Process2 comes up as "invalid."

---

Example Programs

Here is a sample program that sends a socket descriptor via a message queue. If init_socket() is called before open_queue(), then the descriptor received is valid. If visa-versa then it comes across invalid.

send.c: gcc -o send send.c -lrt

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <unistd.h>
#include <syslog.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/fcntl.h>
#include <string.h>
#include <errno.h>
#include <mqueue.h>

int init_socket();
int open_queue();
mqd_t msgq_id;

int main() {
  int socket;
  char msg_str[16];

  // HERE: ordering matters.  If open_queue() is done before init_socket(), then
  // the descriptor is received invalid.  If init_socket() is called BEFORE open_queue()
  // then the descriptor received is valid.
  open_queue();
  socket = init_socket();

  // Put the socket on the queue
  memset(msg_str, '\0', sizeof(msg_str));
  snprintf(msg_str, sizeof(msg_str), "%d", socket);
  if(mq_send(msgq_id, msg_str, strlen(msg_str)+1, 1) == -1) {
    printf("Unable to send the message on the queue: %s\n", strerror(errno));
    return -1;
  }
}

int open_queue() {

    // Create a queue to share the socket
    if(msgq_id = mq_open("/share_socket", O_RDWR | O_CREAT | O_EXCL, S_IRWXU | S_IRWXG, NULL)==-1) {

        if(errno != EEXIST) {
            printf("Failed to create IPC queue: %s\n", strerror(errno));
            return -1;
        }

        // Re-open the already existing queue.
        if((msgq_id = mq_open("/share_socket", O_RDWR)) != -1) {
            printf("Re-opened the IPC queue: %s\n", "/share_socket");
        } else {
            printf("Failed to re-open IPC queue %s: %s\n", "/share_socket", strerror(errno));
            return -1;
        }
    }
    return 1;
}
int init_socket() {
    int master_socket;
    int opt=1;

    struct sockaddr_in loc_addr = { 0 };

    // Create the high level master socket
    if( (master_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) {
        printf("Unable to create master_socket\n");
        return EXIT_FAILURE;
    }

    // Set socket to accept multiple connections
    if( setsockopt(master_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) {
        printf("Error setting socket to accept multiple connections\n");
        return EXIT_FAILURE;
    }

    // Set the socket type
    bzero(&loc_addr, sizeof(struct sockaddr_in));
    loc_addr.sin_family = AF_INET;
    loc_addr.sin_addr.s_addr = htonl(INADDR_ANY);
    loc_addr.sin_port=htons(1200);

    // Set the socket to nonblocking
    if (fcntl(master_socket, F_SETFL, O_NDELAY) < 0) {
        printf("Can't set socket to non-blocking\n");
        return EXIT_FAILURE;
    }

    // Bind to the socket
    if (bind(master_socket, (struct sockaddr *)&loc_addr, sizeof(loc_addr)) < 0) {
        return EXIT_FAILURE;
    }

    // Now, listen for a maximum of 6 pending clients
    if(listen(master_socket, 6) < 0) {
        printf("Could not set the socket to listen\n");
        close(master_socket);
        return EXIT_FAILURE;
    }

    return master_socket;
}

read.c: gcc -o read read.c -lrt

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <unistd.h>
#include <syslog.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/fcntl.h> 
#include <string.h>
#include <errno.h>
#include <mqueue.h>

#define MAX_MSG_LEN 10000

int main() {
  mqd_t msgq_id;
  int master_socket;
  unsigned int sender;
  int bytes;
  char msgcontent[MAX_MSG_LEN];

  // Re-open the already existing queue.
  if((msgq_id = mq_open("/share_socket", O_RDWR)) != -1) {
    printf("Re-opened the IPC queue: %s\n", "/share_socket");
  } else {
    printf("Failed to re-open IPC queue %s: %s\n", "/share_socket", strerror(errno));
    return -1;
  }

  // Try to read from the queue.
  if((bytes = mq_receive(msgq_id, msgcontent, MAX_MSG_LEN, &sender)) == -1)
  {
    // If the failure was due to there being no messages, just return 0.
    if(errno==EAGAIN)
      return 0;

    printf("Unable to read from the queue\n");
    return -1;
  }

  sscanf(msgcontent, "%d", &master_socket);

  printf("Got master socket value: %d\n", master_socket);

  if(master_socket != 0 && (fcntl(master_socket, F_GETFD) != -1 || errno != EBADF))
    printf("... socket is valid\n");
  else
    printf("... SOCKET IS INVALID\n");

  return 1;
}

---

How to run: Just go ahead and run ./read and it will wait on the message queue and then run ./send. If you compiled send.c with init_socket() before open_queue(), you will see:

$ ./read 
Re-opened the IPC queue: /share_socket
Got master socket value: 3
... socket is valid

Otherwise:

$ ./read 
Re-opened the IPC queue: /share_socket
Got master socket value: 4
... SOCKET IS INVALID

What would cause this behavior where the ordering is important?

Answer 1

Problem One:

Both programs will start with file descriptors 0, 1, & 2 open and valid.

When you run send with init_socket first the returned file descriptor for the socket witl be 3. Then you open the queue which will be 4. You send 3 to read process.

In read you open the queue which will be file descriptor 3. You read the queue and find you were sent a 3, which is actually the fd of the queue. So you are "testing" the queue file descriptor.

Conversely when in send you open the queue first and then the socket you are sending the file descriptor 4 to read. There is no file descriptor 4 open in read (the queue will still be opened as 3) so it naturally fails the "test".

Problem 2:

The test is wrong.

if (master_socket != 0 && (fcntl(master_socket, F_GETFD) != -1 || errno != EBADF))
    printf("... socket is valid\n");
else
    printf("... SOCKET IS INVALID\n");

All this is telling you is that it isn't file descriptor 0 and you can read the file descriptor flags. A test that the message queue file descriptor will pass.

A better test is something like this:

void isSocket(int fd)
{
    struct stat statbuf;

    if (fstat(fd, &statbuf) == -1)
    {
        perror("fstat");
        exit(1);
    }

    if (S_ISSOCK(statbuf.st_mode))
        printf("%d is a socket\n", fd);
    else
        printf("%d is NOT a socket\n", fd);
}

I haven't tested your pgms but if you print out all your file descriptors from both programs as you open them and try the above test it should bare out what I am saying.