CPU Affinity Masks (Putting Threads on different CPUs)

Question

I have 4 threads, and I am trying to set thread 1 to run on CPU 1, thread 2 on CPU 2, etc. However, when I run my code below, the affinity masks are returning the correct values, but when I do a sched_getcpu() on the threads, they all return that they are running on CPU 4.

Anybody know what my problem here is?

Thanks in advance!

#define _GNU_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
#include <sched.h>
#include <errno.h>

void *pthread_Message(char *message)
{
    printf("%s is running on CPU %d\n", message, sched_getcpu());
}

int main()
{
    pthread_t thread1, thread2, thread3, thread4;
    pthread_t threadArray[4];
    cpu_set_t cpu1, cpu2, cpu3, cpu4;
    char *thread1Msg = "Thread 1";
    char *thread2Msg = "Thread 2";
    char *thread3Msg = "Thread 3";
    char *thread4Msg = "Thread 4";
    int thread1Create, thread2Create, thread3Create, thread4Create, i, temp;

    CPU_ZERO(&cpu1);
    CPU_SET(1, &cpu1);
    temp = pthread_setaffinity_np(thread1, sizeof(cpu_set_t), &cpu1);
    printf("Set returned by pthread_getaffinity_np() contained:\n");
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu1))
            printf("CPU1: CPU %d\n", i);

    CPU_ZERO(&cpu2);
    CPU_SET(2, &cpu2);
    temp = pthread_setaffinity_np(thread2, sizeof(cpu_set_t), &cpu2);
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu2))
            printf("CPU2: CPU %d\n", i);

    CPU_ZERO(&cpu3);
    CPU_SET(3, &cpu3);
    temp = pthread_setaffinity_np(thread3, sizeof(cpu_set_t), &cpu3);
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu3))
            printf("CPU3: CPU %d\n", i);

    CPU_ZERO(&cpu4);
    CPU_SET(4, &cpu4);
    temp = pthread_setaffinity_np(thread4, sizeof(cpu_set_t), &cpu4);
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu4))
            printf("CPU4: CPU %d\n", i);

    thread1Create = pthread_create(&thread1, NULL, (void *)pthread_Message, thread1Msg);
    thread2Create = pthread_create(&thread2, NULL, (void *)pthread_Message, thread2Msg);
    thread3Create = pthread_create(&thread3, NULL, (void *)pthread_Message, thread3Msg);
    thread4Create = pthread_create(&thread4, NULL, (void *)pthread_Message, thread4Msg);

    pthread_join(thread1, NULL);
    pthread_join(thread2, NULL);
    pthread_join(thread3, NULL);
    pthread_join(thread4, NULL);

    return 0;
}

Answer 1

You're trying to set the affinity of threads that you did not initialize.

Edit: Ok, let me give you some more info:

Don't mix thread handles (the thing you store in the pthread_t variable) and what they represent (a thread of execution that runs somewhere). What you were trying to do is to set a property of a thread before it starts, with an API that requires the thread object. As it happens pthread_create creates the object and starts the execution at the same time, so trying to use pthread_setaffinity_np is not the right way to go (this is useful if you want to change the affinity of a currently running thread).

But... pthread_create has an attribute parameter (you're passing NULL to it). This is storing the information of how you want the thread to be created.

Affinity is one of the attributes you can set through that parameter. See the man-page documentation for pthread_attr_init and pthread_attr_setaffinity_np for how exactly

Answer 2

Here's what you were looking for. I know it is a late answer, but this might help others.

#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
#include <sched.h>
#include <errno.h>

#include <unistd.h>

int   getNumberOfCpus( void )
{
    long nprocs       = -1;
    long nprocs_max   = -1;

    # ifdef _SC_NPROCESSORS_ONLN
    nprocs = sysconf( _SC_NPROCESSORS_ONLN );
    if ( nprocs < 1 )
    {
        //std::cout << "Could not determine number of CPUs on line. Error is  " << strerror( errno ) << std::endl;
        return 0;
    }

    nprocs_max = sysconf( _SC_NPROCESSORS_CONF );

    if ( nprocs_max < 1 )
    {
        //std::cout << "Could not determine number of CPUs in host. Error is  " << strerror( errno ) << std::endl;
        return 0;
    }

    //std::cout << nprocs < " of " << nprocs_max << " online" << std::endl;
    return nprocs; 

#else
    //std::cout << "Could not determine number of CPUs" << std::endl;
    return 0;
#endif
}

void *pthread_Message( void *ptr )
{
    sleep(10);
    char *message;
    message = (char *) ptr;
    printf("%s \n", message);
    cpu_set_t      l_cpuSet;
    int            l_maxCpus;
    int            j;
    unsigned long  l_cpuBitMask;

    CPU_ZERO( &l_cpuSet );
    printf("get affinity %d\n",pthread_getaffinity_np(pthread_self()  , sizeof( cpu_set_t ), &l_cpuSet ));
    // printf("cpuset %d\n",l_cpuSet);
    printf (" thread id %u\n", pthread_self());      

    if ( pthread_getaffinity_np(pthread_self()  , sizeof( cpu_set_t ), &l_cpuSet ) == 0 )
        for (int i = 0; i < 4; i++)
            if (CPU_ISSET(i, &l_cpuSet))
                printf("XXXCPU: CPU %d\n", i);
    for (long i=0; i< 10000000000; ++i);
}

int main()
{
    pthread_t thread1, thread2, thread3, thread4;
    pthread_t threadArray[4];
    cpu_set_t cpu1, cpu2, cpu3, cpu4;
    const char *thread1Msg = "Thread 1";
    const char *thread2Msg = "Thread 2";
    const char *thread3Msg = "Thread 3";
    const char *thread4Msg = "Thread 4";
    int thread1Create, thread2Create, thread3Create, thread4Create, i, temp;

    thread1Create = pthread_create(&thread1, NULL, &pthread_Message, (void*)thread1Msg);
    sleep(1);
    thread2Create = pthread_create(&thread2, NULL, &pthread_Message, (void*)thread2Msg);
    sleep(1);
    thread3Create = pthread_create(&thread3, NULL, &pthread_Message, (void*)thread3Msg);
    sleep(1);
    thread4Create = pthread_create(&thread4, NULL, &pthread_Message, (void*)thread4Msg);


    CPU_ZERO(&cpu1);
    CPU_SET(0, &cpu1);
    temp = pthread_setaffinity_np(thread1, sizeof(cpu_set_t), &cpu1);
    printf("setaffinity=%d\n", temp);
    printf("Set returned by pthread_getaffinity_np() contained:\n");
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu1))
            printf("CPU1: CPU %d\n", i);

    CPU_ZERO(&cpu2);
    CPU_SET(1, &cpu2);
    temp = pthread_setaffinity_np(thread2, sizeof(cpu_set_t), &cpu2);
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu2))
            printf("CPU2: CPU %d\n", i);

    CPU_ZERO(&cpu3);
    CPU_SET(2, &cpu3);
    temp = pthread_setaffinity_np(thread3, sizeof(cpu_set_t), &cpu3);
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu3))
            printf("CPU3: CPU %d\n", i);

    CPU_ZERO(&cpu4);
    CPU_SET(3, &cpu4);
    temp = pthread_setaffinity_np(thread4, sizeof(cpu_set_t), &cpu4);
    for (i = 0; i < CPU_SETSIZE; i++)
        if (CPU_ISSET(i, &cpu4))
            printf("CPU4: CPU %d\n", i);

    // pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpu1);



    // pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpu1);

    pthread_join(thread1, NULL);
    pthread_join(thread2, NULL);
    pthread_join(thread3, NULL);
    pthread_join(thread4, NULL);

    return 0;
}

Answer 3

I think the easiest would be to give the CPU mask as a parameter to each thread and have the thread request given affinity itself, as in example here: pthread_setaffinity_np(3).