CUDA 5.0: CUBIN and CUBLAS_device, compute capability 3.5

Question

I'm trying to compile a kernel that uses dynamic parallelism to run CUBLAS to a cubin file. When I try to compile the code using the command

nvcc -cubin -m64 -lcudadevrt -lcublas_device -gencode arch=compute_35,code=sm_35 -o test.cubin -c test.cu

I get ptxas fatal : Unresolved extern function 'cublasCreate_v2

If I add the -rdc=true compile option it compiles fine, but when I try to load the module using cuModuleLoad I get error 500: CUDA_ERROR_NOT_FOUND. From cuda.h:

/**
 * This indicates that a named symbol was not found. Examples of symbols
 * are global/constant variable names, texture names, and surface names.
 */
CUDA_ERROR_NOT_FOUND                      = 500,

The kernel code:

#include <stdio.h>
#include <cublas_v2.h>
extern "C" {
__global__ void a() {
    cublasHandle_t cb_handle = NULL;
    cudaStream_t stream;
    if( threadIdx.x == 0 ) {
        cublasStatus_t status = cublasCreate_v2(&cb_handle);
        cublasSetPointerMode_v2(cb_handle, CUBLAS_POINTER_MODE_HOST);
        if (status != CUBLAS_STATUS_SUCCESS) {
            return;
        }
        cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking);
        cublasSetStream_v2(cb_handle, stream);
    }
    __syncthreads();
    int jp;
    double A[3];
    A[0] = 4.0f;
    A[1] = 5.0f;
    A[2] = 6.0f;
    cublasIdamax_v2(cb_handle, 3, A, 1, &jp );
}
}

NOTE: The scope of A is local, so the data at the pointer given to cublasIdamax_v2 is undefined, and so jp ends up as a more or less random value in this code. The correct way to do it would be to have A in global memory.

Host code:

#include <stdio.h>
#include <cuda.h>
#include <cuda_runtime_api.h>

int main() {
    CUresult error;
    CUdevice cuDevice;
    CUcontext cuContext;
    CUmodule cuModule;
    CUfunction testkernel;
    // Initialize
    error = cuInit(0);
    if (error != CUDA_SUCCESS) printf("ERROR: cuInit, %i\n", error);
    error = cuDeviceGet(&cuDevice, 0);
    if (error != CUDA_SUCCESS) printf("ERROR: cuInit, %i\n", error);
    error = cuCtxCreate(&cuContext, 0, cuDevice);
    if (error != CUDA_SUCCESS) printf("ERROR: cuCtxCreate, %i\n", error);
    error = cuModuleLoad(&cuModule, "test.cubin");
    if (error != CUDA_SUCCESS) printf("ERROR: cuModuleLoad, %i\n", error);
    error = cuModuleGetFunction(&testkernel, cuModule, "a");
    if (error != CUDA_SUCCESS) printf("ERROR: cuModuleGetFunction, %i\n", error);
    return 0;
}

The host code is compiled using nvcc -lcuda test.cpp. If I replace the kernel with a simple kernel (below) and compile it without -rdc=true, it works fine.

Simple working kernel

#include <stdio.h>
extern "C" {
__global__ void a() {
    printf("hello\n");
}
}

Thanks in advance

• Soren

Answer 1

You are just missing -dlink in your first approach:

nvcc -cubin -m64 -lcudadevrt -lcublas_device -gencode arch=compute_35,code=sm_35 -o test.cubin -c test.cu -dlink

You can also do that in two steps:

nvcc -m64 test.cu -gencode arch=compute_35,code=sm_35 -o test.o -dc
nvcc -dlink test.o -arch sm_35 -lcublas_device -lcudadevrt -cubin -o test.cubin