Unable to unwind stack frame ...when debugging with cuda-gdb

Hi!

I’ve wrote a kernel that make some calculations and use shared memory to improve performance. A sketch of kernel is the following:

__global__ void GALfilterKern(kernArgs *kArgs)

{

	__shared__ Complex shared_sig[BLOCK_SIZE];

	__shared__ Complex shared_sig_d[BLOCK_SIZE];

	int i,n,m;

	unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x;

		

	// ===== INPUT PARAMETERS ===

	const float delta = 1e-2;   // small positive constant for "desired response"

	

	// ===== INITIALIZATION =====

	float absE_f = 0.0;

	float absE_b = 0.0;

	

	// ===== APPLICATION TO INPUT SIGNAL FOR EACH SAMPLE===

	for(n=0; n<kArgs->sig_length/BLOCK_SIZE; ++n){

		// data in

		shared_sig[idx].real = kArgs->sig[idx].real;

		shared_sig[idx].img = kArgs->sig[idx].img;

		shared_sig_d[idx].real = kArgs->sig_d[idx].real;

		shared_sig_d[idx].img = kArgs->sig_d[idx].img;

		

		__syncthreads();

		

		// forward and backward error initialization

		kArgs->E_f[0].real = shared_sig[idx].real;

		kArgs->E_f[0].img = shared_sig[idx].img;

		kArgs->E_b[1][0].real = shared_sig[idx].real;

		kArgs->E_b[1][0].img = shared_sig[idx].img;

		....

}

The problem here is when execute kArgs->E_f[0].real = shared_sig[idx].real: it returns me warning: unable to unwind stack frame and it never ends. I’ve also noticed that when the kernel is launched, cuda-gdb gives me a SIGTRAP. I don’t know what’s the problem. If you need other code, ask for it. Thanks.

I’ve noticed that message unwind stack frame ends after a while and cuda-gdb returns me “Cannot find bounds of current function”:

(cuda-gdb) bt

#0  0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1  0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2  0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3  0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4  0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5  0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6  0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7  0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8  0x000000000040472f in cudaLaunch<char> (

	entry=0x402e1c "UH11�H03�

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

20H11}�H13}��35�����20UH11�H03�

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

20�

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

17

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

21E�H13E�H11E��

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

17

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

20E��-���f

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

17(��

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

17

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

20

(cuda-gdb) bt

#0 0x00002ac2322703f8 in ?? () from /usr/lib64/libcuda.so

#1 0x00002ac2322475e0 in ?? () from /usr/lib64/libcuda.so

#2 0x00002ac232246ed2 in ?? () from /usr/lib64/libcuda.so

#3 0x00002ac232232e08 in ?? () from /usr/lib64/libcuda.so

#4 0x00002ac2322225b3 in ?? () from /usr/lib64/libcuda.so

#5 0x00002ac2322ae49f in ?? () from /usr/lib64/libcuda.so

#6 0x00002ac231f6fe97 in ?? () from /usr/local/cuda/lib64/libcudart.so.3

#7 0x00002ac231f6129a in cudaLaunch () from /usr/local/cuda/lib64/libcudart.so.3

#8 0x000000000040472f in cudaLaunch (

entry=0x402e1c "UH\211�H\203�\020H\211}�H\213}��\235�����\220UH\211�H\203�\020�\017\021E�H\213E�H\211E��\017\020E��-���f\017(��\017\020\005�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9 0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

05�L") at /usr/local/cuda/bin/../include/cuda_runtime.h:763

#9  0x0000000000402e19 in __device_stub__Z13GALfilterKernP8kernArgs (__par0=0xa140000)

	at /tmp/tmpxft_00007686_00000000-4_main.cudafe1.stub.c:10

#10 0x0000000000402e31 in GALfilterKern (__cuda_0=0xa140000) at main.cu:4

#11 0x0000000000404245 in main () at main.cu:196

P.S.: I’m using a Tesla C2050 on a x64 Linux machine with kernel 2.6.18-164

I’ve made other trials: seems to be some problem on kernArgs struct. It’s strange because if I try to print device struct values (when I’m inside kernel), it prints all normally. In main.cu I’ve allocated memory in this way:

...

kernArgs *dArgs,*hArgs;

hArgs = allocHostArgs();  // initialize all hArgs members

cudaMalloc((void **) &dArgs, sizeof(kernArgs));	

cudaMemcpy(dArgs,hArgs,sizeof(kernArgs),cudaMemcpyHostToDevice);

...

Please, help me. :rolleyes:

I’ve made other trials: seems to be some problem on kernArgs struct. It’s strange because if I try to print device struct values (when I’m inside kernel), it prints all normally. In main.cu I’ve allocated memory in this way:

...

kernArgs *dArgs,*hArgs;

hArgs = allocHostArgs();  // initialize all hArgs members

cudaMalloc((void **) &dArgs, sizeof(kernArgs));	

cudaMemcpy(dArgs,hArgs,sizeof(kernArgs),cudaMemcpyHostToDevice);

...

Please, help me. :rolleyes:

To explain better what happens with cuda-gdb:

Program received signal SIGTRAP, Trace/breakpoint trap.

[Current CUDA Thread <<<(0,0),(0,0,0)>>>]

0x000000001b2b7cb0 in GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:21

21			shared_sig[idx].real = kArgs->sig[idx].real;

(cuda-gdb) n

[Current CUDA Thread <<<(0,0),(0,0,0)>>>]

GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:22

22			shared_sig[idx].img = kArgs->sig[idx].img;

(cuda-gdb) 

[Current CUDA Thread <<<(0,0),(0,0,0)>>>]

GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:23

23			shared_sig_d[idx].real = kArgs->sig_d[idx].real;

(cuda-gdb) 

[Current CUDA Thread <<<(0,0),(0,0,0)>>>]

GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:24

24			shared_sig_d[idx].img = kArgs->sig_d[idx].img;

(cuda-gdb) 

[Current CUDA Thread <<<(0,0),(0,0,0)>>>]

GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:26

26			__syncthreads();

(cuda-gdb)

GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:29

29			kArgs->E_f[0].real = shared_sig[idx].real;

(cuda-gdb) n

warning: Unable to unwind stack frame.

warning: Unable to unwind stack frame.

warning: Unable to unwind stack frame.

// Here I've sent a SIGINT signal

Program received signal SIGINT, Interrupt.

[Current CUDA Thread <<<(3,0),(0,0,0)>>>]

0x000000001b2b7cd0 in GALfilterKern <<<(1024,1),(256,1,1)>>> (kArgs=0xa140000) at main.cu:21

21			shared_sig[idx].real = kArgs->sig[idx].real;

After a while it can’t go on and returns me “Cannot find bounds of current function” with the output posted earlier.

Now, it works. The problem was that when I returned from kernel, I can’t read from dArgs (obviously). In fact all my problems arise from this, so I thought depends on kernel. The behaviour of cuda-gdb seems to be absolutely normal.