adding "-rpath,/usr/lib" in the build options of a shared library cause a segfault

Question

I have a hello world program.

#include <stdio.h>
#include <stdlib.h>

int main()
{
    printf("hello world! \n");
    return 0;
}

I add -lmicroxml in the build of the program in the linkage phase in order to link to the library libmicroxml.so

when I launch my program I get a segmentation fault. the segmentation fault is related to the load of the libmicroxml.so. here after the strace of my helleo world program execution:

 strace ./test
execve("./test", ["./test"], [/* 11 vars */]) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777de000
stat("/etc/ld.so.cache", 0x7f944760)    = -1 ENOENT (No such file or directory)
open("/lib/libmicroxml.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib/libmicroxml.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/libmicroxml.so.1", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=4129, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777dd000
read(3, "\177ELF\1\2\1\0\0\0\0\0\0\0\0\0\0\3\0\10\0\0\0\1\0\0\4p\0\0\0004"..., 4096) = 4096
old_mmap(NULL, 69632, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x777b3000
old_mmap(0x777b3000, 1572, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x777b3000
old_mmap(0x777c3000, 1648, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x777c3000
close(3)                                = 0
munmap(0x777dd000, 4096)                = 0
open("/lib/libgcc_s.so.1", O_RDONLY)    = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=78232, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777dd000
read(3, "\177ELF\1\2\1\0\0\0\0\0\0\0\0\0\0\3\0\10\0\0\0\1\0\0006\320\0\0\0004"..., 4096) = 4096
old_mmap(NULL, 147456, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7778f000
old_mmap(0x7778f000, 76928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x7778f000
old_mmap(0x777b2000, 408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x13000) = 0x777b2000
close(3)                                = 0
munmap(0x777dd000, 4096)                = 0
open("/lib/libc.so.0", O_RDONLY)        = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=413076, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777dd000
read(3, "\177ELF\1\2\1\0\0\0\0\0\0\0\0\0\0\3\0\10\0\0\0\1\0\0\253`\0\0\0004"..., 4096) = 4096
old_mmap(NULL, 503808, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x77714000
old_mmap(0x77714000, 405592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x77714000
old_mmap(0x77787000, 7572, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x63000) = 0x77787000
old_mmap(0x77789000, 21036, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x77789000
close(3)                                = 0
munmap(0x777dd000, 4096)                = 0
open("/usr/lib/libgcc_s.so.1", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=169712, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777dd000
read(3, "\177ELF\1\2\1\0\0\0\0\0\0\0\0\0\0\3\0\10\0\0\0\1\0\0\307\220\0\0\0004"..., 4096) = 4096
old_mmap(NULL, 237568, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x776da000
old_mmap(0x776da000, 169036, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x776da000
old_mmap(0x77713000, 1776, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x29000) = 0x77713000
close(3)                                = 0
munmap(0x777dd000, 4096)                = 0
open("/usr/lib/libc.so.0", O_RDONLY)    = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=425968, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777dd000
read(3, "\177ELF\1\2\1\0\0\0\0\0\0\0\0\0\0\3\0\10\0\0\0\1\0\0\267`\0\0\0004"..., 4096) = 4096
old_mmap(NULL, 516096, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7765c000
old_mmap(0x7765c000, 418924, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x7765c000
old_mmap(0x776d2000, 8176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x66000) = 0x776d2000
old_mmap(0x776d4000, 21784, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x776d4000
close(3)                                = 0
munmap(0x777dd000, 4096)                = 0
open("/lib/libc.so.0", O_RDONLY)        = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=413076, ...}) = 0
close(3)                                = 0
stat("/lib/ld-uClibc.so.0", {st_mode=S_IFREG|0755, st_size=28976, ...}) = 0
open("/lib/libc.so.0", O_RDONLY)        = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=413076, ...}) = 0
close(3)                                = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x777dd000
set_thread_area(0x777e4440)             = 0
mprotect(0x77787000, 4096, PROT_READ)   = 0
mprotect(0x776d2000, 4096, PROT_READ)   = 0
mprotect(0x777da000, 4096, PROT_READ)   = 0
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
+++ killed by SIGSEGV +++
Segmentation fault

In the build of the libmicroxml library I found they use DSOFLAGS=-Wl,-soname,libmicroxml.so.1,-rpath,/usr/lib -shared -fPIC in the build of the library (in the linkage phase).

I removed the -rpath,/usr/lib from the options so the new one is DSOFLAGS=-Wl,-soname,libmicroxml.so.1 -shared -fPIC

And then I rebuild the library and then I launched the hello world program and the segmentation fault disappear.

I m building with mips_gcc-4.6-linaro_uClibc-0.9.33.2

This issue is not reproduced with my old gcc mips_gcc-4.3.3+cs_uClibc-0.9.30.1

Can any one explain why removing the -rpath,/usr/lib from the linkage options fix the segfault in the load of the library?

Answer 1

From the trace it seems your program loads different binaries of the same modules, which are supposed to be of the same version:

/lib/libc.so.0 (size: 413076 bytes) vs. /usr/lib/libc.so.0 (size: 425968 bytes).

/lib/libgcc_s.so.1 (size: 78232 bytes) vs. /usr/lib/libgcc_s.so.1 (size: 169712 bytes).

This probably happens because when you use the -rpath in the module link, you force it to load the modules from /usr/lib, but the default search path which your program use is /lib (According to the dlopen documentation at http://tldp.org/HOWTO/Program-Library-HOWTO/dl-libraries.html).

So: Your program loads /usr/lib/libmicroxml.so.1 (note that it couldn't find /lib/libmicroxml.so.1, although it searched for this path first). Then it continues to load the modules it requires (libgcc and libc) from /lib, and finally, because libmicroxml requires loading these modules from /usr/lib (because of the build arguments supplied), they are also loaded from this path.

Once you have two different libraries with the same name and interface (because they're of the same version) loaded together, you can't know which version of which function is called, and this can cause inconsistencies.

I guess you can solve this either the way you did, or by adding the same -rpath argument to the build of your program.

The reason removing -rpath solves this issue is that when loading the required modules for libmicroxml, the loader first searches /lib as the default first directory (since no other directory was specified), and since the modules in this folder are already loaded there's no conflict.

In any case, this situation of having two different binaries for the same module with the same version on the same drive is very unhealthy.

As for the GCC version, I can only assume that the correct libc or libgcc were used (or even installed) with the previous GCC, and replaced by the newer GCC, but I could not find documentation to support that.

Answer 2

Your DSOFLAGS looks like this:

DSOFLAGS=-Wl,-soname,libmicroxml.so.1,-rpath,/usr/lib -shared -fPIC

Did you try compiling like this?

gcc -L/usr/lib -Wl,-rpath=/usr/lib -Wall -o test main.c -lmicroxml

You can then use this line and CC and CFLAGS in a Makefile to make compiling simple, if you're able to compile it this way. There a couple of other ways to link, as well.

Good information at this link.