Non-compatible function addresses between gcc compiler flags and symbol table

Question

I have compiled a project (openssh here but that is not important) with

CFLAGS="-ggdb3 -O0 -lm -finstrument-functions"

The symbol table is (small extract):

nm -o somepath/sbin/sshd
/mypath/install/sbin/sshd:000f3548 d auth_method
/mypath/install/sbin/sshd:0001a90f t auth_openfile
/mypath/install/sbin/sshd:0001ab90 T auth_openkeyfile
/mypath/install/sbin/sshd:0001ac31 T auth_openprincipals
/mypath/install/sbin/sshd:0001d73a T auth_parse_options
/mypath/install/sbin/sshd:0000e362 T auth_password

What I obtain when I print the functions addresses in __cyg_profile_func_enter|exit is:

0xb768f8ee
0xb768f66c
0xb768f66c
0xb76d9ae8

Obviously not the same range, confirmed after having looked at all figures.

What I understand is that nm provides offset addresses. What about __cyg_profile_func_enter when there are many files in the project? Is there an additional conversion to perform?

From the gcc documentation, there should not:

          void __cyg_profile_func_enter (void *this_fn,
                                         void *call_site);
          void __cyg_profile_func_exit  (void *this_fn,
                                         void *call_site);

The first argument is the address of the start of the current function, which may be looked up exactly in the symbol table.

So, what do I miss to make it work?

EDIT: my ptrace.c where I have added a mutex

#if (__GNUC__>2) || ((__GNUC__ == 2) && (__GNUC_MINOR__ > 95))

#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <string.h>
#include <sys/errno.h>
#include <pthread.h>

#define PTRACE_PIPENAME  "TRACE"

#define REFERENCE_OFFSET "REFERENCE:"
#define FUNCTION_ENTRY   "enter"
#define FUNCTION_EXIT    "exit"
#define END_TRACE        "EXIT"
#define __NON_INSTRUMENT_FUNCTION__    __attribute__((__no_instrument_function__))
#define PTRACE_OFF        __NON_INSTRUMENT_FUNCTION__
#define STR(_x)          #_x
#define DEF(_x)          _x
#define GET(_x,_y)       _x(_y)
#define TRACE __GNU_PTRACE_FILE__

pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;

/** Initial trace open */
static FILE *__GNU_PTRACE_FILE__;

/** Final trace close */
static void
__NON_INSTRUMENT_FUNCTION__
gnu_ptrace_close(void)
{
    fprintf(TRACE, END_TRACE " %ld\n", (long)getpid());

        int res = fflush(TRACE);
        if (res < 0) {
            printf("Erreur sur gnu_ptrace_close / fflush\n");
        }

    if (TRACE != NULL)
        fclose(TRACE);

        pthread_mutex_destroy(&lock);

    return ;
}

/** Trace initialization */
static int
__NON_INSTRUMENT_FUNCTION__
gnu_ptrace_init(void)
{
    struct stat sta;
    __GNU_PTRACE_FILE__ = NULL;

    /* See if a trace file exists */
    if (stat(PTRACE_PIPENAME, &sta) != 0) 
    {
        /* No trace file: do not trace at all */
        return 0;
    }
    else 
    {
        /* trace file: open up trace file */
        if ((TRACE = fopen(PTRACE_PIPENAME, "a")) == NULL)
        {
            char *msg = strerror(errno);
            perror(msg);
            printf("[gnu_ptrace error]\n");
            return 0;
        }

        #ifdef PTRACE_REFERENCE_FUNCTION
        fprintf(TRACE,"%s %s %p\n",
                  REFERENCE_OFFSET,
                  GET(STR,PTRACE_REFERENCE_FUNCTION),
                  (void *)GET(DEF,PTRACE_REFERENCE_FUNCTION));
        #endif

        /* Tracing requested: a trace file was found */
        atexit(gnu_ptrace_close);
        return 1;
    }
}

/** Function called by every function event */
void
__NON_INSTRUMENT_FUNCTION__
gnu_ptrace(char * what, void * p)
{
    static int first=1;
    static int active=1;
    int res = 0;

    if (active == 0)
        return;

    if (first)
    {
        active = gnu_ptrace_init();
            first = 0;

            if (active == 0)
            return;
    }

        if (!what || !p) {
            printf("Erreur, what ou p NULL\n");
            return;
        }

        if (strcmp(what, FUNCTION_ENTRY) && strcmp(what, FUNCTION_EXIT)) {
            printf("Erreur, what incorrect\n");
            return;
        }

        printf("----------------------- what=%s     p=%p\n", what, p);

        res = fprintf(TRACE, "%s %p\n", what, p);
        if (res < 0) {
            printf("Erreur sur gnu_ptrace / fprintf\n");
/*            active = 0;*/
        }

        if (res > 0) res = fflush(TRACE);
        if (res < 0) {
            printf("Erreur sur gnu_ptrace / fflush\n");
            active = 0;
        }

    return;
}

/** According to gcc documentation: called upon function entry */
void
__NON_INSTRUMENT_FUNCTION__
__cyg_profile_func_enter(void *this_fn, void *call_site)
{
        pthread_mutex_lock(&lock);
    gnu_ptrace(FUNCTION_ENTRY, this_fn);
    (void)call_site;
        pthread_mutex_unlock(&lock);
}

/** According to gcc documentation: called upon function exit */
void
__NON_INSTRUMENT_FUNCTION__
__cyg_profile_func_exit(void *this_fn, void *call_site)
{
        pthread_mutex_lock(&lock);
    gnu_ptrace(FUNCTION_EXIT, this_fn);
    (void)call_site;
        pthread_mutex_unlock(&lock);
}

#endif

Here is the modified ptrace.c with JHiant ideas. I have linked with -ldl

#include <dlfcn.h>

_

Dl_info finfo;
if (!dladdr(p, &finfo))
    printf("Erreur, dladdr\n");

    /* res = fprintf(TRACE, "%s %p\n", what, p); */
    res = fprintf(TRACE, "%s %p %s (%s)\t\n", what, p, finfo.dli_sname, finfo.dli_fname);

Here is the obtained trace (first lines):

enter 0xb7494974 OPENSSL_cpuid_setup (/home/laurent/Documents/projet/install/lib/libcrypto.so.1.0.0)    
exit 0xb7494974 OPENSSL_cpuid_setup (/home/laurent/Documents/projet/install/lib/libcrypto.so.1.0.0) 
enter 0xb76ac470 main (/home/laurent/Documents/projet/install/sbin/sshd)    
enter 0xb76b34fe (null) (/home/laurent/Documents/projet/install/sbin/sshd)  
exit 0xb76b34fe (null) (/home/laurent/Documents/projet/install/sbin/sshd)   
enter 0xb7749384 (null) (/home/laurent/Documents/projet/install/sbin/sshd)  
enter 0xb770496e (null) (/home/laurent/Documents/projet/install/sbin/sshd)  
enter 0xb77046ec (null) (/home/laurent/Documents/projet/install/sbin/sshd)  
exit 0xb77046ec (null) (/home/laurent/Documents/projet/install/sbin/sshd)   
enter 0xb774eb68 (null) (/home/laurent/Documents/projet/install/sbin/sshd)  
exit 0xb774eb68 (null) (/home/laurent/Documents/projet/install/sbin/sshd)   
exit 0xb770496e (null) (/home/laurent/Documents/projet/install/sbin/sshd)   
exit 0xb7749384 (null) (/home/laurent/Documents/projet/install/sbin/sshd)   

Other lines show null all the time instead of the function name. I checked sshd maps, the right libcrypto build with symbols is used. What is surprising in the trace above is that it shows only one call to libcrypto, which looks like simply a setup as the name shows it.

OpenSSH is compiled with (extract of one line of the make execution in the console - all have the same options - normaly without -fpic, but the resulting trace is the same):

gcc -ggdb3 -O0 -lm -finstrument-functions -ldl -fpic -Wall -Wpointer-arith -Wuninitialized -Wsign-compare -Wformat-security -Wno-pointer-sign -Wno-unused-result -fno-strict-aliasing -D_FORTIFY_SOURCE=2 -ftrapv -fno-builtin-memset -fstack-protector-all -fPIE  -I. -I. -I/home/laurent/Documents/projet/install/include  -DSSHDIR=\"/home/laurent/Documents/projet/install/etc\" -D_PATH_SSH_PROGRAM=\"/home/laurent/Documents/projet/install/bin/ssh\" -D_PATH_SSH_ASKPASS_DEFAULT=\"/home/laurent/Documents/projet/install/libexec/ssh-askpass\" -D_PATH_SFTP_SERVER=\"/home/laurent/Documents/projet/install/libexec/sftp-server\" -D_PATH_SSH_KEY_SIGN=\"/home/laurent/Documents/projet/install/libexec/ssh-keysign\" -D_PATH_SSH_PKCS11_HELPER=\"/home/laurent/Documents/projet/install/libexec/ssh-pkcs11-helper\" -D_PATH_SSH_PIDDIR=\"/var/run\" -D_PATH_PRIVSEP_CHROOT_DIR=\"/var/empty\" -DHAVE_CONFIG_H -c ssh-keyscan.c

OpenSSL is compiled with (extract of one line of the make execution in the console - all have the same options):

gcc -I.. -I../.. -I../modes -I../asn1 -I../evp -I../../include  -ggdb3 -O0 -lm -finstrument-functions -ldl  -fPIC -DOPENSSL_PIC -DOPENSSL_THREADS -D_REENTRANT -DDSO_DLFCN -DHAVE_DLFCN_H -Wa,--noexecstack -DL_ENDIAN -DTERMIO -fomit-frame-pointer -Wall -DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_IA32_SSE2 -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_GF2m -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DMD5_ASM -DRMD160_ASM -DAES_ASM -DVPAES_ASM -DWHIRLPOOL_ASM -DGHASH_ASM   -c -o cms_lib.o cms_lib.c

Why aren't there more traces from OpenSSL/libcrypto?

Why aren't there functions names in the trace except for "main"?

Answer 1

A couple ideas:

1. Create helloworld.c and check that the nm for that syncs up with what you're seeing. in your profile code. It could be an issue with the way sshd is built, not your code.
2. Try adding -fpic to your CFLAGS.
3. Obligatory: Just to make sure, you are compiling sshd from source, not linking it, since function instrumentation is inserted during the compilation step, not the link step. It would otherwise just print addresses that are not the ones from the sshd lib, such as the address of main() or some helper functions. You may consider adding functio names to the debug output via:

 

#include <dlfcn.h>
__cyg_profile_func_enter(void *func, void *site)
{
    DL_info finfo;
    dladdr(fun, &finfo);
    fprintf(log, "Entered &s", finfo.dli_sname);
}

Answer 2

When you load and execute an executable (or a shared library) it will be loaded to an arbitrary address and the linker relocates the addresses of all symbols (like your functions) relative to the load address of the executable. These addresses may vary from one run of the executable to another due to Address space layout randomization You can think of the addresses shown by nm as relative addresses (towards the loading location of the executable).

In order to print the addresses like nm does you need to know the loading address and subtract it from the address given in your profiling function. As I am not familiar with the profiling API you are using I don't know if there's API to find that relocation offset.