Difference between memory indirect call and register indirect call

Question

What is the difference between memory indirect call and register indirect call?

I'm trying to learn something about linux rootkit detection, how can I recognize such calls in disassembled memory? How do they look in C language before compiling?

Answer 1

An indirect branch is a branch where the branch is made to an address that is stored in a register or in a memory location. The operand of the branch instruction is the register or the memory location that stores the address to branch.

See wikipedia page for more information: http://en.wikipedia.org/wiki/Indirect_branch

In C depending on the implementation (and the CPU), an indirect branch is often made when a function is called through a function pointer. And as some heuristics for switch statements use function pointers (through jump tables), indirect branches can also be found in switch statements.

Answer 2

Memory indirect call is a call which takes the address of the callee from the memory and register indirect call takes the address from the register accordingly.

Implementation of calls is very dependent on the architecture and compiler used. On x86, both memory and register calls are possible, so it's up to compiler to choose what's optimal.

Take a look on a simple test:

#include "stdlib.h"
#include "stdio.h"
#include "time.h"

void example_fun(int param)
{
    printf("%d\n", param);
}

void fn1(void)
{
    printf("fn1\n");
}

void fn2(void)
{
    printf("fn2\n");
}

typedef void (*fn)(void);

int main(void)
{
    void (*fp) (int) = &example_fun;
    int c;
    fn fn_arr[] = {&fn1, &fn2};

    (*fp)(2);
    srand(time(NULL));
    c = rand() / (RAND_MAX / 2);
    switch (c)
    {
        case 0: (*(fn_arr[0]))(); break;
        case 1: (*(fn_arr[1]))(); break;
        default: (*fp)(1);
    }
}

The generated assembly is (gcc 4.6.1 w/o any optimizations):

    .file   "indirect.c"
    .section    .rodata
.LC0:
    .string "%d\n"
    .text
    .globl  example_fun
    .type   example_fun, @function
example_fun:
.LFB0:
    .cfi_startproc
    pushl   %ebp
    .cfi_def_cfa_offset 8
    .cfi_offset 5, -8
    movl    %esp, %ebp
    .cfi_def_cfa_register 5
    subl    $24, %esp
    movl    $.LC0, %eax
    movl    8(%ebp), %edx
    movl    %edx, 4(%esp)
    movl    %eax, (%esp)
    call    printf
    leave
    .cfi_restore 5
    .cfi_def_cfa 4, 4
    ret
    .cfi_endproc
.LFE0:
    .size   example_fun, .-example_fun
    .section    .rodata
.LC1:
    .string "fn1"
    .text
    .globl  fn1
    .type   fn1, @function
fn1:
.LFB1:
    .cfi_startproc
    pushl   %ebp
    .cfi_def_cfa_offset 8
    .cfi_offset 5, -8
    movl    %esp, %ebp
    .cfi_def_cfa_register 5
    subl    $24, %esp
    movl    $.LC1, (%esp)
    call    puts
    leave
    .cfi_restore 5
    .cfi_def_cfa 4, 4
    ret
    .cfi_endproc
.LFE1:
    .size   fn1, .-fn1
    .section    .rodata
.LC2:
    .string "fn2"
    .text
    .globl  fn2
    .type   fn2, @function
fn2:
.LFB2:
    .cfi_startproc
    pushl   %ebp
    .cfi_def_cfa_offset 8
    .cfi_offset 5, -8
    movl    %esp, %ebp
    .cfi_def_cfa_register 5
    subl    $24, %esp
    movl    $.LC2, (%esp)
    call    puts
    leave
    .cfi_restore 5
    .cfi_def_cfa 4, 4
    ret
    .cfi_endproc
.LFE2:
    .size   fn2, .-fn2
    .globl  main
    .type   main, @function
main:
.LFB3:
    .cfi_startproc
    pushl   %ebp
    .cfi_def_cfa_offset 8
    .cfi_offset 5, -8
    movl    %esp, %ebp
    .cfi_def_cfa_register 5
    andl    $-16, %esp
    subl    $32, %esp
    movl    $example_fun, 24(%esp)
    movl    $fn1, 16(%esp)
    movl    $fn2, 20(%esp)
    movl    $2, (%esp)
    movl    24(%esp), %eax
    call    *%eax
    movl    $0, (%esp)
    call    time
    movl    %eax, (%esp)
    call    srand
    call    rand
    movl    %eax, %ecx
    movl    $-2147483645, %edx
    movl    %ecx, %eax
    imull   %edx
    leal    (%edx,%ecx), %eax
    movl    %eax, %edx
    sarl    $29, %edx
    movl    %ecx, %eax
    sarl    $31, %eax
    movl    %edx, %ecx
    subl    %eax, %ecx
    movl    %ecx, %eax
    movl    %eax, 28(%esp)
    movl    28(%esp), %eax
    testl   %eax, %eax
    je  .L6
    cmpl    $1, %eax
    je  .L7
    jmp .L9
.L6:
    movl    16(%esp), %eax
    call    *%eax
    jmp .L10
.L7:
    movl    20(%esp), %eax
    call    *%eax
    jmp .L10
.L9:
    movl    $1, (%esp)
    movl    24(%esp), %eax
    call    *%eax
.L10:
    leave
    .cfi_restore 5
    .cfi_def_cfa 4, 4
    ret
    .cfi_endproc
.LFE3:
    .size   main, .-main
    .ident  "GCC: (Ubuntu/Linaro 4.6.1-9ubuntu3) 4.6.1"
    .section    .note.GNU-stack,"",@progbits

In all cases our calls were translated into call *%eax, which is a memory indirect call executing the function located at address pointed by %eax. Adding optimization messes the structure, but mechanics remains the register call.

Calling a shared dynamic library (see example here) resulted in just call <function_name>. Consulting the x86 Developer Manual, call accepts either register, either memory, or pointer operand. Such a call is being treated by linker on startup and the name of the function is being substituted by the actual memory address. So, calling a shared library results (in the end) in a memory indirect call.

Apart from such easily reproduced cases, you can encounter operations with immediate address, such as call dword ptr ds:[00923030h] (described here) or analogous jmp. I can't provide a reasonable explanation where do they come from.

Distinguishing between register and memory calls is pretty easy then: just look on the operand.

If you're interested in rootkit detection, it might be a good idea to just take some existing rootkit and look on it's sources and generated assembly side-by-side.

I'm not very experienced with compilers and architecture so I can mistake things a little.

Hope it helps.