Variable length arrays in C, how are they compiled

Question

I have some given code that uses variable length arrays on the stack in C. I cannot easily change that code to use malloced buffers on the heap, since I am working on some stuff where I have no operating system support for dynamic memory. However when I test my method the stack actually get's smashed instead (i.e. the SP get's set to a completely bogus address). To figure out what is going on I had a look at the assembly and I was completely confused by the compiler output.

I am running the code on a pandaboard ES (OMAP4460 ARM processor).

In order to test how variable size arrays on the stack are actually compiled, I produced a simple working test program. However again the code was much too confusing for me to understand.

Here is what I do not understand:

When I compile a simple function:

int test_method(size_t i)
{
   unsigned char buffer[10];
   return -1;
}

I get some very simple assembler code:

80003a68 <test_method>:
80003a68:       b480            push    {r7}        ; store frame pointer
80003a6a:       b087            sub     sp, #28     ; make stack frame (size 28)
80003a6c:       af00            add     r7, sp, #0  ; set frame pointer
80003a6e:       6078            str     r0, [r7, #4]; store parameter on stack
80003a70:       f04f 0300       mov.w   r3, #0      ; load return value
80003a74:       4618            mov     r0, r3      ; put return value in return register
80003a76:       f107 071c       add.w   r7, r7, #28 ; destroy stack frame
80003a7a:       46bd            mov     sp, r7      ; ...
80003a7c:       bc80            pop     {r7}        ; pop stored frame pointer
80003a7e:       4770            bx      lr          ; return

But when I try this with a variable size array using the following code:

int test_method(size_t i)
{
    unsigned char buffer[i];
    return 0;
}

I get this assembly:

80003a68 <test_method>:
80003a68:       e92d 03f0       stmdb   sp!, {r4, r5, r6, r7, r8, r9}
80003a6c:       b084            sub     sp, #16
80003a6e:       af00            add     r7, sp, #0
80003a70:       6078            str     r0, [r7, #4]
80003a72:       4669            mov     r1, sp
80003a74:       460e            mov     r6, r1
80003a76:       f8d7 c004       ldr.w   ip, [r7, #4]
80003a7a:       4661            mov     r1, ip
80003a7c:       f101 31ff       add.w   r1, r1, #4294967295     ; 0xffffffff
80003a80:       60b9            str     r1, [r7, #8]
80003a82:       4660            mov     r0, ip
80003a84:       f04f 0100       mov.w   r1, #0
80003a88:       f04f 38ff       mov.w   r8, #4294967295 ; 0xffffffff
80003a8c:       f04f 090f       mov.w   r9, #15
80003a90:       ea00 0008       and.w   r0, r0, r8
80003a94:       ea01 0109       and.w   r1, r1, r9
80003a98:       ea4f 7850       mov.w   r8, r0, lsr #29
80003a9c:       ea4f 05c1       mov.w   r5, r1, lsl #3
80003aa0:       ea48 0505       orr.w   r5, r8, r5
80003aa4:       ea4f 04c0       mov.w   r4, r0, lsl #3
80003aa8:       f04f 30ff       mov.w   r0, #4294967295 ; 0xffffffff
80003aac:       f04f 010f       mov.w   r1, #15
80003ab0:       ea04 0400       and.w   r4, r4, r0
80003ab4:       ea05 0501       and.w   r5, r5, r1
80003ab8:       4660            mov     r0, ip
80003aba:       f04f 0100       mov.w   r1, #0
80003abe:       f04f 34ff       mov.w   r4, #4294967295 ; 0xffffffff
80003ac2:       f04f 050f       mov.w   r5, #15
80003ac6:       ea00 0004       and.w   r0, r0, r4
80003aca:       ea01 0105       and.w   r1, r1, r5
80003ace:       ea4f 7450       mov.w   r4, r0, lsr #29
80003ad2:       ea4f 03c1       mov.w   r3, r1, lsl #3
80003ad6:       ea44 0303       orr.w   r3, r4, r3
80003ada:       ea4f 02c0       mov.w   r2, r0, lsl #3
80003ade:       f04f 30ff       mov.w   r0, #4294967295 ; 0xffffffff
80003ae2:       f04f 010f       mov.w   r1, #15
80003ae6:       ea02 0200       and.w   r2, r2, r0
80003aea:       ea03 0301       and.w   r3, r3, r1
80003aea:       ea03 0301       and.w   r3, r3, r1
80003aee:       4663            mov     r3, ip
80003af0:       f103 0307       add.w   r3, r3, #7
80003af4:       f103 0307       add.w   r3, r3, #7
80003af8:       ea4f 03d3       mov.w   r3, r3, lsr #3
80003afc:       ea4f 03c3       mov.w   r3, r3, lsl #3
80003b00:       ebad 0d03       sub.w   sp, sp, r3
80003b04:       466b            mov     r3, sp
80003b06:       f103 0307       add.w   r3, r3, #7
80003b0a:       ea4f 03d3       mov.w   r3, r3, lsr #3
80003b0e:       ea4f 03c3       mov.w   r3, r3, lsl #3
80003b12:       60fb            str     r3, [r7, #12]
80003b14:       f04f 0300       mov.w   r3, #0
80003b18:       46b5            mov     sp, r6
80003b1a:       4618            mov     r0, r3
80003b1c:       f107 0710       add.w   r7, r7, #16
80003b20:       46bd            mov     sp, r7
80003b22:       e8bd 03f0       ldmia.w sp!, {r4, r5, r6, r7, r8, r9}
80003b26:       4770            bx      lr

Where does all this additional logic come frome? I would have thought, that it would have been enough to just enlarge the stack frame by the size i (passed in r0), maybe with an addition of some extra code to keep the alignment. But why are all these additional registers written with 0xffffffff and #15. I cannot really make any sense of this assembly the compiler is giving me.

Answer 1

This is not really an answer just collecting more info that doesnt fit in a comment

typedef unsigned int size_t;

int test_method1(size_t i)
{
   unsigned char buffer[10];
   return -1;
}

int test_method2(size_t i)
{
    unsigned char buffer[i];
    return 0;
}

arm-none-eabi-gcc -O2 -c tm1.c -o tm1.o

arm-none-eabi-objdump -D tm1.o

basically optimizes everything away

00000000 <test_method1>:
   0:   e3e00000    mvn r0, #0
   4:   e12fff1e    bx  lr

00000008 <test_method2>:
   8:   e3a00000    mov r0, #0
   c:   e12fff1e    bx  lr

although bad code this should get the compiler to not optimize everything away

typedef unsigned int size_t;

unsigned char *test_method1(size_t i, size_t j)
{
   unsigned char buffer[10];
   return(&buffer[j]);
}

unsigned char *test_method2(size_t i, size_t j)
{
    unsigned char buffer[i];
    return(&buffer[j]);
}

00000000 <test_method1>:
   0:   e24dd010    sub sp, sp, #16
   4:   e28d3004    add r3, sp, #4
   8:   e0830001    add r0, r3, r1
   c:   e28dd010    add sp, sp, #16
  10:   e12fff1e    bx  lr

00000014 <test_method2>:
  14:   e92d0808    push    {r3, fp}
  18:   e2800007    add r0, r0, #7
  1c:   e3c00007    bic r0, r0, #7
  20:   e28db004    add fp, sp, #4
  24:   e04dd000    sub sp, sp, r0
  28:   e08d0001    add r0, sp, r1
  2c:   e24bd004    sub sp, fp, #4
  30:   e8bd0808    pop {r3, fp}
  34:   e12fff1e    bx  lr

And it did and I think it perhaps answered your question. How the compiler handles arrays that are variable length is that in this case it did math on the stack pointer in the amount of the size of the array, basically allocating the dynamic array on the stack as you would expect. For the static sized array the math done on the stack was a static number not a passed in parameter.