How do I enter 32-bit protected mode in NASM assembly?

Question

I'm learning x86 assembly, and I'm trying to make a toy operating system in NASM, but I don't understand some things.

I made a bootloader that is successfully boots my kernel:

1. Loads 14 sectors from the diskette which contains the kernel file;
2. Search a file in these sectors labeled kernel.feo;
3. Loads that file into the memory to the offset 0x2000;
4. Executes the kernel using a far jump jmp 0x2000:0x0000.

So I have the kernel code located at 0x2000:0 in the memory. CS might be properly set because the using of a far jump. In this kernel code, I want to enter 32-bit protected mode, but I'm not sure how GDTs are working. When I run the code below on a virtual machine (QEMU), it is don't do anything.

I want to please you to help me entering 32-bit protected mode!

That said, you have the following problems:

1. You assume the code is loaded at 0x7c00:0 due to the org 0, but that might not be the case. The only thing guaranteed is the physical address. You should use a far jump to your entry point so that CS is properly set.
2. You are for some reason setting DS to 0x2000 so your code won't find any data at all. You should set DS to match CS, or use a CS override everywhere (not recommended).
3. The protected mode code assumes zero-based segment, which in turn means it expects org 0x7c00 which of course conflicts with your setup. You should switch to org 0x7c00 and segments 0.
4. The VGA text mode segment is at 0xb8000 not 0xb80000 (one less zero).
5. You don't have the boot signature bytes 0x55 0xaa at the end of the boot sector.

I have corrected these things in my code:

1. [org 0x0] is corrected to [org 0x2000] and segments are set to 0;
2. DS is corrected to 0 instead of 0x2000, so now it matches with CS;
3. VGA Text Mode segment is corrected to 0xb8000;

But the code won't work with these corrections, it should print two strings but it don't do anything!

Note that this kernel code should not end with a boot signature 0x55 0xAA, because it isn't a boot sector.

Here is the corrected kernel code (that not working):

[bits 16]
[org 0x2000]

    jmp 0:kernel_start

gdt_start:

gdt_null:
    dd 0x0
    dd 0x0

gdt_code:
    dw 0xffff
    dw 0x0
    db 0x0
    db 10011010b
    db 11001111b
    db 0x0

gdt_data:
    dw 0xffff
    dw 0x0
    db 0x0
    db 10010010b
    db 11001111b
    db 0x0

gdt_end:

gdt_descriptor:
    dw gdt_end - gdt_start
    dd gdt_start

CODE_SEG equ gdt_code - gdt_start
DATA_SEG equ gdt_data - gdt_start

print:
    mov ah, 14
    mov bh, 0
    lodsb
    cmp al, 0
    je .done
    int 0x10
    jmp print
.done:
    ret

uzenet_real db 'uzenet16', 0
uzenet_prot db 'uzenet32', 0

kernel_start:
    mov ax, 0
    mov ss, ax
    mov sp, 0xFFFC

    mov ax, 0
    mov ds, ax
    mov es, ax
    mov fs, ax
    mov gs, ax

    mov si, uzenet_real
    call print

    cli
    lgdt[gdt_descriptor]
    mov eax, cr0
    or eax, 0x1
    mov cr0, eax
    jmp CODE_SEG:b32

[bits 32]

VIDEO_MEMORY equ 0xb8000
WHITE_ON_BLACK equ 0x0f

print32:
    pusha
    mov edx, VIDEO_MEMORY
.loop:
    mov al, [ebx]
    mov ah, WHITE_ON_BLACK
    cmp al, 0
    je .done
    mov [edx], ax
    add ebx, 1
    add edx, 2
    jmp .loop
.done:
    popa
    ret

b32:
    mov ax, DATA_SEG
    mov ds, ax
    mov es, ax
    mov fs, ax
    mov gs, ax
    mov ss, ax

    mov ebp, 0x90000
    mov esp, ebp

    mov ebx, uzenet_prot
    call print32

    jmp $

Answer 1

Programming an OS is an advanced task. You are at least expected to be able to use a debugger to find your own mistakes and understand basic things. You might want to reconsider whether you have all the prerequisites for this endeavour.

That said, you have the following problems:

1. You assume the code is loaded at 0x7c00:0 due to the org 0, but that might not be the case. The only thing guaranteed is the physical address. You should use a far jump to your entry point so that CS is properly set.
2. You are for some reason setting DS to 0x2000 so your code won't find any data at all. You should set DS to match CS, or use a CS override everywhere (not recommended).
3. The protected mode code assumes zero-based segment, which in turn means it expects org 0x7c00 which of course conflicts with your setup. You should switch to org 0x7c00 and segments 0.
4. The VGA text mode segment is at 0xb8000 not 0xb80000 (one less zero).
5. You don't have the boot signature bytes 0x55 0xaa at the end of the boot sector.

The fixed code:

[bits 16]
[org 0x7c00]

    jmp 0:kernel_start

gdt_start:

gdt_null:
    dd 0x0
    dd 0x0

gdt_code:
    dw 0xffff
    dw 0x0
    db 0x0
    db 10011010b
    db 11001111b
    db 0x0

gdt_data:
    dw 0xffff
    dw 0x0
    db 0x0
    db 10010010b
    db 11001111b
    db 0x0

gdt_end:

gdt_descriptor:
    dw gdt_end - gdt_start
    dd gdt_start

CODE_SEG equ gdt_code - gdt_start
DATA_SEG equ gdt_data - gdt_start

print:
    pusha
    mov ah, 14
    mov bh, 0
.loop:
    lodsb
    cmp al, 0
    je .done
    int 0x10
    jmp .loop
.done:
    popa
    ret

uzenet16 db 'uzenet16', 0
uzenet32 db 'uzenet32', 0

kernel_start:
    mov ax, 0
    mov ss, ax
    mov sp, 0xFFFC

    mov ax, 0
    mov ds, ax
    mov es, ax
    mov fs, ax
    mov gs, ax

    mov si, uzenet16
    call print

    cli
    lgdt[gdt_descriptor]
    mov eax, cr0
    or eax, 0x1
    mov cr0, eax
    jmp CODE_SEG:b32

[bits 32]

VIDEO_MEMORY equ 0xb8000
WHITE_ON_BLACK equ 0x0f

print32:
    pusha
    mov edx, VIDEO_MEMORY
.loop:
    mov al, [ebx]
    mov ah, WHITE_ON_BLACK
    cmp al, 0
    je .done
    mov [edx], ax
    add ebx, 1
    add edx, 2
    jmp .loop
.done:
    popa
    ret

b32:
    mov ax, DATA_SEG
    mov ds, ax
    mov es, ax
    mov fs, ax
    mov gs, ax
    mov ss, ax

    mov ebp, 0x2000
    mov esp, ebp

    mov ebx, uzenet32
    call print32

    jmp $

[SECTION signature start=0x7dfe]
dw 0AA55h

---

Your updated question still seems to be confused about where the code is loaded: you say offset 0x2000 but then talk about Executes the kernel using a far jump jmp 0x2000:0x0000 which is of course wrong, because it has one more zero in the segment, and should be a zero-segment far jump anyway: jmp 0:0x2000. Other than that, verify that your code is indeed loaded into memory at the correct place. Learn to use a debugger.

Here is a small boot sector which loads the above code from the second sector to address 0x2000. It works fine, the problem is not with the GDT stuff, especially if you don't even get the real mode message printed (you were not clear about that either).

[bits 16]
[org 0x7c00]
mov ax, 0201h
mov cx, 0002h
mov dh, 0
mov bx, 0
mov es, bx
mov bx, 2000h
int 13h
jmp 0:2000h

[SECTION signature start=0x7dfe]
dw 0AA55h