continuation of "qemu-riscv-virt"
Goals:
- Debug a bare bones application running on qemu, using gdb
- Gain insight on RISCV boot (ZSBL: Zero stage boot loader)
Required files and dependencies:
- kernel.c
- entry.S
- linker.ld
- ubuntu-24-04
- riscv-gnu-toolchain ( nightly release from risc-collab repo )
- qemu (qemu-system-misc, includes rv64 and rv32)
Instructions
- compile keeping debug info and disabling optimizations
$ riscv64-elf-gcc -g -O0 -Wall -Wextra -c -mcmodel=medany kernel.c -o kernel.o -ffreestanding
$ riscv64-elf-as -c entry.S -o entry.o
$ riscv64-elf-ld -T linker.ld -lgcc -nostdlib kernel.o entry.o -o kernel.elf --static -L riscv64-elf/riscv/lib/gcc/riscv64-unknown-elf/16.1.0/- run qemu, wait for gdb
$ qemu-system-riscv64 -machine virt -bios none -kernel kernel.elf -serial mon:stdio -nographic -S -gdb tcp::1234- run gdb, connect to qemu
$ alias riscv64-elf-gdb="./riscv64-elf/riscv/bin/riscv64-unknown-elf-gdb"
$ riscv64-elf-gdb kernel.elf
(gdb) target remote tcp::1234- list functions
(gdb) info functions
All defined functions:
File kernel.c:
18: void kmain(void);
10: void print(const char *);
5: void putchar(char);
Non-debugging symbols:
0x0000000080000000 start
0x0000000080000024 bss_clear- check our starting instruction
(gdb) x/i $pc
=> 0x1000: auipc t0,0x0
//program counter is at 0x1000
//so were not at "start:" (0x80000000) yet.
//lets see how we get there first- check all instructions until we jump to "start:"
(gdb) disassemble $pc, $pc+0x17
Dump of assembler code from 0x1000 to 0x1017:
=> 0x0000000000001000: auipc t0,0x0
0x0000000000001004: addi a2,t0,40 # 0x1028
0x0000000000001008: csrr a0,mhartid
0x000000000000100c: ld a1,32(t0)
0x0000000000001010: ld t0,24(t0)
0x0000000000001014: jr t0- how were this instructions generated?
// in qemu's virt, the code before hitting "start:" corresponds to the VIRT_MROM region.
// it is populated by qemu's riscv_setup_rom_reset_vec()
// here we can see how it's generated
// RISC-V bootup process first step is to run the ZSBL (Zero Stage Bootloader).
// the ZSBL inializes some registers and then jumps to other address
// for this case, that would be our elf's "start:"
//from https://github.com/qemu/qemu/blob/master/hw/riscv/boot.c
void riscv_setup_rom_reset_vec(
MachineState *machine,
RISCVHartArrayState *harts,
hwaddr start_addr,
hwaddr rom_base, hwaddr rom_size,
uint64_t kernel_entry,
uint64_t fdt_load_addr)
{
const bool rv32 = riscv_is_32bit(harts);
const bool big_endian = harts->harts[0].cfg.big_endian;
uint32_t reset_vec[CODE_WORDS + DATA_WORDS];
/* .text (RISC-V instructions are always little-endian) */
reset_vec[0] = const_le32(0x00000297); /* 1: auipc t0, %pcrel_hi(fw_dyn) */
reset_vec[1] = const_le32(0x02828613); /* addi a2, t0, %pcrel_lo(1b) */
reset_vec[2] = const_le32(0xf1402573); /* csrr a0, mhartid */
if (harts->harts[0].cfg.ext_zicsr) {
reset_vec[2] = const_le32(0xf1402573); /* csrr a0, mhartid */
} else {
/*
* The Zicsr extension has been disabled, so let's ensure we don't
* run the CSR instruction. Let's fill the address with a non
* compressed nop.
*/
reset_vec[2] = const_le32(0x00000013); /* addi x0, x0, 0 */
}
if (rv32) {
reset_vec[3] = const_le32(0x0202a583); /* lw a1, 32(t0) */
reset_vec[4] = const_le32(0x0182a283); /* lw t0, 24(t0) */
} else {
reset_vec[3] = const_le32(0x0202b583); /* ld a1, 32(t0) */
reset_vec[4] = const_le32(0x0182b283); /* ld t0, 24(t0) */
}
reset_vec[5] = const_le32(0x00028067); /* jr t0 */
/* .data (must match the firmware's data endianness) */
if (big_endian) {
stq_be_p(&reset_vec[6], start_addr); /* start: .dword */
stq_be_p(&reset_vec[8], fdt_load_addr); /* fdt_laddr: .dword */
} else {
stq_le_p(&reset_vec[6], start_addr);
stq_le_p(&reset_vec[8], fdt_load_addr);
}
rom_add_blob_fixed_as("mrom.reset", reset_vec, sizeof(reset_vec),
rom_base, &address_space_memory);
riscv_rom_copy_firmware_info(machine, harts,
rom_base, rom_size,
sizeof(reset_vec),
kernel_entry);
} - so, lets comment our ZSBL, using what we just learned from riscv_setup_rom_reset_vec()
(gdb) disassemble $pc, $pc+0x17
Dump of assembler code from 0x1000 to 0x1017:
=> 0x0000000000001000: auipc t0,0x0
//saves current $pc to $t0
//$t0 = $pc+{0x0,12'd0}
0x0000000000001004: addi a2,t0,40 # 0x1028
//pointer to fw_dynamic_info struct . unused, for linux boot only
// (QEMU's default boot mechanism, from OpenSBI)
//$a2 = $t0+{40+2**12}
0x0000000000001008: csrr a0,mhartid
//(HART ID) hardware thead id. unused, for linux boot only
//$a0 = hardware_thread_id from ctrl_and_status_register
0x000000000000100c: ld a1,32(t0)
//fdt (flattened device tree), unused, for linux boot only
//$a1 = *(32+0x1000) // 0x87e00000
0x0000000000001010: ld t0,24(t0)
//$t0 = *(24+0x1000) // 0x80000000
0x0000000000001014: jr t0
//jumps directly to "start:" at 0x80000000- Check fw_dynamic_info struct generation
//from https://github.com/qemu/qemu/blob/master/hw/riscv/boot.c
void riscv_rom_copy_firmware_info(MachineState *machine,
RISCVHartArrayState *harts,
hwaddr rom_base, hwaddr rom_size,
uint32_t reset_vec_size,
uint64_t kernel_entry)
{
struct fw_dynamic_info32 dinfo32;
struct fw_dynamic_info64 dinfo64;
void *dinfo_ptr = NULL;
size_t dinfo_len;
const bool rv32 = riscv_is_32bit(harts);
const bool be = harts->harts[0].cfg.big_endian;
if (rv32) {
dinfo32.magic = be ? cpu_to_be32(FW_DYNAMIC_INFO_MAGIC_VALUE)
: cpu_to_le32(FW_DYNAMIC_INFO_MAGIC_VALUE);
dinfo32.version = be ? cpu_to_be32(FW_DYNAMIC_INFO_VERSION)
: cpu_to_le32(FW_DYNAMIC_INFO_VERSION);
dinfo32.next_mode = be ? cpu_to_be32(FW_DYNAMIC_INFO_NEXT_MODE_S)
: cpu_to_le32(FW_DYNAMIC_INFO_NEXT_MODE_S);
dinfo32.next_addr = be ? cpu_to_be32(kernel_entry)
: cpu_to_le32(kernel_entry);
dinfo32.options = 0;
dinfo32.boot_hart = 0;
dinfo_ptr = &dinfo32;
dinfo_len = sizeof(dinfo32);
} else {
dinfo64.magic = be ? cpu_to_be64(FW_DYNAMIC_INFO_MAGIC_VALUE)
: cpu_to_le64(FW_DYNAMIC_INFO_MAGIC_VALUE);
dinfo64.version = be ? cpu_to_be64(FW_DYNAMIC_INFO_VERSION)
: cpu_to_le64(FW_DYNAMIC_INFO_VERSION);
dinfo64.next_mode = be ? cpu_to_be64(FW_DYNAMIC_INFO_NEXT_MODE_S)
: cpu_to_le64(FW_DYNAMIC_INFO_NEXT_MODE_S);
dinfo64.next_addr = be ? cpu_to_be64(kernel_entry)
: cpu_to_le64(kernel_entry);
dinfo64.options = 0;
dinfo64.boot_hart = 0;
dinfo_ptr = &dinfo64;
dinfo_len = sizeof(dinfo64);
}
/**
* copy the dynamic firmware info. This information is specific to
* OpenSBI but doesn't break any other firmware as long as they don't
* expect any certain value in "a2" register.
*/
if (dinfo_len > (rom_size - reset_vec_size)) {
error_report("not enough space to store dynamic firmware info");
exit(1);
}
rom_add_blob_fixed_as("mrom.finfo",
dinfo_ptr,
dinfo_len,
rom_base + reset_vec_size,
&address_space_memory);
}- Compare values in memory with the struct fields
(gdb) x/6xg 0x1028
// x: examine memory
// /
// 6: 6 times
// x: as hex
// g: giant (8 bytes, rv64 word size)
// 0x1028: starting address
0x1028: 0x000000004942534f 0x0000000000000002
0x1038: 0x0000000080000000 0x0000000000000001
0x1048: 0x0000000000000000 0x0000000000000000
struct fw_dynamic_info {
unsigned long magic; // 'OSBI', 0x4942534f
unsigned long version; // Info version. The current latest is 0x2
unsigned long next_addr; // Next booting stage address, 0x80000000
unsigned long next_mode; // Next booting mode.FW_DYNAMIC_INFO_NEXT_MODE_S=1
unsigned long options; // OpenSBI options
unsigned long boot_hart; // Preferred boot HART id
} - TUI debugging on gdb
now that we understand how we get to "start:", lets step through our elf using TUI
(gdb) tui enable
(gdb) b kmain
//set breakpoint at kmain()
(gdb) continue
Continuing.
Breakpoint 1, kmain () at kernel.c:19
//gdb breaks at kmain, before our print()
- run next line
(gdb) next
- continue to inf loop
(gdb) continue
Continuing.
// kmain will print any key input on qemu's terminal
// press Ctrl+A then X to terminate QEMU
// type "exit" to quit gdb