Add support for a classic monolithic OS

Cargo.toml

@@ -62,6 +62,9 @@ default = [
 ## Note that this feature is not complete yet.
 common-os = []
 
+## Enable the support of classical fork
+fork = ["common-os"]
+
 ## Enables support for memory management system calls.
 ##
 ## This feature enables functions similar to [sys/mman.h].

src/arch/aarch64/kernel/exceptions.s

@@ -243,13 +243,52 @@ el1_sp0_error:
 .size el1_sp0_error, .-el1_sp0_error
 .type el1_sp0_error, @function
 
-el0_sync_invalid:
-   invalid 0
-.type el0_sync_invalid, @function
+/*
+ * Synchronous exception from a lower EL (EL0) using AArch64. This is the
+ * SVC entry point — the dispatcher in do_sync decodes ESR_EL1.EC.
+ *
+ * On exception entry from EL0, the hardware automatically switches to
+ * SP_EL1, so we don't need to flip SPSEL here.
+ */
+.align 6
+el0_sync:
+      trap_entry 0
+      mov     x0, sp
+      bl      do_sync
+      trap_exit
+      eret
+      // speculation barrier after the ERET to prevent the CPU
+      // from speculating past the exception return.
+      dsb     nsh
+      isb
+.size el0_sync, .-el0_sync
+.type el0_sync, @function
 
-el0_irq_invalid:
-   invalid 1
-.type el0_irq_invalid, @function
+/*
+ * IRQ from a lower EL (EL0). Mirrors el1_irq, including the optional
+ * task switch when do_irq returns the saved-SP slot of the next task.
+ */
+.align 6
+el0_irq:
+      trap_entry 0
+      mov     x0, sp
+      bl      do_irq
+      cmp x0, 0
+      b.eq 5f
+      // switch to the next task
+      mov x1, sp
+      str x1, [x0]                  /* store old sp */
+      bl get_last_stack_pointer     /* get new sp   */
+      mov sp, x0
+5:
+      trap_exit
+      eret
+      // speculation barrier after the ERET to prevent the CPU
+      // from speculating past the exception return.
+      dsb     nsh
+      isb
+.size el0_irq, .-el0_irq
+.type el0_irq, @function
 
 el0_fiq_invalid:
    invalid 2
@@ -293,14 +332,14 @@ ventry el1_fiq                  // FIQ EL1h
 ventry el1_error                // Error EL1h
 
 /* Lower EL using AArch64 */
-ventry el0_sync_invalid         // Synchronous 64-bit EL0
-ventry el0_irq_invalid          // IRQ 64-bit EL0
+ventry el0_sync                 // Synchronous 64-bit EL0
+ventry el0_irq                  // IRQ 64-bit EL0
 ventry el0_fiq_invalid          // FIQ 64-bit EL0
 ventry el0_error_invalid        // Error 64-bit EL0
 
-/* Lower EL using AArch32 */
-ventry el0_sync_invalid         // Synchronous 32-bit EL0
-ventry el0_irq_invalid          // IRQ 32-bit EL0
+/* Lower EL using AArch32 — not supported, route to invalid */
+ventry el0_fiq_invalid          // Synchronous 32-bit EL0
+ventry el0_fiq_invalid          // IRQ 32-bit EL0
 ventry el0_fiq_invalid          // FIQ 32-bit EL0
 ventry el0_error_invalid        // Error 32-bit EL0
 .size vector_table, .-vector_table

src/arch/aarch64/kernel/interrupts.rs

@@ -30,6 +30,12 @@ const PPI_START: u8 = 16;
 const SPI_START: u8 = 32;
 /// Software-generated interrupt for rescheduling
 pub(crate) const SGI_RESCHED: u8 = 1;
+/// Synthetic IRQ slot used for page-fault accounting. The number does not
+/// correspond to any GIC interrupt — it is purely a bookkeeping ID for
+/// `IrqStatistics` so the page-fault count shows up in `print_statistics`
+/// alongside the real interrupts.
+#[cfg(feature = "common-os")]
+pub(crate) const PAGE_FAULT_IRQ: u8 = 14;
 
 /// Number of the timer interrupt
 static mut TIMER_INTERRUPT: u32 = 0;
@@ -170,43 +176,138 @@ pub(crate) extern "C" fn do_irq(_state: &State) -> *mut usize {
 }
 
 #[unsafe(no_mangle)]
-pub(crate) extern "C" fn do_sync(state: &State) {
+pub(crate) extern "C" fn do_sync(state: &mut State) {
+	#[cfg(all(feature = "common-os", feature = "fork"))]
+	use crate::arch::mm::paging::do_cow_fault;
+
 	let esr = ESR_EL1.get();
 	let ec_raw = ESR_EL1.read(ESR_EL1::EC);
 	let ec: ESR_EL1::EC::Value = ESR_EL1.read_as_enum(ESR_EL1::EC).unwrap();
 	let iss = ESR_EL1.read(ESR_EL1::ISS);
 	let pc = ELR_EL1.get();
 
-	/* data abort from lower or current level */
-	if (ec == ESR_EL1::EC::Value::DataAbortCurrentEL)
-		|| (ec == ESR_EL1::EC::Value::DataAbortLowerEL)
-	{
-		/* check if value in far_el1 is valid */
-		if (iss & (1 << 10)) == 0 {
-			/* read far_el1 register, which holds the faulting virtual address */
-			let far = FAR_EL1.get();
-
-			// add page fault handler
-
-			error!("Current stack pointer {state:p}");
-			error!("Unable to handle page fault at {far:#x}");
-			error!("Exception return address {:#x}", ELR_EL1.get());
-			error!("Thread ID register {:#x}", TPIDR_EL0.get());
-			error!("Table Base Register {:#x}", TTBR0_EL1.get());
-			error!("Exception Syndrome Register {esr:#x}");
-
-			if let Some(irqid) =
-				GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1)
+	// SVC64 — user-space syscall (ESR_EL1.EC = 0x15). Dispatched here
+	// before any of the other branches so we can return early without
+	// touching debug/breakpoint/FPU paths.
+	#[cfg(feature = "common-os")]
+	if ec_raw == 0x15 {
+		dispatch_svc64(state);
+		return;
+	}
+
+	// User-mode instruction-fetch fault at PC=0: the entry wrapper of a
+	// freshly spawned user thread (`std::sys::thread::hermit::Thread::
+	// new_with_coreid::thread_start`) returns with `ret`, popping LR
+	// from the user stack. The trap frame we crafted in
+	// `Task::create_user_stack_frame` zeroed every register, so LR=0
+	// and the implicit branch lands at PC 0 — there is no code there.
+	// Mirror the x86_64 page-fault handler and treat this as a clean
+	// thread exit instead of crashing the whole process.
+	#[cfg(feature = "common-os")]
+	if ec == ESR_EL1::EC::Value::InstrAbortLowerEL && ELR_EL1.get() == 0 {
+		use crate::scheduler::PerCoreSchedulerExt;
+		debug!(
+			"User thread {} returned from entry; exiting cleanly.",
+			core_scheduler().get_current_task_id()
+		);
+		core_scheduler().exit(0);
+	}
+
+	/* Data Abort from current or lower EL — the primary path is a COW
+	 * write fault from EL0 (EC=0x25). EC=0x24 covers a kernel write to a
+	 * COW-marked page, which can happen e.g. when the kernel writes
+	 * argv/envp into the freshly-mapped user page during the loader path.
+	 */
+	if ec == ESR_EL1::EC::Value::DataAbortCurrentEL || ec == ESR_EL1::EC::Value::DataAbortLowerEL {
+		#[cfg(feature = "common-os")]
+		increment_irq_counter(PAGE_FAULT_IRQ);
+
+		let far = FAR_EL1.get();
+		// ESR_EL1.ISS layout for Data Abort (ARM ARM D24.2.45):
+		//   bits  5:0 = DFSC (Data Fault Status Code)
+		//   bit     6 = WnR (1 = write, 0 = read)
+		// Permission fault DFSC values are 0b001100..0b001111 (level 0..3).
+		let dfsc = iss & 0b11_1111;
+		let is_write = (iss & (1 << 6)) != 0;
+		#[cfg(all(feature = "common-os", feature = "fork"))]
+		let is_permission_fault = (0b00_1100..=0b00_1111).contains(&dfsc);
+
+		#[cfg(all(feature = "common-os", feature = "fork"))]
+		if is_write && is_permission_fault && do_cow_fault(VirtAddr::new(far)) {
+			// Faulting instruction is retried on `eret` from the trap.
+			return;
+		}
+
+		#[cfg(feature = "common-os")]
+		{
+			use core::ops::Bound;
+
+			use align_address::Align;
+
+			use crate::mm::FrameAlloc;
+			use crate::mm::vma::VirtualMemoryAreaProt;
+
+			let addr = VirtAddr::new(far).align_down(BasePageSize::SIZE);
+			let current_task = core_scheduler().get_current_task();
+			let current_task_borrowed = current_task.borrow();
+			let guard = current_task_borrowed.vmas.read();
+
+			if let Some((_, vma)) = guard
+				.range((Bound::Unbounded, Bound::Included(addr)))
+				.next_back() && addr >= vma.start
+				&& addr < vma.end
 			{
-				GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1);
-			} else {
-				error!("Unable to acknowledge interrupt!");
+				let layout = PageLayout::from_size_align(
+					BasePageSize::SIZE as usize,
+					BasePageSize::SIZE as usize,
+				)
+				.unwrap();
+				let frame_range = FrameAlloc::allocate(layout).unwrap();
+				let physaddr = PhysAddr::from(frame_range.start());
+				let mut flags = PageTableEntryFlags::empty();
+				flags.normal().user();
+				if vma.prot.contains(VirtualMemoryAreaProt::WRITE) {
+					flags.writable();
+				}
+				if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) {
+					flags.execute_disable();
+				}
+
+				paging::map::<BasePageSize>(addr, physaddr, 1, flags);
+
+				// clear page
+				let slice = unsafe {
+					core::slice::from_raw_parts_mut(
+						addr.as_mut_ptr::<u8>().cast::<u8>(),
+						BasePageSize::SIZE as usize,
+					)
+				};
+				slice.fill(0);
+
+				#[cfg(feature = "fork")]
+				crate::mm::frame_ref_inc(physaddr);
+
+				return;
 			}
+		}
+
+		let kind = dfsc_kind(dfsc);
+		let access = if is_write { "write" } else { "read" };
+		error!("Current stack pointer {state:p}");
+		error!("Unhandled data abort: {kind} on {access} of {far:#x} (DFSC={dfsc:#x})");
+		error!("Exception return address {:#x}", ELR_EL1.get());
+		error!("Thread ID register {:#x}", TPIDR_EL0.get());
+		error!("Table Base Register {:#x}", TTBR0_EL1.get());
+		error!("Exception Syndrome Register {esr:#x}");
 
-			scheduler::abort()
+		if let Some(irqid) = GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1)
+		{
+			GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1);
 		} else {
-			error!("Unknown exception");
+			error!("Unable to acknowledge interrupt!");
 		}
+
+		scheduler::abort()
 	} else if ec == ESR_EL1::EC::Value::Brk64 {
 		error!("Trap to debugger, PC={pc:#x}");
 		loop {
@@ -232,6 +333,64 @@ pub(crate) extern "C" fn do_sync(state: &State) {
 	}
 }
 
+/// Convert the 6-bit DFSC (Data Fault Status Code) of `ESR_EL1.ISS` into
+/// a short human-readable label for diagnostics. Mapping per ARM ARM
+/// D24.2.45 (ESR_EL1, Data Abort).
+fn dfsc_kind(dfsc: u64) -> &'static str {
+	match dfsc {
+		0b00_0000..=0b00_0011 => "address size fault",
+		0b00_0100..=0b00_0111 => "translation fault",
+		0b00_1000..=0b00_1011 => "access flag fault",
+		0b00_1100..=0b00_1111 => "permission fault",
+		0b01_0000 => "synchronous external abort",
+		0b01_0001 => "synchronous tag check fail",
+		0b01_0100..=0b01_0111 => "external abort on translation table walk",
+		0b01_1000 => "synchronous parity/ECC error",
+		0b01_1100..=0b01_1111 => "parity/ECC error on translation table walk",
+		0b10_0001 => "alignment fault",
+		0b11_0000 => "TLB conflict abort",
+		0b11_0001 => "unsupported atomic hardware-update fault",
+		_ => "unknown data fault",
+	}
+}
+
+/// Dispatch an EL0-issued AArch64 syscall (`svc #0`).
+///
+/// Hermit's user-space follows the Linux-like AArch64 convention:
+///   - syscall number in `x8`
+///   - up to 6 arguments in `x0`..`x5`
+///   - return value in `x0`
+///
+/// The handler entries in `SYSHANDLER_TABLE` are typed for the SystemV
+/// x86_64 ABI but are reachable just as well via the AAPCS64 ABI: in both
+/// cases the first six 64-bit args land in the first six argument
+/// registers and the return value goes into the first return register.
+/// Registers in `state` were saved by `trap_entry` and are restored by
+/// `trap_exit` — writing back `state.x0` is what propagates the return
+/// value to user-space.
+#[cfg(feature = "common-os")]
+fn dispatch_svc64(state: &mut State) {
+	use crate::errno::Errno;
+	use crate::syscalls::table::{NO_SYSCALLS, SYSHANDLER_TABLE, invalid_syscall, sys_invalid};
+
+	let nr = state.x8 as usize;
+
+	if nr >= NO_SYSCALLS {
+		error!("Invalid syscall number {nr}");
+		state.x0 = u64::from((-i32::from(Errno::Nosys)) as u32);
+		return;
+	}
+
+	let handler_ptr = SYSHANDLER_TABLE.handler(nr);
+	if handler_ptr == sys_invalid as *const usize {
+		invalid_syscall(state.x8);
+	}
+
+	let f: extern "C" fn(u64, u64, u64, u64, u64, u64) -> u64 =
+		unsafe { core::mem::transmute(handler_ptr) };
+	state.x0 = f(state.x0, state.x1, state.x2, state.x3, state.x4, state.x5);
+}
+
 #[unsafe(no_mangle)]
 pub(crate) extern "C" fn do_bad_mode(_state: &State, reason: u32) -> ! {
 	error!("Receive unhandled exception: {reason}");
@@ -444,6 +603,8 @@ pub(crate) fn init() {
 		.unwrap();
 	gic.enable_interrupt(reschedid, Some(cpu_id), true).unwrap();
 	IRQ_NAMES.lock().insert(SGI_RESCHED, "Reschedule");
+	#[cfg(feature = "common-os")]
+	IRQ_NAMES.lock().insert(PAGE_FAULT_IRQ, "Page Fault");
 
 	*GIC.lock() = Some(gic);
 }