Kernel/aarch64: Implement initial page fault handling

The shared code is moved to a common PageFault.cpp file.
This commit is contained in:
Timon Kruiper 2023-01-08 17:42:07 +01:00 committed by Andreas Kling
parent a532d28905
commit 55d756a813
Notes: sideshowbarker 2024-07-17 11:33:34 +09:00
5 changed files with 195 additions and 133 deletions

150
Kernel/Arch/PageFault.cpp Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <Kernel/Arch/CPU.h>
#include <Kernel/Arch/PageFault.h>
#include <Kernel/Arch/Processor.h>
#include <Kernel/Arch/RegisterState.h>
#include <Kernel/Arch/SafeMem.h>
#include <Kernel/PerformanceManager.h>
#include <Kernel/Thread.h>
#include <LibC/mallocdefs.h>
namespace Kernel {
void PageFault::handle(RegisterState& regs)
{
auto fault_address = m_vaddr.get();
bool faulted_in_kernel = regs.previous_mode() == ExecutionMode::Kernel;
if (faulted_in_kernel && Processor::current_in_irq()) {
// If we're faulting in an IRQ handler, first check if we failed
// due to safe_memcpy, safe_strnlen, or safe_memset. If we did,
// gracefully continue immediately. Because we're in an IRQ handler
// we can't really try to resolve the page fault in a meaningful
// way, so we need to do this before calling into
// MemoryManager::handle_page_fault, which would just bail and
// request a crash
if (handle_safe_access_fault(regs, fault_address))
return;
}
auto current_thread = Thread::current();
if (current_thread) {
current_thread->set_handling_page_fault(true);
PerformanceManager::add_page_fault_event(*current_thread, regs);
}
ScopeGuard guard = [current_thread] {
if (current_thread)
current_thread->set_handling_page_fault(false);
};
if (!faulted_in_kernel) {
VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
bool has_valid_stack_pointer = current_thread->process().address_space().with([&](auto& space) {
return MM.validate_user_stack(*space, userspace_sp);
});
if (!has_valid_stack_pointer) {
dbgln("Invalid stack pointer: {}", userspace_sp);
return handle_crash(regs, "Bad stack on page fault", SIGSEGV);
}
}
auto response = MM.handle_page_fault(*this);
if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory || response == PageFaultResponse::BusError) {
if (faulted_in_kernel && handle_safe_access_fault(regs, fault_address)) {
// If this would be a ring0 (kernel) fault and the fault was triggered by
// safe_memcpy, safe_strnlen, or safe_memset then we resume execution at
// the appropriate _fault label rather than crashing
return;
}
if (response == PageFaultResponse::BusError && current_thread->has_signal_handler(SIGBUS)) {
current_thread->send_urgent_signal_to_self(SIGBUS);
return;
}
if (response != PageFaultResponse::OutOfMemory && current_thread) {
if (current_thread->has_signal_handler(SIGSEGV)) {
current_thread->send_urgent_signal_to_self(SIGSEGV);
return;
}
}
dbgln("Unrecoverable page fault, {}{}{} address {}",
is_reserved_bit_violation() ? "reserved bit violation / " : "",
is_instruction_fetch() ? "instruction fetch / " : "",
is_write() ? "write to" : "read from",
VirtualAddress(fault_address));
constexpr FlatPtr malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE);
constexpr FlatPtr free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE);
constexpr FlatPtr kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE);
constexpr FlatPtr kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE);
if (response == PageFaultResponse::BusError) {
dbgln("Note: Address {} is an access to an undefined memory range of an Inode-backed VMObject", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be uninitialized malloc() memory", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be recently free()'d memory", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be uninitialized kmalloc() memory", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be recently kfree()'d memory", VirtualAddress(fault_address));
} else if (fault_address < 4096) {
dbgln("Note: Address {} looks like a possible nullptr dereference", VirtualAddress(fault_address));
} else if constexpr (SANITIZE_PTRS) {
constexpr FlatPtr refptr_scrub_pattern = explode_byte(REFPTR_SCRUB_BYTE);
constexpr FlatPtr nonnullrefptr_scrub_pattern = explode_byte(NONNULLREFPTR_SCRUB_BYTE);
constexpr FlatPtr ownptr_scrub_pattern = explode_byte(OWNPTR_SCRUB_BYTE);
constexpr FlatPtr nonnullownptr_scrub_pattern = explode_byte(NONNULLOWNPTR_SCRUB_BYTE);
constexpr FlatPtr lockrefptr_scrub_pattern = explode_byte(LOCKREFPTR_SCRUB_BYTE);
constexpr FlatPtr nonnulllockrefptr_scrub_pattern = explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE);
if ((fault_address & 0xffff0000) == (refptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (nonnullrefptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (ownptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed OwnPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (nonnullownptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed NonnullOwnPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (lockrefptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (nonnulllockrefptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
}
}
if (current_thread) {
auto& current_process = current_thread->process();
if (current_process.is_user_process()) {
auto fault_address_string = KString::formatted("{:p}", fault_address);
auto fault_address_view = fault_address_string.is_error() ? ""sv : fault_address_string.value()->view();
(void)current_process.try_set_coredump_property("fault_address"sv, fault_address_view);
(void)current_process.try_set_coredump_property("fault_type"sv, type() == PageFault::Type::PageNotPresent ? "NotPresent"sv : "ProtectionViolation"sv);
StringView fault_access;
if (is_instruction_fetch())
fault_access = "Execute"sv;
else
fault_access = access() == PageFault::Access::Read ? "Read"sv : "Write"sv;
(void)current_process.try_set_coredump_property("fault_access"sv, fault_access);
}
}
if (response == PageFaultResponse::BusError)
return handle_crash(regs, "Page Fault (Bus Error)", SIGBUS, false);
return handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory);
} else if (response == PageFaultResponse::Continue) {
dbgln_if(PAGE_FAULT_DEBUG, "Continuing after resolved page fault");
} else {
VERIFY_NOT_REACHED();
}
}
}

View File

@ -8,6 +8,7 @@
#include <AK/Platform.h>
#include <AK/Types.h>
#include <Kernel/Arch/RegisterState.h>
#include <Kernel/ExecutionMode.h>
#include <Kernel/VirtualAddress.h>
@ -44,6 +45,8 @@ public:
{
}
void handle(RegisterState& regs);
enum class Type {
PageNotPresent = PageFaultFlags::NotPresent,
ProtectionViolation = PageFaultFlags::ProtectionViolation,

View File

@ -5,12 +5,14 @@
*/
#include <Kernel/Arch/Interrupts.h>
#include <Kernel/Arch/PageFault.h>
#include <Kernel/Arch/TrapFrame.h>
#include <Kernel/Arch/aarch64/InterruptManagement.h>
#include <Kernel/Interrupts/GenericInterruptHandler.h>
#include <Kernel/Interrupts/SharedIRQHandler.h>
#include <Kernel/Interrupts/UnhandledInterruptHandler.h>
#include <Kernel/KSyms.h>
#include <Kernel/Panic.h>
#include <Kernel/StdLib.h>
namespace Kernel {
@ -53,10 +55,35 @@ static void dump_exception_syndrome_register(Aarch64::ESR_EL1 const& esr_el1)
dbgln("Data Fault Status Code: {}", Aarch64::data_fault_status_code_to_string(esr_el1.ISS));
}
extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame);
extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame)
static PageFault page_fault_from_exception_syndrome_register(VirtualAddress fault_address, Aarch64::ESR_EL1 esr_el1)
{
PageFault fault { fault_address };
u8 data_fault_status_code = esr_el1.ISS & 0x3f;
if (data_fault_status_code >= 0b001100 && data_fault_status_code <= 0b001111) {
fault.set_type(PageFault::Type::ProtectionViolation);
} else if (data_fault_status_code >= 0b000100 && data_fault_status_code <= 0b000111) {
fault.set_type(PageFault::Type::PageNotPresent);
} else {
PANIC("Unknown DFSC: {}", data_fault_status_code);
}
fault.set_access((esr_el1.ISS & (1 << 6)) == (1 << 6) ? PageFault::Access::Write : PageFault::Access::Read);
// FIXME: Set correct mode
fault.set_mode(ExecutionMode::Kernel);
return fault;
}
extern "C" void exception_common(Kernel::TrapFrame* trap_frame);
extern "C" void exception_common(Kernel::TrapFrame* trap_frame)
{
Processor::current().enter_trap(*trap_frame, false);
auto esr_el1 = Kernel::Aarch64::ESR_EL1::read();
auto fault_address = Aarch64::FAR_EL1::read().virtual_address;
Processor::enable_interrupts();
constexpr bool print_state = true;
if constexpr (print_state) {
@ -65,7 +92,17 @@ extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame)
dump_exception_syndrome_register(esr_el1);
}
Kernel::Processor::halt();
if (Aarch64::exception_class_is_data_abort(esr_el1.EC)) {
auto page_fault = page_fault_from_exception_syndrome_register(VirtualAddress(fault_address), esr_el1);
page_fault.handle(*trap_frame->regs);
} else {
dump_registers(*trap_frame->regs);
dump_exception_syndrome_register(esr_el1);
PANIC("Unexpected exception!");
}
Processor::disable_interrupts();
Processor::current().exit_trap(*trap_frame);
}
static Array<GenericInterruptHandler*, 64> s_interrupt_handlers;

View File

@ -23,8 +23,6 @@
#include <Kernel/Thread.h>
#include <Kernel/ThreadTracer.h>
#include <LibC/mallocdefs.h>
#include <Kernel/Arch/CPU.h>
#include <Kernel/Arch/PageFault.h>
#include <Kernel/Arch/Processor.h>
@ -201,135 +199,8 @@ void page_fault_handler(TrapFrame* trap)
dump_registers(regs);
}
bool faulted_in_kernel = !(regs.cs & 3);
if (faulted_in_kernel && Processor::current_in_irq()) {
// If we're faulting in an IRQ handler, first check if we failed
// due to safe_memcpy, safe_strnlen, or safe_memset. If we did,
// gracefully continue immediately. Because we're in an IRQ handler
// we can't really try to resolve the page fault in a meaningful
// way, so we need to do this before calling into
// MemoryManager::handle_page_fault, which would just bail and
// request a crash
if (handle_safe_access_fault(regs, fault_address))
return;
}
auto current_thread = Thread::current();
if (current_thread) {
current_thread->set_handling_page_fault(true);
PerformanceManager::add_page_fault_event(*current_thread, regs);
}
ScopeGuard guard = [current_thread] {
if (current_thread)
current_thread->set_handling_page_fault(false);
};
VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
if (!faulted_in_kernel) {
bool has_valid_stack_pointer = current_thread->process().address_space().with([&](auto& space) {
return MM.validate_user_stack(*space, userspace_sp);
});
if (!has_valid_stack_pointer) {
dbgln("Invalid stack pointer: {}", userspace_sp);
return handle_crash(regs, "Bad stack on page fault", SIGSEGV);
}
}
PageFault fault { regs.exception_code, VirtualAddress { fault_address } };
auto response = MM.handle_page_fault(fault);
if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory || response == PageFaultResponse::BusError) {
if (faulted_in_kernel && handle_safe_access_fault(regs, fault_address)) {
// If this would be a ring0 (kernel) fault and the fault was triggered by
// safe_memcpy, safe_strnlen, or safe_memset then we resume execution at
// the appropriate _fault label rather than crashing
return;
}
if (response == PageFaultResponse::BusError && current_thread->has_signal_handler(SIGBUS)) {
current_thread->send_urgent_signal_to_self(SIGBUS);
return;
}
if (response != PageFaultResponse::OutOfMemory && current_thread) {
if (current_thread->has_signal_handler(SIGSEGV)) {
current_thread->send_urgent_signal_to_self(SIGSEGV);
return;
}
}
dbgln("Unrecoverable page fault, {}{}{} address {}",
fault.is_reserved_bit_violation() ? "reserved bit violation / " : "",
fault.is_instruction_fetch() ? "instruction fetch / " : "",
fault.is_write() ? "write to" : "read from",
VirtualAddress(fault_address));
constexpr FlatPtr malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE);
constexpr FlatPtr free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE);
constexpr FlatPtr kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE);
constexpr FlatPtr kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE);
if (response == PageFaultResponse::BusError) {
dbgln("Note: Address {} is an access to an undefined memory range of an Inode-backed VMObject", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be uninitialized malloc() memory", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be recently free()'d memory", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be uninitialized kmalloc() memory", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be recently kfree()'d memory", VirtualAddress(fault_address));
} else if (fault_address < 4096) {
dbgln("Note: Address {} looks like a possible nullptr dereference", VirtualAddress(fault_address));
} else if constexpr (SANITIZE_PTRS) {
constexpr FlatPtr refptr_scrub_pattern = explode_byte(REFPTR_SCRUB_BYTE);
constexpr FlatPtr nonnullrefptr_scrub_pattern = explode_byte(NONNULLREFPTR_SCRUB_BYTE);
constexpr FlatPtr ownptr_scrub_pattern = explode_byte(OWNPTR_SCRUB_BYTE);
constexpr FlatPtr nonnullownptr_scrub_pattern = explode_byte(NONNULLOWNPTR_SCRUB_BYTE);
constexpr FlatPtr lockrefptr_scrub_pattern = explode_byte(LOCKREFPTR_SCRUB_BYTE);
constexpr FlatPtr nonnulllockrefptr_scrub_pattern = explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE);
if ((fault_address & 0xffff0000) == (refptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (nonnullrefptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (ownptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed OwnPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (nonnullownptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed NonnullOwnPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (lockrefptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
} else if ((fault_address & 0xffff0000) == (nonnulllockrefptr_scrub_pattern & 0xffff0000)) {
dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
}
}
if (current_thread) {
auto& current_process = current_thread->process();
if (current_process.is_user_process()) {
auto fault_address_string = KString::formatted("{:p}", fault_address);
auto fault_address_view = fault_address_string.is_error() ? ""sv : fault_address_string.value()->view();
(void)current_process.try_set_coredump_property("fault_address"sv, fault_address_view);
(void)current_process.try_set_coredump_property("fault_type"sv, fault.type() == PageFault::Type::PageNotPresent ? "NotPresent"sv : "ProtectionViolation"sv);
StringView fault_access;
if (fault.is_instruction_fetch())
fault_access = "Execute"sv;
else
fault_access = fault.access() == PageFault::Access::Read ? "Read"sv : "Write"sv;
(void)current_process.try_set_coredump_property("fault_access"sv, fault_access);
}
}
if (response == PageFaultResponse::BusError)
return handle_crash(regs, "Page Fault (Bus Error)", SIGBUS, false);
return handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory);
} else if (response == PageFaultResponse::Continue) {
dbgln_if(PAGE_FAULT_DEBUG, "Continuing after resolved page fault");
} else {
VERIFY_NOT_REACHED();
}
fault.handle(regs);
}
EH_ENTRY_NO_CODE(1, debug);

View File

@ -17,6 +17,7 @@ set(KERNEL_HEAP_SOURCES
set(KERNEL_SOURCES
AddressSanitizer.cpp
Arch/PageFault.cpp
Bus/PCI/Controller/HostController.cpp
Bus/PCI/Controller/MemoryBackedHostBridge.cpp
Bus/PCI/Controller/VolumeManagementDevice.cpp