From c04c8080bf24dfee2886c5508a6bcb03bb9cbe26 Mon Sep 17 00:00:00 2001 From: hky1999 Date: Sat, 19 Oct 2024 19:15:06 +0800 Subject: [PATCH 01/48] [feat] functions essential to hypervisor implementation (#29) * update aarch64 hv part in arceos (#15) * Modify platform configuration of aarch64-qemu-virt-hv for GICv2 with virtualization (#20) * refactor: totally remove hv related trap handlers from axhal/arch/aarch64 (#22) * Add gic trait impl (#26) * [refactor] remove hv related scripts and configs from arceos --- Cargo.lock | 15 +- api/axfeat/Cargo.toml | 3 + modules/axhal/Cargo.toml | 5 +- modules/axhal/src/arch/aarch64/mod.rs | 49 +++++- .../axhal/src/platform/aarch64_common/boot.rs | 153 +++++++++++++++++- .../platform/aarch64_common/generic_timer.rs | 36 ++++- .../axhal/src/platform/aarch64_common/gic.rs | 143 ++++++++++++++++ .../src/platform/aarch64_qemu_virt/mod.rs | 2 + modules/axruntime/Cargo.toml | 2 +- modules/axtask/Cargo.toml | 2 +- ulib/axstd/Cargo.toml | 3 + 11 files changed, 396 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f713d8cdb4..272bbf376d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d25e73c949c69f75d1b9dba39c5475523403b31eb8c2fdc99da4dc33bc1aca" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2?rev=7acc0dd#7acc0dd1e1fefa90b4cd45342072d2350a74b071" dependencies = [ + "crate_interface", "tock-registers", ] @@ -446,6 +446,8 @@ dependencies = [ "axlog", "bitflags 2.6.0", "cfg-if", + "cortex-a", + "crate_interface", "dw_apb_uart", "handler_table", "int_ratio", @@ -814,6 +816,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "cortex-a" +version = "8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8256fd5103e10027467cc7a97c9ff27fcc4547ea24864da0aff2e7aef6e18e28" +dependencies = [ + "tock-registers", +] + [[package]] name = "cpumask" version = "0.1.0" diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index ab86e937b5..804531f671 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -58,6 +58,9 @@ driver-ramdisk = ["axdriver?/ramdisk", "axfs?/use-ramdisk"] driver-ixgbe = ["axdriver?/ixgbe"] driver-bcm2835-sdhci = ["axdriver?/bcm2835-sdhci"] +#Hypervisor support +hv = ["axhal/hv"] + # Logging log-level-off = ["axlog/log-level-off"] log-level-error = ["axlog/log-level-error"] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index b08438ef73..d2a5b8d38e 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -18,6 +18,7 @@ irq = [] tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] default = [] +hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "arm_gicv2/el2", "dep:crate_interface"] [dependencies] log = "=0.4.21" @@ -37,6 +38,7 @@ page_table_multiarch = { version = "0.5", optional = true } axlog = { workspace = true } axconfig = { workspace = true } axalloc = { workspace = true, optional = true } +cortex-a = { version = "8.1.1", optional = true } [target.'cfg(target_arch = "x86_64")'.dependencies] x86 = "0.52" @@ -53,7 +55,8 @@ riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.4" tock-registers = "0.8" -arm_gicv2 = "0.1" +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", rev = "7acc0dd" } +crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } dw_apb_uart = "0.1" diff --git a/modules/axhal/src/arch/aarch64/mod.rs b/modules/axhal/src/arch/aarch64/mod.rs index 19c0134bb8..9a850cb47f 100644 --- a/modules/axhal/src/arch/aarch64/mod.rs +++ b/modules/axhal/src/arch/aarch64/mod.rs @@ -5,6 +5,10 @@ mod trap; use core::arch::asm; +#[cfg(feature = "hv")] +use aarch64_cpu::registers::{TTBR0_EL2, VBAR_EL2}; +//Todo: remove this, when hv is enabled, `TTBR1_EL1` is not used. +#[cfg_attr(feature = "hv", allow(unused_imports))] use aarch64_cpu::registers::{DAIF, TPIDR_EL0, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; use memory_addr::{PhysAddr, VirtAddr}; use tock_registers::interfaces::{Readable, Writeable}; @@ -49,8 +53,13 @@ pub fn halt() { /// Returns the physical address of the page table root. #[inline] pub fn read_page_table_root() -> PhysAddr { + #[cfg(not(feature = "hv"))] let root = TTBR1_EL1.get(); - pa!(root as usize) + + #[cfg(feature = "hv")] + let root = TTBR0_EL2.get(); + + PhysAddr::from(root as usize) } /// Reads the `TTBR0_EL1` register. @@ -68,8 +77,17 @@ pub unsafe fn write_page_table_root(root_paddr: PhysAddr) { let old_root = read_page_table_root(); trace!("set page table root: {:#x} => {:#x}", old_root, root_paddr); if old_root != root_paddr { - // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff) - TTBR1_EL1.set(root_paddr.as_usize() as _); + #[cfg(not(feature = "hv"))] + { + // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff) + TTBR1_EL1.set(root_paddr.as_usize() as _); + } + + #[cfg(feature = "hv")] + { + // kernel space page table at EL2 use TTBR0_EL2 (0x0000_0000_0000_0000..0x0000_ffff_ffff_ffff) + TTBR0_EL2.set(root_paddr.as_usize() as _); + } flush_tlb(None); } } @@ -92,10 +110,24 @@ pub unsafe fn write_page_table_root0(root_paddr: PhysAddr) { pub fn flush_tlb(vaddr: Option) { unsafe { if let Some(vaddr) = vaddr { - asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) vaddr.as_usize()) + #[cfg(not(feature = "hv"))] + { + asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) vaddr.as_usize()) + } + #[cfg(feature = "hv")] + { + asm!("tlbi vae2is, {}; dsb sy; isb", in(reg) vaddr.as_usize()) + } } else { // flush the entire TLB - asm!("tlbi vmalle1; dsb sy; isb") + #[cfg(not(feature = "hv"))] + { + asm!("tlbi vmalle1; dsb sy; isb") + } + #[cfg(feature = "hv")] + { + asm!("tlbi alle2is; dsb sy; isb") + } } } } @@ -108,8 +140,11 @@ pub fn flush_icache_all() { /// Sets the base address of the exception vector (writes `VBAR_EL1`). #[inline] -pub fn set_exception_vector_base(vbar_el1: usize) { - VBAR_EL1.set(vbar_el1 as _); +pub fn set_exception_vector_base(vbar: usize) { + #[cfg(not(feature = "hv"))] + VBAR_EL1.set(vbar as _); + #[cfg(feature = "hv")] + VBAR_EL2.set(vbar as _); } /// Flushes the data cache line (64 bytes) at the given virtual address diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index 31a5e222db..e8dbab6f72 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -1,6 +1,9 @@ use aarch64_cpu::{asm, asm::barrier, registers::*}; use core::ptr::addr_of_mut; -use page_table_entry::aarch64::{A64PTE, MemAttr}; +use memory_addr::PhysAddr; +//Todo: remove this, when hv is enabled, `MemAttr` is not used. +#[cfg_attr(feature = "hv", allow(unused_imports))] +use page_table_entry::aarch64::{MemAttr, A64PTE}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::{TASK_STACK_SIZE, plat::PHYS_VIRT_OFFSET}; @@ -14,6 +17,7 @@ static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; #[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; +#[cfg(not(feature = "hv"))] unsafe fn switch_to_el1() { SPSel.write(SPSel::SP::ELx); SP_EL0.set(0); @@ -57,6 +61,7 @@ unsafe fn switch_to_el1() { } } +#[cfg(not(feature = "hv"))] unsafe fn init_mmu() { MAIR_EL1.set(MemAttr::MAIR_VALUE); @@ -101,6 +106,7 @@ unsafe fn init_boot_page_table() { } /// The earliest entry point for the primary CPU. +#[cfg(not(feature = "hv"))] #[naked] #[unsafe(no_mangle)] #[unsafe(link_section = ".text.boot")] @@ -141,6 +147,7 @@ unsafe extern "C" fn _start() -> ! { } /// The earliest entry point for the secondary CPUs. +#[cfg(not(feature = "hv"))] #[cfg(feature = "smp")] #[naked] #[unsafe(no_mangle)] @@ -169,3 +176,147 @@ unsafe extern "C" fn _start_secondary() -> ! { entry = sym crate::platform::rust_entry_secondary, ) } + +#[cfg(feature = "hv")] +unsafe fn switch_to_el2() { + SPSel.write(SPSel::SP::ELx); + let current_el = CurrentEL.read(CurrentEL::EL); + + if current_el == 3 { + SCR_EL3.write( + SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64, + ); + SPSR_EL3.write( + SPSR_EL3::M::EL2h + + SPSR_EL3::D::Masked + + SPSR_EL3::A::Masked + + SPSR_EL3::I::Masked + + SPSR_EL3::F::Masked, + ); + ELR_EL3.set(LR.get()); + SP_EL1.set(BOOT_STACK.as_ptr_range().end as u64); + // This should be SP_EL2. To + asm::eret(); + } +} + +#[cfg(feature = "hv")] +unsafe fn init_mmu_el2() { + // Set EL1 to 64bit. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Device-nGnRE memory + let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; + // Normal memory + let attr1 = MAIR_EL2::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL2::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc; + MAIR_EL2.write(attr0 + attr1); // 0xff_04 + + // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 40 bits. + let tcr_flags0 = TCR_EL2::TG0::KiB_4 + + TCR_EL2::SH0::Inner + + TCR_EL2::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::T0SZ.val(16); + TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); + barrier::isb(barrier::SY); + + let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + TTBR0_EL2.set(root_paddr); + + // Flush the entire TLB + crate::arch::flush_tlb(None); + + // Enable the MMU and turn on I-cache and D-cache + SCTLR_EL2.modify(SCTLR_EL2::M::Enable + SCTLR_EL2::C::Cacheable + SCTLR_EL2::I::Cacheable); + barrier::isb(barrier::SY); +} + +#[cfg(feature = "hv")] +unsafe fn cache_invalidate(cache_level: usize) { + core::arch::asm!( + r#" + msr csselr_el1, {0} + mrs x4, ccsidr_el1 // read cache size id. + and x1, x4, #0x7 + add x1, x1, #0x4 // x1 = cache line size. + ldr x3, =0x7fff + and x2, x3, x4, lsr #13 // x2 = cache set number – 1. + ldr x3, =0x3ff + and x3, x3, x4, lsr #3 // x3 = cache associativity number – 1. + clz w4, w3 // x4 = way position in the cisw instruction. + mov x5, #0 // x5 = way counter way_loop. + // way_loop: + 1: + mov x6, #0 // x6 = set counter set_loop. + // set_loop: + 2: + lsl x7, x5, x4 + orr x7, {0}, x7 // set way. + lsl x8, x6, x1 + orr x7, x7, x8 // set set. + dc cisw, x7 // clean and invalidate cache line. + add x6, x6, #1 // increment set counter. + cmp x6, x2 // last set reached yet? + ble 2b // if not, iterate set_loop, + add x5, x5, #1 // else, next way. + cmp x5, x3 // last way reached yet? + ble 1b // if not, iterate way_loop + "#, + in(reg) cache_level, + options(nostack) + ); +} + +#[cfg(feature = "hv")] +#[naked] +#[no_mangle] +#[link_section = ".text.boot"] +unsafe extern "C" fn _start() -> ! { + // PC = 0x8_0000 + // X0 = dtb + core::arch::asm!(" + // disable cache and MMU + mrs x1, sctlr_el2 + bic x1, x1, #0xf + msr sctlr_el2, x1 + + // cache_invalidate(0): clear dl1$ + mov x0, #0 + bl {cache_invalidate} + mov x0, #2 + bl {cache_invalidate} + + mrs x19, mpidr_el1 + and x19, x19, #0xffffff // get current CPU id + mov x20, x0 // save DTB pointer + + adrp x8, {boot_stack} // setup boot stack + add x8, x8, {boot_stack_size} + mov sp, x8 + + bl {switch_to_el2} // switch to EL1 + bl {enable_fp} // enable fp/neon + bl {init_boot_page_table} + bl {init_mmu_el2} + + mov x8, {phys_virt_offset} // set SP to the high address + add sp, sp, x8 + + mov x0, x19 // call rust_entry(cpu_id, dtb) + mov x1, x20 + ldr x8, ={entry} + blr x8 + b .", + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + options(noreturn), + ); +} diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index 8b5bcb97b6..8af7ffc728 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -34,8 +34,8 @@ pub fn epochoffset_nanos() -> u64 { /// Set a one-shot timer. /// -/// A timer interrupt will be triggered at the specified monotonic time deadline (in nanoseconds). -#[cfg(feature = "irq")] +/// A timer interrupt will be triggered at the given deadline (in nanoseconds). +#[cfg(all(feature = "irq", not(feature = "hv")))] pub fn set_oneshot_timer(deadline_ns: u64) { let cnptct = CNTPCT_EL0.get(); let cnptct_deadline = nanos_to_ticks(deadline_ns); @@ -48,6 +48,23 @@ pub fn set_oneshot_timer(deadline_ns: u64) { } } +#[cfg(all(feature = "irq", feature = "hv"))] +pub fn set_oneshot_timer(deadline_ns: u64) { + let cnptct = CNTPCT_EL0.get(); + let cnptct_deadline = nanos_to_ticks(deadline_ns); + if cnptct < cnptct_deadline { + let interval = cnptct_deadline - cnptct; + debug_assert!(interval <= u32::MAX as u64); + unsafe { + core::arch::asm!("msr CNTHP_TVAL_EL2, {}", in(reg) interval); + } + } else { + unsafe { + core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); + } + } +} + /// Early stage initialization: stores the timer frequency. pub(crate) fn init_early() { let freq = CNTFRQ_EL0.get(); @@ -77,10 +94,21 @@ pub(crate) fn init_early() { } pub(crate) fn init_percpu() { - #[cfg(feature = "irq")] + #[cfg(all(feature = "irq", not(feature = "hv")))] { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); CNTP_TVAL_EL0.set(0); - crate::platform::irq::set_enable(crate::platform::irq::TIMER_IRQ_NUM, true); } + #[cfg(all(feature = "irq", feature = "hv"))] + { + // ENABLE, bit [0]， Enables the timer. + let ctl = 1; + let tval = 0; + unsafe { + core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) ctl); + core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) tval); + } + } + #[cfg(feature = "irq")] + crate::platform::irq::set_enable(crate::platform::irq::TIMER_IRQ_NUM, true); } diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index f8b21c9113..5fdf37eb2b 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -7,9 +7,14 @@ use memory_addr::PhysAddr; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; +#[cfg(not(feature = "hv"))] /// The timer IRQ number. pub const TIMER_IRQ_NUM: usize = translate_irq(14, InterruptType::PPI).unwrap(); +#[cfg(feature = "hv")] +/// Non-secure EL2 Physical Timer irq number. +pub const TIMER_IRQ_NUM: usize = translate_irq(10, InterruptType::PPI).unwrap(); + /// The UART IRQ number. pub const UART_IRQ_NUM: usize = translate_irq(UART_IRQ, InterruptType::SPI).unwrap(); @@ -58,3 +63,141 @@ pub(crate) fn init_primary() { pub(crate) fn init_secondary() { GICC.init(); } + +#[cfg(feature = "hv")] +mod gic_if { + + use super::{GICC, GICD}; + use arm_gicv2::GicTrait; + struct GicIfImpl; + + #[crate_interface::impl_interface] + impl GicTrait for GicIfImpl { + /// Sets the enable status of a specific interrupt in the GICD (Distributor). + /// + /// # Parameters + /// - `vector`: The interrupt vector number, identifying the interrupt to be enabled or disabled. + /// - `enable`: A boolean value indicating whether to enable the interrupt. `true` enables the interrupt, `false` disables it. + /// + /// This function locks and accesses the GICD controller, then sets the enable status of the specified interrupt vector based on the `enable` parameter. + /// It provides a mechanism for controlling whether interrupts can trigger CPU responses, used for interrupt management. + fn set_enable(vector: usize, enable: bool) { + GICD.lock().set_enable(vector, enable); + } + + /// Retrieves the enable status of a specified interrupt vector from the GICD. + /// + /// # Parameters + /// - `vector`: The index of the interrupt vector, used to identify a specific interrupt source. + /// + /// # Returns + /// - `bool`: Indicates whether the specified interrupt vector is enabled. `true` means the interrupt vector is enabled, `false` means it is not enabled. + fn get_enable(vector: usize) -> bool { + GICD.lock().get_enable(vector) + } + + /// Get the type of the GICD register + /// + /// This function locks the GICD and calls its internal `get_typer` method to retrieve the type of the GICD register + /// + /// # Returns + /// * `u32` - The type of the GICD register + fn get_typer() -> u32 { + GICD.lock().get_typer() + } + + /// Get the Implementer ID Register (IIDR) of the interrupt controller + /// + /// This function locks the GICD (interrupt controller) and calls its `get_iidr` method to retrieve the value of the Implementer ID Register. + /// This register can be used to identify the specific hardware implementer and version. + fn get_iidr() -> u32 { + GICD.lock().get_iidr() + } + + /// Set the state of an interrupt source + /// + /// This function updates the state of a specific interrupt source in the GICD (Interrupt Controller). + /// It first locks the GICD and then updates the interrupt source state using the provided interrupt ID (`int_id`), + /// new state value (`state`), and current CPU ID (`current_cpu_id`). + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `state`: The new state value for the interrupt source. + /// - `current_cpu_id`: The ID of the current CPU. + fn set_state(int_id: usize, state: usize, current_cpu_id: usize) { + GICD.lock().set_state(int_id, state, current_cpu_id); + } + + /// Get the state of an interrupt source + /// + /// This function retrieves the current state of a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// + /// Returns: + /// - The current state value. + fn get_state(int_id: usize) -> usize { + GICD.lock().get_state(int_id) + } + + /// Set the ICFGR (Interrupt Configuration and Control Register) + /// + /// This function sets the configuration of a specific interrupt source in the ICFGR. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `cfg`: The new configuration value. + fn set_icfgr(int_id: usize, cfg: u8) { + GICD.lock().set_icfgr(int_id, cfg); + } + + /// Get the target CPU for an interrupt source + /// + /// This function retrieves the target CPU for a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// + /// Returns: + /// - The target CPU ID. + fn get_target_cpu(int_id: usize) -> usize { + GICD.lock().get_target_cpu(int_id) + } + + /// Set the target CPU for an interrupt source + /// + /// This function sets the target CPU for a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `target`: The new target CPU value. + fn set_target_cpu(int_id: usize, target: u8) { + GICD.lock().set_target_cpu(int_id, target); + } + + /// Get the priority of an interrupt source + /// + /// This function retrieves the priority of a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// + /// Returns: + /// - The priority value. + fn get_priority(int_id: usize) -> usize { + GICD.lock().get_priority(int_id) + } + + /// Set the priority of an interrupt source + /// + /// This function sets the priority of a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `priority`: The new priority value. + fn set_priority(int_id: usize, priority: u8) { + GICD.lock().set_priority(int_id, priority); + } + } +} diff --git a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs index 13a865384d..1d50d10d0b 100644 --- a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs @@ -30,6 +30,7 @@ unsafe extern "C" { pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { crate::mem::clear_bss(); crate::arch::set_exception_vector_base(exception_vector_base as usize); + #[cfg(not(feature = "hv"))] crate::arch::write_page_table_root0(0.into()); // disable low address access crate::cpu::init_primary(cpu_id); super::aarch64_common::pl011::init_early(); @@ -38,6 +39,7 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { } #[cfg(feature = "smp")] +#[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { crate::arch::set_exception_vector_base(exception_vector_base as usize); crate::arch::write_page_table_root0(0.into()); // disable low address access diff --git a/modules/axruntime/Cargo.toml b/modules/axruntime/Cargo.toml index e6029da263..3f76289949 100644 --- a/modules/axruntime/Cargo.toml +++ b/modules/axruntime/Cargo.toml @@ -37,7 +37,7 @@ axdisplay = { workspace = true, optional = true } axtask = { workspace = true, optional = true } crate_interface = "0.1" -percpu = { version = "0.1", optional = true } +percpu = { version = "0.1.4", optional = true } kernel_guard = { version = "0.1", optional = true } chrono = { version = "0.4.38", default-features = false } diff --git a/modules/axtask/Cargo.toml b/modules/axtask/Cargo.toml index 6232c42f9d..f5d0ad73aa 100644 --- a/modules/axtask/Cargo.toml +++ b/modules/axtask/Cargo.toml @@ -40,7 +40,7 @@ cfg-if = "1.0" log = "=0.4.21" axhal = { workspace = true } axconfig = { workspace = true, optional = true } -percpu = { version = "0.1", optional = true } +percpu = { version = "0.1.4", optional = true } kspin = { version = "0.1", optional = true } lazyinit = { version = "0.2", optional = true } memory_addr = { version = "0.3", optional = true } diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index 4adb9125b0..9801bc7c9c 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -66,6 +66,9 @@ driver-ramdisk = ["axfeat/driver-ramdisk"] driver-ixgbe = ["axfeat/driver-ixgbe"] driver-bcm2835-sdhci = ["axfeat/driver-bcm2835-sdhci"] +# Hypervisor support +hv = ["axfeat/hv"] + # Logging log-level-off = ["axfeat/log-level-off"] log-level-error = ["axfeat/log-level-error"] From abdea81fd2e4d057d640693b8be34126d9e4090e Mon Sep 17 00:00:00 2001 From: Debin Date: Tue, 22 Oct 2024 11:10:22 +0800 Subject: [PATCH 02/48] [feat]Adapting for rk3588. (#28) --- Makefile | 2 + doc/figures/RKDevTool3.3.png | Bin 0 -> 92129 bytes doc/platform_rk3588.md | 9 ++ modules/axhal/build.rs | 2 + .../axhal/src/platform/aarch64_common/boot.rs | 4 +- .../axhal/src/platform/aarch64_common/mod.rs | 5 +- .../platform/aarch64_rk3588j/dw_apb_uart.rs | 147 ++++++++++++++++++ .../axhal/src/platform/aarch64_rk3588j/mem.rs | 68 ++++++++ .../axhal/src/platform/aarch64_rk3588j/mod.rs | 66 ++++++++ .../axhal/src/platform/aarch64_rk3588j/mp.rs | 20 +++ modules/axhal/src/platform/mod.rs | 3 + platforms/aarch64-rk3588j.toml | 68 ++++++++ scripts/make/rk3588.mk | 13 ++ 13 files changed, 404 insertions(+), 3 deletions(-) create mode 100644 doc/figures/RKDevTool3.3.png create mode 100644 doc/platform_rk3588.md create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/mem.rs create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/mod.rs create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/mp.rs create mode 100644 platforms/aarch64-rk3588j.toml create mode 100644 scripts/make/rk3588.mk diff --git a/Makefile b/Makefile index f71378e3d8..c30c887f9d 100644 --- a/Makefile +++ b/Makefile @@ -142,6 +142,8 @@ ifeq ($(PLAT_NAME), aarch64-raspi4) include scripts/make/raspi4.mk else ifeq ($(PLAT_NAME), aarch64-bsta1000b) include scripts/make/bsta1000b-fada.mk +else ifeq ($(PLATFORM_NAME), aarch64-rk3588j) + include scripts/make/rk3588.mk endif defconfig: _axconfig-gen diff --git a/doc/figures/RKDevTool3.3.png b/doc/figures/RKDevTool3.3.png new file mode 100644 index 0000000000000000000000000000000000000000..d390f4219b81a1752fe22216aff5c200f51c0af5 GIT binary patch literal 92129 zcmeFY1yr2Nwl3NUPJje=cMqc98-qmln!3lC5K`V7EDe>93Ti1dgK@EG?I67Hj4-2m!`K|X$r^hfJ}fBT@Jqhg?7 zK0!i${6_`LX8-`oLpeG&>JwySObnDq0Av(Yw8u#3j|edch=_HQQ)ck+NyvmjI(nXs zi0N5ecdx{xR9=2TA?@1whNj6WQd&(9Ov|_gItC_Y76FB@x;{o8J`pJmGq-p2uVm${ zZ611|`j_beejmrfp#JCx9Ru^BTpbti2pRCuDHb;J13*Q35bt9oWE5OfG(3D-9u2d& z>JxMVLS89dh)c@&1cI1^j_;Kw=w0HcFO#QQ@wqib^bF>!7>xV^f)9RxDi*J<7_Z)(-Cj2{sTgW{@0~)#fr{#4z3QPAP|B~F8*m%%&5SZBx>EPi zjN_BP?YJ5cW2PYfHoGvF$R0KqvA0sUqt}QMSF_x-AFb7WTpFkwU8T*;qU2L=uT#*W zRixJw)^v*K(A>mOBA~N}!>C3P28pr;TJ8cJtiTrqS(wISM3oU{h#kH1{6+|oIp$K& z*ZHitQcDZV;De+_rf4}}AJ2#@h@@NxPEpcM811m!$206owS%a%ukPQ1(dE|nM?^%V zv2oq9IiHXXAA4U@h!3PMv8cVUNAn$PTHR8f-hJi0uMQ*@dd0}{Q`DR`mgJhOmRf@r zOVT5ioxrl>##&JP^Uz$I9R5r3x?7LWZ%V5(udFD+qQ(wRjJB|IZ?h)j4sVou`?Z=a zsD#~6R{U*-amd97o}TCnyD@Ln#ShiqJ#1qYEWuo+j+xO6eYcBSl}a{Lg;qM^My1M* z){D<=bG62Gj2y}sJC0+`1*g<2xNcP!`|c+?l5LkNb_c4nyIpeW?`+HhciUo?U4A6G zHqi%+33}t7ZTtevXw#bfu4un6A4&(b_60|}+j2@ue;Jl+z`B{W!9};z7yib@Dh#iC zZIEk`%tR2Ampe^&Bf*>+KDNO0KpTVY7PC7|KXiuAVBJ^>XGmk~EN6ZW8_=$yB0|cS zD1Hi#i;gciFCDStN&Y6wj8{qS@k}~Vg^i&+gU&=$U%b;F59n*$A#EqKz&Wdzp0z*H z6M?buLx}>a>ww#RCTs5`c$rcwBv1j?5yk~xILu9mWHyNC6p&{=^^h|8jMb(gjwqcWaj zXjU#2(SBM$CZg?D&OfZwLo%@W^jVFZi7tCPq2pX(wwbxo6{okVvQd7YP(CXq8NZxV zDw(UB^?IaUs3N|qhOo*e4Ly96e*rauxr~8|-`L3rx{r*|f*R*WqDAU@=j=7 z)BKD9RA!I}m{qpwj=z?vuY}Qja#vB)E=efQF*s~f!H+9%mjON8fK(LZl24<7(oZ;le;=BFN@nJy3;h8Z^;9tiQu}*EUzfv z#6y@(S4E)e*MG$fTn^+6)?zp2uB3(NA`v|_JCOxxg%kmOwYiBE-u4m^jOfc^Jmxl5 zY+XG5&-Rz5bqsBq()&46EC=Vb=6GMdNEsPA@W-y8yrJjmgM*$)KV=b#1QYk8l~xEx zFcwbj{?4b#pp3JPr$79i{KG6WVeV zEtcNcoPzLVR%;6%EUvC59Myqyup4dV8E25eegWb)=0-|WTvQo;z$Gt7`nCSV)4wkk z%GO(6_aT1eIUZb*Z+*&KHq;=C_>#tDH75Eb?^`G>L(>#`GZ%dy+tJp72FG%FOJ7F5 zgMVey(?;!y=;ra+3S$uIfX;?YYJ~uK19ik2%VDyZ@3X3nIbX6XHj!G9(hMWSvhl&0 zQ4%4lVt1CyC9@E1A&n4EsNrdH$!96|0Z>Sss2b zF>@4^5H9))fH7E165G#9a7YQz;pUS@#kNavG>}O7Ycc zv8YkgzMrCC35XQ97PQ*q|I zs)h3JQh52hl3|a%^>iS;GsOSl|MyISa=tfUy89P^`tO=(bxwV?WZ(SHzh$D9y*yC~ z;<|XCb;IeDl-t(8kAF+od%3SF7{BkEw)AIq{Vh`>>6mXySuYPbrgO#mJElr`G|Ka@ zUHm;FsEzV34G$3gyF_rXPHn$t8vS>PK)CN;NB{ST;GZM^mv@2RO+e~_2>vPaKYZ^0 z%WeEqTmBPmIsF1hfIl{5vL`zkc6gIw>+2vQn-MCdGg@IJB4ET|@eJ^y(WOOH!=!G% zz(A{_d9rfo?6Cz&CydU#DcK&*{PQxK^7t3PIriTOSuN|gH-tYEa@~ZcEG)u)m7E<2 zD+0Dqp>X}k;cE8lV7BTyPT9B#Y}q^rFip6-qrOK!Mg5PrOz=?Xg-1e zWpT$o%ZN>VG+%Q`Q7yQLXC$`RzElv0T2Y+^B4jchFz)_D_NXB1Ji+%%4S&Fe=krC| zB>XouM*D|^Ye}ZuAXyzl`@-?0Cm>L1qZ6n+sJn80x>OfzIAaiCz@@7;pwm7pJ>=?! z4EOck%+NWUt(7IP8dVsVB{0ZMXUR>A+g9P9RN>z(;GZl^w^I?dd-(e=zfYUKmYMu4 zS}eEkZqTQIDB&0R=Ep@;>a?}wA&j=*g1XFX2c@1hvM$_&#jV+@t&8$`a@#9!O>lqW z^3+aEawr`)+6_{S2x=gpKsqrHkjXYVV#UZrM?$$Fak>Peli!^;Rm2DiG*BSZfO~Yl zQb}Np;ZFSs9cri#6O#Dx_#|d09GYF2X1)}Y*aeyXmKx1Shfwlu*^0E9U{L5$%xp0{ z8IB&%^%m6Ps2vNEfZ4RA7v9>$!`O1!*b42wnV<-i(D!=_#LR7z2+sGRH4Y%FHls!~p$2XGY$h3`)ET2kw;;vEn@skzq?M&UA9ied1 zf#yK!i)?p7fdv7JsS!efDWk%t_~2NG!f8ufZkW5@yn%gZnv5pQfzcV3sw%Jj;x)A8 z8j~rDN4BQXk>#l_6Rm+LqS&l1Eo617K(TlndKixhcFn%gf6Tzc{@O1_z5#1m7`K!H z+Y&e}G+1k|N*#9aQMLw6L*;VXj7^wgXLsJ=5f$VPuu*GwTg%;dmOu=;72tKkS6 zN*31ib_lmZ`GvxoA&2n+{P}Zy;C$kQp$ol}xm1$i0TS2FU@Fx8XS9{d7WQQFB2ST0l=BLI8muDJrZ{b4;LVHt*;GBGX=o-fKr7BX}OX%aYfsFP ztzp9Gndl)jAMGMT;at2aDL4y}?Ad9J)Td9NZ;vYm(JOR&6;q3iF%+ z6{<07s2sel=q=sgBs~=~8ExsaYM?I6Q|yoF!+zYoaRWChd-YCD6aJFRTr6w!g-Zn% zP&&n-=~>&-(Hm57SBhgsG9TS$Ez2B8zcJnIVq1y7@{18Kj_;V_LEFM?^Je9ik86#8W=cwv#P|fba9`dY--6} zfViyAUr^z`I}|AELnYlKBhgYSr77e?KYKM~)yO5No?*XfC4y^6wzVB%U0cK+7sB6J zd=NSj(UL)zOA3@1h=cRtGuN0v1&z=Idx^|Qc0ZL$i6~Ajxi5=fWNp_G?X`ln_i;qQ zBF1)H{ascxVhf`y+77s$%&#KCnQZ&$=yGBGunU!+iE)+=uG$V7gGO=)0gyo+N_lG8CB}?x zqxI!S0F~I*5+TJAfu-ZJa=Wslqto4Zx$;bBap~$y06=xYr^uva0{}pBY7LCPeG35S zHpWQ20RXJG+2QzM007>2 z;Iuw37(Tir{uW`AHoUmRUrMErk=}o4tukaK2=Z_N4_c;KrZE|z^O;DrX^6Z9V7;Xv z&tls~h%bF!`@BsQ?&883PywGN$DefB9ds%mWm6?j z&IFTtgM%Eu^bE!OKx-SB#a`5hKWFhvui0Q3Gk^KGAYCzulD=U%Jih<%8qPN&xYT*+ zj0Bp4lP9>&IcR(oZ)%hhf5GRv>lq&aHwl_@^M-)md-+&zPaHG+bZJ9d*kS$!&=33t z*q8qWxD6T?0*E73MZ~ueSqc+mR~20Bx%GtrGvJfi{IUv?;PS8F;MK zQTg;gYCO7pboP&@yZ%uF;Q#Xfd-K!@AZ-3=m>Y|IMQ8INq9Q$UQiaQ~)PPIl7XTe0 zx?fJr3WK^&s@TmKxKBsGiVWa{t!RswWr??)zIH?$Y7vG=Gc|!Mr&HFDeU<4 zs`&wJe&pd|=VeZ|H9R`H`JRRI8^%t&&lAZxa3ApIv+e+`tRMb?hF+Q*=~$|9d~IA> zIzJVq_zjb@{*-f-oZXK%IX&^)3w&IU{%LkZkfP}YJOXqDJ z+rfectYywdp7<}VNaBC`9oOVc+3@|0 zhI2DH&g`}P@UM74-nM_a0@7BJ#iKTNln-K``{x4ua{)yEPgp>w>25-|c~tDaURmX5wzZ{-LtPen zTo5i50;DaYmeO9Dtk~YI_B2>SL~i2WEpF6;%H-x3V0Ar%S}7{-vj3dw6VlL%(1f0r3LD+ahHQw^X{}yOP_Yl%vd7fqtx~&sZ=U!MQccQ`(3xnV2Lz`)j|b>hgev`zT4%h zs<0G$oRQ9gnX!%JRkln66+7Rad1M8$e3lBT4ng-iGa~jCTVw(9s(hYmZzzR6gp5eyIoDC znZWZ#C7uB|vrO3W4qQ%S@k9jjdE1Wvj3;8%!S%*q-NCX>XVG;u^$=I^tQJ)W)W}Ec zTe{1fP=ve#pJcZ-*=p^q2&81T`JhjzaHYiCX+mK$gn*r6v@jQEumQ~ZOjDsRJp8;BTTJAQ3wWwghvw3 zk~NlVA}@Do^bRPalX_gopQN~Yx*Big>Km`V6BgdrVR}1dPoV9xwus)G3o7e!py!!A z%G;& z=(pr^K#Z@{@(aLdq8AyN398z$<18F(`!Ln{CN0Oykml77J=q!DtkV^$=&ruJ52@JF z?rjm{i7uC)Xu6(mObb5$tZx%ntX60;>vj6E=Zg(KC?VcO2UZ`KbGXUzDJm|fc8q7 zdCaCyVm>xTLeje&Qr#fLWo;l`-z3nWp)URGyBL$k3u;yo}{*y)36 z^a1<(iOIq-qZBS<$2EWs#tt$EzUBFo5ufk1vOLpbdJ}NW37|BNSRuSn8@+WWB2wBx+3Jn+Z!Q;23irzUk`#~&Icc3 zH%cd;LrP-RFUqE^L_z3-%KX-Kog*>%f?>G&P7`SbaQ^iGQ6(?&`zbZmKLI9 zddX+T)a1BlE{5@I=pnr|b!JhQouz({H6yYr4hr9lXsYs5TTHXDoizxFxnd|jk|I{A zstBAarG8-lRc_4e!A=ehhib?`@>axpc;XR@M{O+qmAyspao?2A+5-BK*pYDY=2jXq zx^#VIV@46#a1$k*Sa#e%Oh39aBvoU|jL=3eid5W7jol_y1ZLDx`1&zC)S_(42<+;Wt15cup1x508dVni z&9+H9hu~mQu|G4y9KJr2>u`gOQd&JzebVJW8?g~%+RCZmJ=Ypt)-O5-hxBQ_NpMGf zkz#>m>9nELVPVBF@OYhJ@%yK=ND)a=3s>D1+P2Onlqyv_zoP!Ha4O;wQaEzebkZ4C zAI15(N)Hw9gl7J^yY%7ys^rW5gb8V~^qk7#Pj>{8b_uzx9z0Dw?cbwTh-Z((%B`Gm z$+RpLPnAMH@F?dn%G)c}FxX!yu*F`fn@Fk22IST*W4C!UcnU9m;G+})Ry^eqlB6@T z`GRgu7Cd_{a*hA#XlL;zF=tXjoI2x~Y3j0%6i_PCl9U6sz%pf*x}i-xWtPZRL;#N~ zcrHSs@frW1rMdJ&mG7r-QvvN(Eg^cgym=)P>#xmqOonl5d`-|al0F7-iY=y8O@;5ns^jKs&pSbWf8vH&+> z8)9syBhmzoUO{q?=UM@qN*Kn$RyqnVlX%){S$lCHRAwgmNO`mfaT!Pi_%J;fedpc0|Cz++qrTTAsN`g41EwKcnFI9$nb8U{3Iwd6cL2j`U2PeeF9)wdB1bE79Qig$SNX_1&P97}@|(&CALM6xb)E zMT>#^)vDs!j>TAqN>%F2GW$GKU(LVbr^QL6Eq?R7XTDHoCfb_>YsE#K4x+F4dRp>$ z#|Va|WuLWo!{674)O6*dwp5WiK$C`r#VWpW8YL|0gb@b%#>tK*jI(-QNG2Xm99Mck z%-!*QU*<9oPJ?aZ2~!$4#8P#9({f-DGi3G}T}0x#@tOpcOdGF0jJHi5$7jEWj{flP z!^%l>NoTKrMPC6~f42Sw-jgk8ctgQE^&y76|ARN8EFtntCZxe!0JO$d$>9aUtSSFvj z*9k_%SGbE>Komz6doh@PjB&{ILaYt7(lR)`j;6#Ql{C$SvpA`#rx@(|R19$x*XrP& z8ixGt{4&3-7;R4G0i~g21gj5CUv!A#B(#T;)D&+M>MC9pQU0tu{sV8)Q~LQ(`_RR5 z?{K@!%|L2*omxB&rOQgTrrht|ui=GLP4jBX9rz6cMFlB}kb&K^hx_#sdlh(1`lCf> zNLz+Rjy$@NHB?)1r|jI?Q~EX4151riD;gm!uS9(R?&X0X&AWk*QKlq_iSP~VtCo^R z=6NVAQWu}f8onj5+x}ExWT#~GYlcUJBcfWkjl)GSMcs(GNrQRqLS}LmU-t7CpnH{< z`TmHqwvILAZN$u~pPA_Mt*zy$;3;Rx$Uxj9WIp{joCkp}samutLwR>h_E~HIW+iw_CZAjiED+IKVjp!;_A!iUIaMi+vo>d_> zqpVJOmK!-lb0-`|bTxpB5gG<-r@GR_uKLt2H?bSsnwiRWEoR;s&xB%`bYMM&OKjK& zsnYUscv$$3On5N|k34W|O9>Y#UMM zJ)_FVB|>HlAakRz2bPr#`~q;vb9flBb5>!+H?6yNy2S4TUn@bQn3OzcFM3Kbxbd4% zt&f~#;172&L#>!C3pixYkV#rc+~vVxn|##9gPbWc?hK7q$x3l)g$kd~Z*1edGRA8% zAP3dxdQsZUI&f`;J33{hxN|yDhuYfy40QU3aEx@zPd#=6Zo#lNcj-%?%urg(vx8{g z(NY^C+_q?Uw~y8B)0;r%h<|_Nqu?S|Rq`IAAq%^Jcre$(sy}{K&YDKxbBI<{%xt~hR;1-*NF zWG(r1sJLcX#U0^Tr1tYu&i#$_F95}B-6;O9*%|aal$}Bz-B0=`pMPq{LQ~g53C$$F z80)vrIHlHpF_P|Ub$}*{;i>T7Wzra$mr7t?jil!vh`*1Ymw#1lU;v*>ovEKW(|&WQ zy*~^3c`@zsQyWxt9q2EhRgt7sk+TF6-K|L9FKo64%w%F@V~2~HjLG>gIBEc9+e)ZcSQL(c8_DdD9)e40Xl18oaNp z_nARGO7Foc5dW+@fDz(ZO`f-zIGcgYq5K`nD~E&1uYnKt{Vei@MGl;g>}Ll}Nf!r5 z8D$ysU|@V{QngIYV+yt=V|V50b|?)wzY1BGW2py?HjR)6iezch;|tP@!@Mb_-=}l? zEn3P48QX(C3ves%DD;H9uf6l4(a3&-&Z?eS^;7w3QwwTrjEuNUnrjyBXSRhO<*^6i7f{NhBEQaq%Y9t z*8%m>lXqC9j(KwCN|9Ekm4`G7awNXpPTE^`2?4&wJY_#=L6HDQQ>V_v&FPYKVY(|# zh$yscy_O#5pZe_8k7mZosCbB7G3Y#dv}VajD>B?bHNtmu(FboK-q0qJBkgBpGRo44 z-;A&IvFzc=f9c|yB`VJ_rX8io8W)`X6}U{2D8)*^S^j*m`pFui0oNVHU9n=)Ix1CN zNXX<{{?B2(pZ6>UwA@c`*e zqAF)q9*^pc8HI&naUlBH5&s(r^6y?N9tKGnn&M=IdGX-lLeJ-WLA+95$fS=H0j(Jv zaw>9&M3QkfD^|MhvZ2+4uaB)Y*B8G;a|z|$iakV9va=%_yHOWbhQ4!_Vv+W-mohRu zbi535B~-)$%B6MMiBbG9?1c631`urExridPIM?#bi=iCZ!jT>Njt zX7390Jn3G{VKLRcy=%>*U^NY9bK}#bA#%X2?m*-?GjgwLEFR#Pp=0cFI z)7JwSN?QtFv8fwKA@Q-92R&=>Fchlpd8HcAwKJ-_5O+;k%?Kye&H!R$FO6J_1t7oS zPn$IGQ@C~P1(qoyt_dYpuv1s>h&rS7AFDVj_C@tZls_J;3r;@s=d(Hz<(tF0O=EE7 z<4>GOFn4?&N)keNuv!nq&9iXbb5rZ*P!^wVP7B4bN}9lJ6mU@ zDQE^M@Fu4VHPH1;Z*cwe7blAucr$4=+r}trcrwxQF}fPddOOq7HG$z^-GP=~XxxYk zv1?+it`SJ0n?b2L@d4y-);rXf_iV%D(1V8%l;&ig~8InbM`tepqO2b^*$fbQUi9U=T+Q&{G z<1IpK&L7$YzEjY2CWIk33Jfu?F=&N!pWSgfR?`wMONy;sPmgsw#5{I6`%p`-oV@uWTo1Te{M1xGVHuF(R_OfM1RU^mVjR}2>Z-FG| z8MEt&_Irol!w2t5yy-&%*%8nf#mE^5%hx>Dngg%V0#6H3TV6YjUf2@e5%+J@=rV5DV41h2!EUwahdeQ`iie^Gn~YG}ARR9jF|dohj#^DIoOA3DT(psrW~CFttIo5oZrIDmsoh zym%~9K6-^B&b>4}qt7$Gp6PZtrynKbe+FQ%{6%N~RW{~nqG@{mFMx{hzz<+|Zdb&g z)9W|=KdycOYA8G(-w=QNx&G%3>M0s;-|dH}F51zbi&uVze%KD}No)9oO!m``ix&jX zkv@xI+=qQ%+H*?fym{JJ>DOI(yYWT+FY+V*I9O(Ed?z0M)enH$u0zD@qBbya#1mE2 zZ~z<+H>|3G^V*-(FeFi#_xW$dfifrZ*iu zZE84A+*|uhR+a`iZ?Ctprt%qn?zc6~6!{)3Ke{?vY3Dv3DV@JlG+v*}e0=w|fI?+! zJ9~)!C~;-qO}T24%*BCKgj+eG)AF;)Eg>wUWG4=f!9e5LQgGrB5N&LcY14z+==tmF z7TJWuuigw1{^i;Zf+~6g`FA_tR0=wfe*s4M14bAmnySC`M4P-@qt+c+J>&e&d=JUH zvZKB1i9zLXxp?~Qw%r+1^>o?QhYEOocy2t==>>Si_+K&YUz9&G;f*}k&e<&@l?>NQ z)PwvfX%r|zoxO4;9sSLe?t;4Y|G~CCAD(ji{Q{hvj>Hu7D%@^(Lupn$<3tq_zFgmj z<`t6EgJzq2Bd~&RISh?$EVKy_1`P&RAxeh2&a#J=!E8@bi-2|==0P8hwd}7FCkqyQ zv`o_N3QTEgEhVYi&ha|-6Ht^sUn1?ziiJE}HBl)kDOpPvD`sV7A#Htg`|dmrUwoje zNR_l=HOP7-74CAAAarEZ7H~Ouv{cc&4=Z>2^bgKoucv^@_#D2##XVZSQI)w-YK5v? zj(DbrcZYwrg+;*YdF_wug{~7mXY6e*s|tyDA-vSy7Cc)C*>ZgN;wBIKb$~Ikk$Tv- zgt9HLbs(xy`PQ+6rFr!ILYX^r_K$;wv-Ydy7KvA82TRAZsfRyX?2kuRZ0e6G_f3B6 za5tgfnGd!W^ev$^-P0od2L#}u|6YQIvp5ZOZCFr!k0(Yp=v&b6c1Nwo1>3a2WzZK& z5kF4O%sJPeK*X6@r+t}WeK`Bvd<2yhE}KAnxT`Y%Ir*jh0ugAFOJe6m-zhrze?UO& zOPPMM0ab<(P`JW@gY1Q#d#(rSN5n1naf5H0cXE+yb#0u`;j!-?bg~eAWaGBs zTBqIP88N#h$Wc0IBuHr;)^rtYSBEcJ-` zx8!f|e|rf&jRT`_f`ASqeRVnUxY>00ELo85PDlIP(dCvw)@bjj%t~Y1F-%A`mEP3x zFh%Pqso8v7i*T9UB6YT=k`;0-btD<~68ZgOiQkgH!T;?gOfRQ_5GMOFpuB|vcWzfX zew6@f<#?cf?b7~my;v4BGrYsucSReV^<1ufZh@<4iQTOyw~)P6r;n%F%0zShC}=V8 z*;@=`WdHw2zWWdO&yxQI=>&ypOaqm*9P(Gn*B-*{>fSVryvB^A=Q`_0h4B@>eb2&lw$d{ zV%{zH_koVok^2Rwn2o@PJ$+K4bJC6lrz*71jo)%^dSmADLuZJ@Nvemwic9a^v`WbS z=RQqwS!~oGiAcbmsK_f#U)7fl1yMln436kKP_ig;cUzW`q^)PKL!(zx*tfAsP{ z&qw82HD$X;Z^ zKjb`Ib`6j}yz6=kk;z$TF`2GyZ>Z=x^sj0-o%c^0$=QRndLoaP@lFfndyNw2eHY;D zYvl2B#w`gpOA@p02C-iu?#`t&v6mN5K%wqQ8Rg-*!5Nwgy)6^FMDaUYq2FfsBbD5O zkjMS0yIM?}Nsi*o>z*r<#W>BshpH}ipNi~FlrHEx=@glIW@XfuKGTh{x3jRQKaKZR zjCX=A32QzDt5$XkTS?t~D2~9@z>**K6r+`*-Hy3~v2mSYswF2z=U~CVtLA4%7dA4b zINJ8B={m}(^=aC_2XEmhsBa%s31kK`SC@lMGMHdOr*S|9x!DXVDG5yr;qSn2gEHwW zV8C(Ol0itEF?W|>UqdZDF-jm~+nNyt^t`qU5lC2O^E3;%xCO zh-P9?TSS_cAj?lQ-S5*+%Ogn&Ug((5TGt$YbEoTof)GD%o2@L+^7vyS$eZ%X#JCu_ zs5SL1{2+P_vu5#fFQ7hfSom=j)P2uEfqfqScHcoqZwV-Pa!XupM&_N{l;q=CZEc!@ zSzzzuRW`2M7~T`S=3;ia<;v7FEPA)kD>qqH-8F0Vwux1sx8@*&ch!V(NK-mPTuAl1 zJk3DU`cp}&BW!51d{X(pUCP%@^3cT8{Fh7l5*A+hOtC8L^Wh2H)=SBYE%ps0O=ax^ zmW|K!hkHHj!0Gh{oJ25DU+*e}qfjHzWt{d=)8b17E)`a66l|6>nQk|?&S0%c{J|6k zg|_$8oCbwa&Gn1P{vwOcHVv4seS5Y}P4wKwvPR3;QWm*9wX!-tni>1|uAAyE?h3=2 zTAN^R=F>Yn(`td3_NY;mqUJXl*IAHd4ldC8)?ze{*FZYVzwxPUlPcv zpSJL9rZVq)-TC09thOaYR%1e8QwB8W0CJICmPqc+v;_$>)H*Bko0QucHVQ|JHXMfK;a~-tx_a{FQ9U6#pAqT(jIFgX&N7fn8dC1f6A9yIcEY%&i2@dO75R- zmjG66SqvSaKh|-ro6{$Ea(KeajpjsFNx52Z;2gE^6xlj*nFr{>qF7Tw2xmBW7q=>M zx@AJ1UNE~ZW*&gdkS*gc^F$$Xh?hQ#)zD$X9DG}~ow)L(x6B#eE3f^)F<1a;$GC2U z$U(B^DywF5D67^w>@b}1+lC>tc;W0IX=m7;O`Ml>FIH2xxq=g^$aUv7hLA&Th}zaY zaZz*T4`z4fSXW_>`Rc>ERXPTwQN!tpHjX~B0pqw%kG%gj=j zW9)EJcJ7}@OiHxqdqg8lh9u@@*FRKsg~o|r%Jrvqws%^eFXk>c{!*Uu(?-hUd=>#10KPub#X zK=vkIR7>k5Z8Fn=k%HlCt>m;QUpmOQK26_OMxruv1Z-!|^oy+T(z4Ti)t$!^ zV^UfOA;&3nEs;{#k0Ub?>{+}tYdt&kwya6I2#Viw6RU~#nA_tnZLnfzt2MeUVr!># zr(cf&1;p0LFR%YDvfj1RWNY1i7O4b|Cc2r8=dJ6jv?xt`@(yitM+EL3L2&=X{QS&F zrWy-Yrb6_I|@n%I+o_V;c%48(0zs%tJwfMgXPXApU{{FvX@Y63M{Bn@c3+bFRfGD zs6BH{pp_s%TAk695WdvieInz3B*B{%9UxiHx~nDS|HD+BfW4(x!!HWsPc#J>Q6(B6~3<{O!GN$ubaLPdWd|#pDHR)LAgG4qFd>2^QRdN=wQSx=?$@O z>1Y6J^|=qlboZM%g=9rxud(q=Eq9Pyt9Dh@K{BHd4IesHc)I~Z0jgD~8XGP$F@)I- z9#Y@#F?wPIbm4AuC}BNB`tUSm)OZZ-g^HcGOSMDXeIil!3MvQFiYpSKYLwzn>wM(B#ct-Sz+xB~eod%J}z6!yq z$f-h~Xz@!C$UO5fe5CHc%dGCMoQE5t*oE3aUiolMQkwhPbR)pxGYRZ&ut+5=oH$}; z(@!jt^gK@R9Y@aEdmgjdMhfwwXq-mUnOSNHOMbH9@Hbbaoja5-H{SnP3JBnz%n;$S z=lMWW;ODet5IZ}8^|~B9%57vqkJR|px$Vfe6E_IF8u4c_WHg3l>m!m!Htppw!iEZ?}iQ z8-G@BcKeSo7#*uK(&qKqtGJ6v&Pw7Q)uEh28Kk1V7`1$-@s92UB z_+8A|#JX8TtBH`WYX5<^ze?t4lB&hJD+|zm&kZk>d%eYnQGm47fMa5R@P=14utQ8V zBWgjfMcb$;O_(g);>X31yc=z-nm{J2*(XZ|!XDZ6@0nSWW0(^VhERv5J&N=L&w0bo z(>~|WP|A-VJCgIl(Vmg&C-x7aVX3h8>Ggk$TgI-N0OMGhyLAg;wIlikzMtc!F!!=y zgp@Kl>w$sxE_WhTE?rU84eL5Uu~6b=rwx4BY_7E}+zF!aqLr#W$b7R&sY8TYdbr_c zBVW2>I4ZCGTJpAqwNP>sb8ng7O0bc_aJbN0`|C_B@N)yZSqXiV?aw{yOhog3^fU!u z5U#|Dao1@>^~)u{ON^RSwNH|IKJlXDWBa|JMO4jM>oGd7mHcBvY!vU2P9=7y{?)zf z0HO53!#P!4N_6*m;Lq7$zS?#WL< zJtjl3OyJp`UVRc;-4q-BYQcc-?o%<*FF?1i#;JW{qKYhXVh({m+FwlriKkF7k4~fg zio!$&VskJb{Vi59RA@ZoHdWxusT`1TAAG)%!2#~d`%CTpu zyS$~ILUv125&|Sud+kyO`*hcIqn20u20Zb<85Xlc-fVO zPRZJ19&Q5XE7o8-*$d!%HyxBX2)4NM=X0lu(-;WnR(x`EjEnv$6yV zhb)w4*Dv;U#gP=3g%Mtv-}5c9v>BY1E#~~r0wio+ zF^xm44U!5-ANg|QZJ@&_rBb#8kjY7~#TeFgRFlH7rPak_*J5>(Ha9oo^`--CEY~ii z7zD9rwX&Fu^j(YTc#}?tdK&qn+9~7hqBIu|co-ywzpM@i`4B>k{|9?-9TnH}?TL~E zLI{%J?(Xgu+#Q-m6I>g2jo>uaxQ5`+Xd{gi+#$F_pm7ZvG#T!^w^s67>we$NyEAL% zy}RbWb55Oes!pA%z4vGDU7rdNyIxdu5!^B*5M}MWP0cd4&TS!Agka4}xQE$c**tZK zv%@veg4u}0Tegy=q!U;$1BkmK1q9jdU6id5dnl3AtqUldJLWAlU-uIZ>y~{?3qILh zZ<_{50G6?CUudrz==-+dCF!bWxw4rGr~VDty8lIeFTK&MJR#Z{^7)jNj`ky${4bm* z@lQIx_wtE!%U{6mf6ctGcPGxfIhFAU79bzJ5x$m3jX0Y!V^0p;3Q`zZW^`6;!~{=v zrHJslnyQt0p@yc}x%X_@&=@VCQHrB0jmEyr>TSrfU;D7qKf<_Ry=}!sqDZgf)ROni zYn-ivY)cJv|Dk5dQLG?58U`iL%L4N1k#Y?Uh6F9SZQ*AxEXhc3fiiRJOK!!oxv^;6 z2Q_SS#CmKycX&Cq_tVHH5N>Gx+EYvQJcXzCfF7i}nuSt* zt==*u#6Znv3xoQl(x~xAAFwyzwc>7Wc6p{ooi<}Lx~77C`M})9s9Udf-30KXtXgDl zim-)yzQ1wH{9*KAKq;+xeHBP;Hu(ntHgM=5DQ%W8Q#l*8C5ma8hA`e`pUOekE0hK9 zRPgEa?DW!-?JwW0XeSm1u&!_hex2~f*U8~9H#kg;%;M;Uqs>ZZDbVp-pC^Pgulb5l z@nW}D-am2?(4{6P4G86^m|>Jurcw&#wJs+UztaD7ENsQr#AxAUXwhyt;IsB@2}1iY zDn7Xb02V8_TZForND}1=Bc5&g*U3(1Z1eCC7KbDBP)vNi_(L~~T2U#ruOXi!JynjL zdSi*MxRtqwyU&o1v$R3Dm!3ja-C^ikP{EeOyt#Bjv4Q!-X(oM0CZz1fk+BX+ojYLP zP{G`;wbD0>yHZB5E7ETAW)yE_>H~w3yRhV^rEWa9=!+k6VQ0}K{$u#A1mAlsXnvf4 ziPqZ|U4mkwx&u!O8Ga1rX0~OH1#!Ri2MPT|&mxuK{L9vB3h&dY8SlR;6lbyJ?1-6u zKWCxOz0SaD);MUn{62l8&+VnCUfzU#RyOHq7}Tsa$K?OQ=sh3p6m6O8WQQ1N^Na9U za?)~in`q{eIBoLfMF8 z+h@o2RIEwM_G$;X4nc8Pg~_819?j(GNAdgdsD#zT&POg{narZZ3?^0BV8Z3G0{r9t z%P|gmLlMqKFwUB@ql}BagC!lWW1}3F8T;AUppbFv(ezD6P5mllAYv=Sx6q$9ifM6= zE@9MyWJeg=i|m<6tXO|Y4)fL|`5YCawif?3v?R|0yt&($qc_&;p5K{o-?=kY<6;RG0+mta*h8SZ zSE{V?*on4kr6(#`=e3D)IY2oxhAg24?H~$cyL5(U8w`hVBo61;~wZl0j~JuoXJ3VX8SQu$HUV_XSR7ZeBS%->4Z0n}!5dqE!R}_Ba@gkS9j1 zY@f$l2zonBb#bZjOf2i;!WU<@;1=~InmsmfiFIwXBef>khoM%Uk%u&df9COpx5;hF z6oz$9>~0FY{%s+NRK!)k`ww>VsRz8i)6A7pfdA~B21YlxgFeWsf~CO;xJ(s<0JL7w zErc;sECosC{YC<3-hkJR8s_^N$Zj^pq^3y6OU#C76M}JgzI2BO_z617aKo;d&J4Xy zaymh69p1m@%y*?lHxK_1TloKJygV-q$p_U;a)Oey zO|2HCgZldNBrW+jl5rZ#^M5Gd_I`PzU(oP_F+t?j`-2Zzrlg6IH#CF~YsTbpeUWE` zlv@TJRhZj@+x#qa%6j|~{)q?#YP zpI%18d6sJZE7T59BT87G{bdpN!^)%@W1r-D;7F&-Huq1CpOQd7#we3zBU*&{RXe5f zN|Qj&a^6Qd)fsF;&1iM_0!DcZ@InZe51b9W5fmBWuMn*GR;zRLfflhZCHUm0)g`Vz zxJYvi630jfPsGtYg76zoLiVG}@{cA*4xY2&6m5}h*0S&yeL5XnEO>5V1~!hkts9oFFnuwg{s~!wiY-a?kSuZl z%J|Y~a_Ghb!N)xHtM@Tq!{%Ec({x|5m$FI)n7_F-T)AE=+{{l@B4vZRWRK;0pEq)=j%ld4XiK5jIz z>uKQEj=1=)$^HFlZZa0O3brx@%-SLQp&^@0@3E57lI)Iv_46IhRX&%QgCY!*CciDu zgG#j(qCWm8uD#rqXp31f`!!L>>Bos4hG%L|vcMqA5u^A5RaRnEr{ZFq_Tm7Do3Cj9 z@qGErWC0ps?VGRCtq`DxYE_SA+i<&JQ8?{-CL4}&jSMyXT8r^QROLr z7puZ4qVk2o(57uP)4v8` zz4-hqsCG-1bZdA`D&zAI0|5q8N5pcy{W?wvJVC~D)+zeJ0A_R7p{+}rzc}~J(V4XO z;w@HJ^`RRpVHI!+6VDMp*O9m0GQkMkG%)R+kvXb2s^jw;>23W|;G#@fjBqEHDy=d% znLhT{tt$DLz*m!S4Qp?;XAq%@d0xXxK>cw0rfP!ssEU+iT3X^Zp+*JabrN#utMy|| z8aZ$nB#E5b(3KC3gHAY+je14W6JJekW788t z;9fH+cdR^^BSY8jz#PmJJg#8F0OijvY)dnN7^>xEG);{%Otg9%lYk<>XUOC)QWp@- zqJq~OPNQ*VdbrAsFuTjY^)M|&jzZd+Qy8jv@hQn-?ejjL*Hi}ew*_rv^!1&XWyH;~J>coGHpMhO!>)7NHt?^_#b&(`k4RP^X|}%$4=?Y0F+X%x@R^JTbe2z-;0N zIWvad4q&w0)Cc4@}6R@|@ePQg7?he@{T!#LD1{0l6tJywClsA0(dEuWYy*=EQll ztUUW4twws7^3q>_8<(>6n7g9D?xP_G!eT(5Ox>o;G7++%hID{ZL?=&Y6uGM@Ojg`H zp&gq%Jw+0`#A8(6Cn$gpC4&^v3+?aibT2|}NBM>;j1EDg8GU47+SM7VA^g;&eD9Sv zq!;`#YkzVSmab|^wAn^3GS^o&U@XfUDeC0S1k)QPYB2|^iK0(Cl%u%2-M*>lYuJOw zCPwn}kKv_#E3=4ttL!A8Z@Q_6n>}v3my>qDG#O0?$M*aw-fO4KFKahYG%iLlluAxg z+D0xv^T_|CrsqIVI3m|rS>X9O%4<8qHk2iWdd6B93?s$A^rj%R&dfHiXXaXPKnQT% zxvzGY^Ws)WpfZrngg2B+NG>O8`_1PE<2wn??^LQfz%S6>RPPHw_w|+d!bGmh_KU~} zwooNj#3USP{4%6$(A#XVyFmTmsoH$Eb#}wY{3}JzU2l5AZNI#t5tjr`X(esl)?|`$ zC5NN5>DsIhP@#3!vE$ipOy&W56$OrK@T%*iB`2Rzpn^TqXetQDIAE>O?rKuo*t&BY z_S{X&+R{C$C!;9d`;|Gd)UQlC;TrN~R^jr#sZ|@f(%zzd{Pd{Q3I-0rQ>9$jPU}}4 zXc?ter?f;eB4LW%R_x`6{*v-{ZF2!Em-Jml^GlVw(I(gr6*T~JOmZNYVDPDAfo zb-xtTII?NENXz_w(ZZ;L71xgRX;bHpWU|4q(bSH?&&vhwCCPHM!GwhebPyemyi#0s zon-Bo0}*{GdRfZMysjRp+^oE}=2IUAPE1^0XEDVqHv-TG*yUZ); zje5>dXj!WbI&gozE%Nq05i4vqPUYFDQ@{Up1I?^9-2OPBeWo1XsNDN=50jO9cTzWF ziR+L#X`#?Z%~A4ob|bsE?%)kIpL{v$^Ebs}S6AO4=O4K>B8=H*3$=_KW4T~Fr-5LE zJXc7_C-yXVP9_U<9sQh2Vguhszu~brb!$!n@NSMGb+>gt)fnMHR6=(3SD_G%LZPHz zwItov#FM7>$$kpHb>{|hAVYwhER<;E2zEwL7n7Wte36w{!uW-3#f3#xnPKUX$iQHE~JysJ5N49uDC z@F%fap=hb)2VCvkGom}Dc>x6UHM~I;49a%s_?{dNR5tVk)(S%9N@aGaE?3k^lzR<{ znb<2`wHYHI@of*oEJLR+25F^!ljewEybM$_f_Ha6(#ulb;&6xXmRN$hr{*8Yb|JX3 z_Fr8rU%HPwgU~pIXz{K?yH@P=@EZmY=(L=K^J9D#FBBDd7Q%VX!no@BefQUsQ!~TM zTQ||wt`H6ZJo}bMz+YRm*|KHKq}Oen26T{gPkhA;x2&>wfUG8&x}5`rtz>DBwP z?b&V2iIS5Fe=v&jc9)x}6+eHq6jNT?+q|*cEVW|I@3c<&5dA9FSa>X1Y;ONtkzhHI z(foPyY-WwB;iB0Zp#%1jnLsr_v+BLKRMPB-XoJX{oq{*5`34c?Z=@JASB+5dDSGY5 zq}8LFD4nou9@Y#!Rr|j9)~wj$K)PyN3IPqJzi_X;v9VS zt_!-R?)xPOfNJx{v^7|_SnN;HBB2&X6Nfjok@fr!pFw5=w}2!o!N@aa(%S-Em2m^{ zo*!Pj3o=|~4S|>yAp5Wlmi)-E@{|-UwUJhkyvkp?D@d^5Hhp%R5j^gWHr-H9RdAc8 z%k&uqc!l0#SQ;`>8ewz{APgH1pEx>{M%VW@Ew75nuIa9;l)ZnZ+Co`>0JcMkzNWMo zYd$bL$^ZB!<{D=bM7eT2@%6`|P{pr}?~QUXLmICrkVC~MLF|-C zyxsb@gMfl*<+oGnU6&*W*DZgPKdGH*LH)Q59+W;%%R$0tDc_Lz*=MZxd&ZAR2_BM9 zq4tErX(}0z*;;Ys%bpq3LHX!-ZtY}Jp ziLVY-zI3^Ar~gA!MNZ)WBh;E+7lO_;j9g(^(^gUS6e>RmC-pUmADvW3?fMibP2HbP zlEkPRcITf@(_(rip`YMEW?Lk-?9h{r78R1A2Jvf1pM9N)JoJTukSZ21Xryab`%$|zVUr}TyA^`H|M!JSpTMHvr3iXD#k;%Y|;@**q^UE}h_FN9SqXn$X1+`)eC zJA&zKp>`CnPA0JIviYm0XvLzBX{uHaze=U83HZCFb~hp)O3nZ`HI}aw4l(~Gifw?% zt%n{1JuV^Buh(+$8+Lf0i*Xhw^86@Oj*`!(S_Sc^Z7lJyCdo-E!q+v|o3o}!XBc`* z?HEvE=e$@N%(`Z?P5RMlwcrj?kqedZoa;Ts=>&+*g=1?BAT2KP`4H{*JW=u3v1Fa$ z&uVXAb_cHq;e_RKux93Ab%&#?O!7^rr=A9{831L`D{5>zipKJqJqVmH7G^28Usg@npm286Zm zNIK~^Sxy(MiQKo)0PgtuA;~I)u9Ar1V->JMbVF~de`I6cZoPDyt#b}fir(=G5~ zuuRFVEi$eCplkM>?4CQMK?HlzNy$u@>({w`9<+a!K!4Q|(E^+BIihdf2*?#VEG<(I z*mZrKKxnjEO<${j#5=u>ymOVhnBtyB;YJ(Jw%LT)L@@{=5KVwhdekoS(5Se!be(a8 zG*7D;DL5y2vbHF@SoV@Il&}ab?W*-lI@2J)fVB}%x*Y@PtA{jg&zG6D1=Ns1W$#Qe zLS3FvdQ05kkXa8Km#0hVdsf)La!k(YQ|%1Zl~_4i**TO$g(RjV^bl5RIkO@OS@Ml& z0zInoJ+|HN={L76xOqYVL+17H;l@dA{^(Rzx}r%BXO?Z=cof=uO>>(&PXdb?p&`5BHf;e(^^{!xQoIaU zsR&p|^yAv=-Eru^g%buUCS3YL*X70|d5(R)g#nZQ#aEQ@zoe|_A1Ouo(Ak1#j^3uJ2AwzdKV|z;ZZ=@WidHZnqm5mwh+Ij> zo?5*m>9%i}y@h))8XXmamABo>S3 z_WCV~Ey-K$lj=QjD^<98W%>OL&h%%(GSx^OC%=)C6uY&2rg*;H;Fn63Q<0Xet;p8R zhEoseyJ~1fs(u(W;2p3`9kt@zLUjYEo)OcZ8>rmKY`6b~l~4x)4DV>g-W*g-m}dKj zR3WiR{<&3ZIXO`*OetVi`oo_8~9ZxxKU2dnu3e{Zf)`Nz}lpY)dF z)R4FQNgYY#e^vT7`1?QA{Acbo;q9G=cNW`(`o~>gGyMOWX?0;-n>_S);rQ#NXhIbm z%1Q)aQqbpG`;ClS*8)6gK7Er4zUiU7p#0$`)Q$s1JQ{ZwE4dr_wM>*A#GoiemA?SZ}EqLX+4P!{kGWb zc#pa}sbk}5^(Gdb(!yJQ>lvs>^^`3upyD~dvcVxR#~3p~2AP}tOoc7h(Y0?*{Mf;; zEV=w2B?kZB5<^1fUp)+@2Ft1Ur%FLFd=Gmvcy@V16rvsd<ecMpA)}34W{KUzJrMr1(bhc+gdbE_z!sNkw zp0X6vV>j5!sq7+J#Z#sNlW^i*^q1=jhPPtfGvy!umU+Ivrph#ZG@^mqW&iKxp2GoL z`rZiob_?J`!#Cn@ohEDGl$#iftED%=aK;B>>~-f($@3V>TBTQ&e=S80hz>&gjg)2k zkf9jpYS9@@*7mGNqAIg1vDG_kH5}4*7cl2hor8p=^ya^{nBX7E^}%`fGmN)(eHt48 zdIGY4IVGeVQ`mtLFB2ij0i{`(j;>HZx?P3M6iasO@l;5mkuAO2?A{Bs?{q^n(T zkG&ObQ~lF-W?8}YrcB9Au=2xNs@xnNjgVWSv-P&kNge!S%Dts|oWu8gv>(?;qAMM& zSN^||KE%u%r7npEiq6?NtKBV*oJg#-VS43|3q?wJ49%vf@Grhl+fCUFLZqah9=t^7 zS~PSUn;MYUgaV9-{sP_X za7-`b5B7|Z@vJ0!OD8|r$e_c7cJw~IO5ZFgM9|n#HiCIT(N#+yFX5AO=_wSabh+^R zjzpi;Xhq$H;02s#fbB64UzmNQ!R?WMG;XuC;~O1f=KE7A?D2;WzmbkV{YFZBcF?CR z33(i^ZxZA5KmAAn*xN-7y6oTW#r6l6SbP*ETYR=Pma(Yi%vm94g>y*gMo-?<0P(Ir zZ*U^7_2j8-aP-pqrhOC`+WfU#sK4oF6of*+V(H=gpg*+bEtiM-4-cs8WxUdzR^PT6 z6+3&$338Jn#Ekv6$YUaVH%svg&I-IPiN&~$;7H5&-R@Jy{#vZQP1Pl#HC1ESs*&D| z-u#(A3z=l0Ll3*1;rY?mZPVNj=392t8FGcJcF15NO%ctQ#Hc8F%=PM)n$XwXHDYRg z(_HHt?R@46{=wiL%j~AFMKB!)N5{(P;#z^6k+l{{J0@q2wKn)^``ps8lHKI}+qxt? zT`Q)vh8vm`W6fn%8mVeXWC+}|BHN3U z0DJCsTYWn(TN$H2o+G?>SeLzGYox-8Zj5t z2WQuINq{xa6`swNvpi&4%jL7~9Q<{KcEf#vF~V1~S0ztX zo{+dsV4c;FxF1yQ7J`gHJs z=}d~0>OyLUZ1uLl-KARF8&OwP%WMOfJ_oaoRNgexYcctjEB+??#E_We**4CUHaYy4 zW1I+c_rs;8wv1PR;Xs@>ImJZs?5e>QXtNB0mVL_kBWRE@CAl&zAzv>wFGX}lH&&`; zrNe-*1RFMgSzsEWBZIWn(h*VVOYvt>uE)u+ov3x9J7h|=%l^7cF4>7l;S8hxg-ZYl zSf}t1BiUP#s1iNKN`~mb$;m~u=8WGVK|7XMCsB9?V=XJq*#H1x&{6C?S^Qeu?3}cN zC@%|!Yqfu=XcU|{ zs*e0qsy;IuhvHry%Bjwq&-%g@CT@f4!0#<(BfSqIe)55*JFa&yn#i<@h{sFzqjB0# zzR?lGx33cu3ijf8F$;brucqtBE>5Q`j0-PXrNGF+lQG+KUC?7#JE~dKCs$BWdWtik zJIFY0h1VI>1<44^BuJLsc@K+t`%>(i3W!1Mn)GW<#Cd= z@+YnK-cOJ`7n-I_&!>SOEaoZ=8Vz6Hol|m*kQTirC8U^$rvTd}x9=}+fX~-&-K4!N zC-$CP4EhRg=ZofO;GIpJNw8E;Td?AGTVF0<$u*&~Xz>nUfm=pj{G?vGCR_Kic$QDK z#fbdeDmqbd{*slSx`-BF9Vw`2RU0ujqA#9&yi*tPU5dRa;}k^TPPDd~VU9&4pO_<6}gM z3Q#XT6;LcmZl&|km>9SF8)=~>?p$^cvd8K=9kp@XutCWDQ81M|dl$xL%<^TS_@5j627ays44`SWE0j3XChK)unUj%#7A~i>H;%^s@^AdO zivV{}_bA@+@V!B_g+zq{)G7MSwG8mFub9rT!oYP|w-&fu12c2ZQkY5HjW=%ejI~;BT?Lqx<|&Q2lKb76b_f{7$JX#EqNW`Q?2tSe~#P3(#2Y3kasGX%o4)aXQM`Lnrj&$0O zvMnb=44nbITKP%V%QzVns++xuOwM4iZ-4p0d45`t%b@kLmGr&-2lLZDgU|M#ZB&g1 z`pG2e+VUcT$##004Dq6J;#Kstpb5^ZS#dA=%dn01od&l2$S^zu42<1K;#GzTrm`a^ zm37=kd}evR3T+0hoXWiu&}DzwT)@mcllHQSm`5q*`Ubq6x(zQ9rfseVZ+H@82Jyut zE&(rzzfLI0--%Cg4Of1IW;|{NO`lARuluR=uzksyOEdi{_%Dfyp)r)6KWV66I-jLv z-o&%jSOA3>wcD%3sQ|U(6V;1N+b=*1bphFpMLg=|3#_SlrMVOKcrPeuP3}mR>y^6X zhYn0rhfVV$(f@iWtFd6SV0bOW{l*N+&`4a0ofxF)C6>&CMVxi2Boqxm_arwTue9u5 zkR<6r$;Q0*^T(OUjvr<>f+~GuEmdn>O;xSMU%`agH{L~iK0_zoJ)InGmy0Jcwj03O z{ZO>#fpAiXQ+iqVCF1mC89w(@K<8R#e#$d|CLk#if7l;EqwH8$xSXrtT91K<=FwSq zuue8~_yio1iU2tHZS!4{yiI&N`P?TlYS9Jo)Nnyz^8)J!C z$ZV}HM%|nzukvBzJ4nRZ>ke&XQNg_zvSfcJECSSOT#Z22@YV;GZq@j9@`^{7^nUBs zpy%v4F7>mLb?rKB=FI_sIBOHl?)_I3I>mAnf`sy##(f15z+a_it1aApIt2XrNzv#v zL=Z@8N>Xvc#D%uf6lHxrvmtfD+gA*QmkkKaNM7l&os7q9;&(B+}M+T=DsEtF}Ukjm$XY8d?vT|wp)XV$X zC&|Q~iH&rV*I0N#c_bs?{NBRm*0=d0O{D-i%f{0WcM(pr(O00@Lq?JHQeMIzsMxEP zNlAzv`DcZ~U!i!%?lOX)DgiPk!l@MvC6JlOg!V9o$G|l@p^#IiG`74L&8CW3EtirK zDo-#)P2e~s&U={NbE}wa9S=AOPm)6t)#CS=$VFZkD)~ufG|!L233+`{;3G3XQfv9* zSk_Hl9^a{Vc(sfauUE8aAIaWb1hRQW14Fh!v`rg>p!&e9Y&NE4XIYfu7HGNd*?jBw z^qk&l^IfxINa}{SZW~XY(7MkG@i?nM>cU{1R?J5qZ_*8SC{EgfrWSb3)e>0?vgc-O zIJ9gaYIzGgV+3J0H$mmCI5$=BaqwnV*hy|y|2f27J`@SnQ=PnwXN~|%=s(~zV zVhPN2dnFJ-+8xDx`I%Iw>m9EeoQID4e!WzzECq?Y_G3~6CvsTe#EM9uOFLnjj_IE# zSS^HFx#~?9V4L-=f_hH&|hex=e_aGcKERzy?C4TBtGF*%z$1o8^Wl-soJWU2(p|xlWvC!+4A%c4?VTTR3y@IFQPA{G61!yNTb2Vfs& zviCABMK-(`tl3+-+GL79@=_KjwRZZZCIGN1Daou{NoF}Hs6jp}iX-o+%rIn8tv}a4 zfSZpF?LF=L=RZL!_jVi?ccP%?ZVLUc{wA7W%okqc<`ZH#bZi#zuww#YVO7yroRDAb zg~KcF7mC0qpz}8HNzmH7*U?FE{gp}Cjydx8e?s$5CJz4f=JVAe<}7|E>Son)arV6k40_DpSAG@iki4P-p$fwf$<3C5?9ihw=WHFUQ&!t;mH zqj$);T)JEM$Y27Ra){4{d7d@HV`NfWo!1BQ(eDFO&?mZaY$3xI`Ilr&<}P4Y+LNS{=5biBYJdhOBaB807qZ87Irl-vSh!$SWq_ zdzof1sFdk`VbgBo7kkm1qQkyCW0?~9;%eeG$$-*e(zJShDl06qJ~l(pEty`XU@$*P z_1EcT`EC@IqCC4RRl3Oc7;~bT_sy<&XVgg+4s6@zte&bvN}q-z`<;v4iev}vq4(ns zkTe5Y^wQ`6mwNMX_M%<4+vbFGw4$l%>PQ8Iu-|^um_8{i3Av{*s~34@*g=QcccK@WFdiHX#L zl6^_AEB;YD@!RMT@Ou~amW8WG=5>1sjS6-3!CB_F(V5sua&cQ|X4VRnAZ0sZPWr|X z+LTEA?9*tP;k}>sl3QG#cSx3wz9ii^ z#wA`+ymT**%8noHyI+Vq3V&{;z7Hx1{6K)Hi`EwQNkG$~8q2n9HP(*DLH^wQj6m03 zFwRwcE$&83Tong7=&NOoHjKM5q6c)E8q0I_wkD@mnreq61F_{$vfSV#O7W(>b^W=$S&Sd9uLg#Ee3C^O)p_t=Mp|Nh# zK{N&$pR%@Kft@l-PuxIS@c=8~(7iE0#V*y^8I;*nDgO03pLmliPZyh)J*^7U`nd)X z;9cYy1(GY1S$&9g8OBg{2-HU^o@4~W_MDo?+UiB%{h&|lvdX%=zBePeA#9t0)rM|q&yM)>tHIYrenci8@r+qG!^#IGwh82wUJ^z6O1L@_ z^r50SJKhe~HAs~v<=8R`WHh|{$U3N@=d<|6I1_1r4m8uwFtMr=lKk|lRZcaYH;mj; zVtz92C6DkR^KlS2Pw$N|zNf|Fy+=PdJ>m9-jbvbhTc(qic`a6u8d{y~bbTdROWnR7 zwslAReLs2%=E=xr2Drlk^rZz+X7ig_=r-mj3{IizQ?KaAT0QHXz`01o*?=aDC!Q4y zJ`p$X12gtshV1Z`BGB?`E%kMZlUi)uO^o%lbPRbulg@xx z(2#$wqiT)Wn0&Uuiw>H0TE?uP_bS3}LKm9jioIc^F%9<9^fK+z@ofEZ&DQFHb4?}R zvyv>8+?lLX?&g@Lmrg(OaWf#ggPFKC#KUyoHB=hp_Kb~^{J3&99uQRdxHYmm;9=kb zvo@^BfEimi%vsV4SX}nddmrRtJbN4r+I7ft1ASovg3|_UZDSZdD{qEF;B?1%%bxPD z38Z%BN)UfRnuAb&!SdH6?poenZ5 z8GSRBmD9suYni|q%@pc{}Q)kE4oYC zY7HKE0g;T{p3jjCv!)ICvXI^{-W!c=d4(HW>RSRv~t-sxk8Nir+H>Y zkJr}xjU9osduh`$sE@t_{~1NizK*%F+v*!jWH{w8u44^lH`iWw^&t=915J+)4B2(; z72t_>IcoV@!q}AdlVT?YU1?E+>$KTaY}Zk#pbT`@1Ig8!U&#*OdIW<^C4dK*dO=Z#~#`j4o(`ehvaS9cgGlh z);>sYmz%DGSXp)8!)z-MwB^NyJX!trQ#zqzR_rv{>ULZ&I}@u(%j|2=oAV9OI*3Qo zcDUYVY@qUPOK7Fgij%9np6-lO*S%rjdlSFDqte;##GGRZ(j2Pan%|Qnk%kR=QtG)h zd?N&MBC+0^WgY2r_;#Fr2=ePg=~GPy6Mab{;*l$A$ckf7Ju>iXEn#HST+kKe5XIq< zYF|?9w`8^>-h0ixFgO}UBo_d)6P#RGa~j6BX!}XiTOrIL>7i8p8wuHGDK@i; zVPol-1M!67E;6)_RgPbA_(Nj1^^FB<1qCi+kxK$u+a(|9R(WRlV*feOa$*t!)_6GK zZ)F*D0nSqmUS9gRDeKk1-8j$LgnP&2CZacpq2U2r-Cq!;5kz3Ne)r|hY{H6YB3HgQ zr>!IRMp*gC9-LBadUIw!?nT~wvdIdNN;7`xvRwb_I8qw?d|0?BYci2`yK;YIC@#J! zKKBP?cJ?j19sAeMtb!1aJ`2SfJi}~Ug%&v|bD^bZ{DRGy55ZH*6|$~t4GJP*!}S5% z!0tFmKqp}r*uXB6U5;hqAbEy<*fNhd>P1}lcu0!!a1bl6Oz$06hJtGeQw>6>2)!Yaywcg<4(ox(lv?Q}Y9f=f`~NW_7Ya`WAY9Sq5gU+m4M|BUy^3&XomZ z3?*P#ip!*1o@-uzM0&g71GL`Z!|?mb%rdl$%pqb_%7wwwHO|vd{MnOxVnMD;A9V`F z)OrSv%C#Mod;R?khVa-g3w}I0*7hv3swMT@NhKF|q_Z{K136HKU z#f%hMY1fOkcx%Q1f0wVX*6Q`dEQ1fiB-5yMUSQVF$64#?+t)4g$dy}Boitv0)EkWz zqwVlmDjMJLCtDGc*cWc)r;H|XdTlA+X6KHjl-5~C&(@c%!sa`55|i+M0Hc(ZaHH&_ zd@>A#`T2$F5oIr5qMF^J2c+jFD|xGY3B45zS}t_;te80XEDu{N3T?V~V+o_?mc z+Vx^g@z1NdZ0ifL?MV)pc}<718vS{%7@Zh6u9FR=Fz83Xx7_Z(9kWrWok+WEU*@-vt++5Z?n4c zm1f$5D^rTpWLXfSD4vsYAHDw}Z>;E5le?Rpv!^s~KSavjw_rv^Q&H@E0aMVyI-%#0 zk9Mv@6b+;wnKXXMM>&9dH5GF8g%ixE8&>-*_(9@_wOsbyK6*QLyhD6E+Gp3y1y&5V zl`Z65>gzSz=jANPXUr@(oY^9?3`f!=sw+8lmJEsWnh8j&(PdlTvBUi)o_f=y4a&np zDwfsCJ{CeXC44Tu*zBBU6tFV+D@fLO{hBv9=+{@35bEaqZApK7mEXOXiuCb@_)Mk6 zI9s9O{5YXIE#@X!9l`-3(W?J>giNKoHL)+N$HAXT=^avxP-?g@V~lXb?3Jb~C)Sw; zi92@1;g?4R%eP@lzL~?fO>49u)zd0EC}uYz4~K=MAidoDkGK5*SCIM?w8xVtk5c2R zM-I78V<7X4gt1!-eXI*BTn_I)!0Fw~A-4NcATe`>+5FvXxql>r#cGYElY2yQqXVs} z%xYjvd;En@PrCQ~9~3lWcKGVET1kO?C4m5!vd;~c<*%S^2ZHeX*o#sHhR01)X2vWL zX%(W~j5VX*2gAQySO_1VIy75{_>$7=nv4Qt*dCT$*xgKZ9p&DesD>DSaD5;T*8|RK z8^zN%Jxyz%2+y0v6EOM7Br#k=>DIM~gR@+{9Zdt+L8Sr0)DxzsH+_X;V?pGxxogfP z>;veIvkn=bp*yie#c+a^$h5Recld_T!xXdi<05&JIQo|Gc`%x~M<|`=!Z&D&mh)pj zpfGH->j>ygarHC+*w|f>_6#hGo0m>XgaN<`pQ$E)NaHl5(+6%|PSwKK>HeH=l>OOJ zslBtqd`@E8QU>SI_*Sc9GTjatDxG-VK=7+!6{i-U`44qx4XjcD66?BzGtBB}$Ryiu zq-2QGo8qQ~q)poLvUh^`H4hediZ#D%Kc1NwILRd#wGD~|T0MptRjNeZ!H}s1CA~PG zNYD{4dq zyFx}|koQ$at;#UHg>@z~_`utWSOFOr4qo8)8A2e)77`Bn?g_Rz$;5N?DXKe2h@U!X zT3E#r^A!Ye`gJ#9t!zwv1ZJGQL!iUF(tQpQ?^^kLg&XC~QmN^KD44(UZoM>SNH+{n zO{4YDzlF69ENj z*c%8*)LCKxtXWUHh?)W#-1cdDJ@_o!_eI97&So$SMg9JU`FtPqfHsNxt$5;A+&3*A{7QTDVch%3I1Qd%fWndw(3QdhLJq({@be1V8+b3*NE*WnNtRp{ODj~wOOU109Fmhngr&H`0e zUO#Q(*-$K)f*4iX^TesntHmcOF&eZFgI4R_WDgYaXqA6q^~SZ&Ew!J1l_PEvO=489 z)m=2|AP*l|t!zU7Bd7n$jlco^qs(P}?$i}qp44^U(l5aBDUkk1(gOMJC2ul$SdwDB zj)``G_MtgEiL5_iGBtS+J_xN$yF{Jsj*>+Qm2f(o zN}v5{H|BHWW#^=Y`N-x^Q4^VruUX^Hc_bkTR3-2BGh;%L`(T-O0=)a0AoW=JmY^qI zG;qro-o(};v@%w#wQ0~;UJWwwcd`ZKYy-cTBI&vaK6$h}FEXCCM?C9AC$9b|J0KL; zek~G^qp6fa_W~{Idk~{MOKe1Y7+Xx!c>+rDK;Zt>J1Rpz$xO9F=AoymjrUPvQamzO zTYF5AR073+)YYdyEq+P3%vK%jA=*5#Jko!I1fl!Tw`?TdTG>9fB}NwKjnk5p#I?$( zq)HRNrr3Y!{j@bYsZ4zi54LM=l3C+S}SzH1O6pUsip7&ICll zV0~kBVO-<4g%6zg3tYYVYX`+Bef6QE_DFQ&Ho1v{^5Co*z(68Nwe`V|NGSLURR42pLUkI^=N z+!E-~8({Q9;~t@KI4N@wO@RU2+NGer3|OV4S^F5gLn3Jnp)>~mFz@GlbNULp{iEQ7 zd`IxhW>wG{{|X!9dn6pqzi{LKn?rmAQwP7&uMF43>*Pqe@P9q+KTo(h`i*2Mbc}Mw zDRtMM+8%uEv)w<5pbK;p>o0k8_H+C<(krt0TVu<(N` zd_s~uP?I+l{-@wyI2FENcT45$kB*G5TL|YItzv++R>@fH;mb>sfr49sm#qT(t`A%7 z@Fvh@n?K-YwYMeMarMd{{rF#!GZnD1&<1~^MXMc?`7)xHH(7@HpJoz^Rn-WJ?HJ&2 zTZ^y75YXPVMx15Or+we}L=qTHqh>Bj#-dJ4%uk)R>Y51frIcc{-P)L)ied^*nE#a5 zfZCF;uhD9oy5yU63;FWmoIc*vU0uZ@>05n#!P$QAA-7TlF-Eyt#2J=2MKb z%Y4F+(@yduD`E0WDqiASD2<#IzH$o3$h+pnm>yaGFf_xwGb7uN)b@@u-qOtkEpI`S z8h>SLBE5_zN*=#4<&U#sI0z;|6WlBkRr?j9=D>9B0vnXIiF)cE$px!%6G&15>QpPq z_vT+Il~A(U&Jfyic+6zjcDMmMiz~8&Y_X@$4upwTPDhx~@uX^D*8+fQCCo-bKDlv7E z+`OcbXoiuv)-EXh+KFF54fu-GfE~C^5CuF+fs&_;l|cuYdMe6V@UT_{f^PEnqhtAd zWp=V+=X4^K^yW`J*jQUkqR~0{>-lxw2N1g^Iu1jOYF(A3i$3Du*r|}rr)-(SoeV3P z!Ep`-OIxNX&ZWeYH%rr+p7QJYDTCj%M4~T><7ss3LVHDx`e1L>a0xr!_1u?ch|?GP zWk>d;Jfg8;2cKD$y*%@uhkqUZSDf}gv!qhBaj?N^kbR-CkXlCy4UMlb#!D-U~R|XA;SGnHaX*Wy+Sj)fvEMF{E)E6UsSBSM6ltVVgBOlKC({ zqGT*mky`R?YEk+f(14q&*ELA4!v1p{WTcbQ$X?PeSe82}=O~d_aHA<6`lSS7O_Fq2 z$9DH8Rz>1CN5Vrf8BIWC+HdzkD={{`quDA;dGgSPM;45{w;?{lP`SsEa6t`@Eiev^ zbHn_QxaDM%i1q?eqz1CkweItjpUk*>OM8?eA5E^!M9P~n_q{y8zPQD5PIc}b4DlYS zR<>v%;s!BIq>Rm18u~Bny#-X9U6(Eh2}yuJ5wgf;$ur!QBci z+_fOVt$^T8fVs;$Yu@sApcQ}zo{Juq(MlxtSE#n4y!C0x|CC2U!{Lno5ZnhgP^_H&; z9^(U?&C2>|q-~wa1u%;tG^e!b^w`7Cp%!Z4+6=r9B{rj@q0euoR_msi-dJxUJz-I# zmfvps^po{5H#_SkEa`NF*@X@>Ze30?r1pY{0!qnKETCC%a?!P;u`Dxy78ddfjZEsj zS@5BNt0E+~r9D1O2H)Ltl%6{(&7mHjzo6Dj2R2KZIOHrSu3)U zRklL|Rnc-z-Mue5YP{3i##IpT;hAM%tu3`f{R))!ZRVBM=s``OGyV*k!qJXrlDG%J z05CGdym^fE%C|MuCQ%nO^;Pn9BktzhXeZsg$GVj0Ow<^(NbdP567 zp$h%v&A_Y)25E&7SE^~p>e2{xp6+A&INiutUYwqUiK!s+DI&DWaNo(E)XHr)UKz`w zQN?^8ZGc0s{s!0#3Dq6Eg_w7`c-t>l+7L_Vk&(*3xz)>MdT8AYCe7cJ=UQVCXVm}uqkhk4zGHs`ccLg+< zNodpWwPkOU$I>uUVmV?-=>hai%e2==fUbn*w()98jNMGPpXE>?ytHuU5S98Rd#npTb5t z)~8|$LA2ac6;lh%tG2B~4C@PzX;xjyIQ1i#`rMq;%-XHrmFh8e9BLQ|B`ncXhtDhM zE19(-7p8HTHP>2ZzkS$KP4vuw?bGwiAAC$3fB;2^3B=+x>u^ybD<@l_xG)LDFdH?0 zdY>sSv^_dfOKyg3$DxePl*b~X-h6;#6yrc$$F*%MczxnLY%6IqM7}zJLg9gne=rp8 zSN(rLl13&#s%qGJ!HOI(^q#RnL{n4OkVHnEz0cDJN9v;=4{D38+P3vH0IPtiwvv7} zt#i<*tLnZPBnHbi-89`oKrcW;dRc*SFlvr_Il`?e+O#7b z!0gwD7_TsNOPCSgTRSj6)Ku;#GKBn+WW0n0;4Zkrtj@-zV%rn-Y-757*3C4F{S7El zA6hgTQ-o+Sb;KWnG%H<;y|fp@sVYSuDL4G{%Q1-^?c;DJ7Ov0FJ?1H3S1>zRa#zRp zi=AV9hnRz32uII;Ar#EL$rN=s-uAz#J|0@gIoHBQn5DN5d^32)cqWe);J7)pLCQ^Y zI9$EgM5BX`9z$ggl^#8h0*ydVB3KK%;}~LSfQUCWQ~TCEaRsq2WWh?RvmD0BL7E>U ziFbc`d}xv@bCaCUh+&tIgJj_2QE_|&8}8{Kk6u21BbsHC`i zVTGgRQCfuW@k&!wQsljYRdMrjQQQ(eJVWn6Ek!>yk0IKY-EfXhi$-gFYB0K7{feCn z$#;NrCmxUJ@En@bqWxHri1aEUW#`lZn2ldk-R|{M71AQ^_Z3~6(;m;Iy%AO(Y zN3u3{0p(jzsO&=>_r>_*&~({TDejYfqsE-MFnEf>vYL}?VDGWPmoZHei@3&z0d`w6 zOjh#Tq_xcG;77Kys~GvbSuF%I=$h^l6R!xdvZmKz9#Jz?gEI>iQ&i1g4FPRxuc-DU z?mtqf?U8Y&R^i(ndm6ugCuJ-4y+Gw4ZDI`^8FO!<(;7Ig!$H_`}GrQQ=j;x zLzv2YTT?A}BsfI6z9@(3F)!@OxZlOYPdt1h94B-^GplO2fb!|y!T_VA8yzqnK;8WpE<6R$Ktn`OYp$9$m`nI+hwDaK}ta@_A;x6ns>f>(k3&B$O9?*QZj^*#p{tIEu-%SGg3n6c)0M)}g+i7K|X!j72 zQ-y`6Kspo|RZ@Q7EEYh|mCs`zQNUupRCjlQ+@?fT_arxybOaClktgrn{PY%ZV^cqH z#$u%M*-t!?iKUa3Seg&*e|$8IAJ+6*5-OhJZEuFaL@#1DB!3~iE>mXcQ)h4mCT)Op z+EC3~n+L1yrO-&|xDG+Eg)I*{SVEqw;1Bw)b3c3k9UF;ow*@yOSUYbw+Z_Pn6}Cw zQo?cH5E#d#T17B}LM0^#$(vJ86+~oHK&HQ~Jbahhw=P)FTPHP|SqVE)#a zGqm|cWMXM5!uf-@DCT_WX4K|-oNRApcJZbjG;8kz2{%xY|34V%^9BsagXQ~{W>pL5 z4yr`MQEhS|_UghmvXWcoT7(x9hG$>KBF*jQ*`2_WmQB)qQ%Unm3LR=9r7G&SmeEwP z!=}~Jj-6EiQp5+&|1k;1vIt044AA$3;RuKuaZ zA1Txbk|BTJOzr=tfT#IfFNtb$Lp%a~z(&~0gSBo5JbJ`Tg{V`Qw)o+T8kGQQ^$K4l zbucC4oV0Q>ZD~n~s{Ty7LJp6G&POKggg3GJH{k{s!-1a+p9Q#K-D8}zPA{?Ooc^!B z2T_+n#+{ROw|;-CIhFnX+Ab?=-G~2f-PiHJ8p?Xs^*(H^B4P{f;amLC?6M*Ucktb! zoc}cY$+nt&>-OU-e{Ms4=Ou2d#Pn(_+zUCoqO;^ncyt9*4s+;r=JFqYoqXHL6wW8B zfDXJUx{l4KR*iS9qmKW1ZKJCf~K*134 zNhIr(!)u0-So|OsT?UK%7nH|}e%#c73jGET25)hnZo zineq){+Lg}#hBJ)IF__eqW~7kE*QeD0wEiHhPTH|a1Hx?%Q zC=%WqG6f@})|oX69P-e^DC8Q4_KfH=@$9ILCm{7#!^H+Qhqf#A=H)iSt^}!nSfvw2 z5`Dp-d<-#w){QWUU~Qy}ziSSsSsLw!=mZl8Z%8cutC?jJVKkYzcTv^PrFm`=*dIl= z4G+5FHuCqYDo!pl6_e*1?LE$6Cy--wOC?N9N=c7%#&Abq^ty0&ET7c%86cnCB&{BQ zS|A=Hw13|pT(ukX%2CJg1g~>J{Ca80hsVq|2Ej4N%CWn$?baO`(|sj<4MM?0916_y z8TrBPd+Uq~_9rRSqRQUta&5G`6yT(`7dDmSNMz`bKYxiR;{WBv`~0vA zURg~wZG#{&!k_`{Re(l0*vI$VG+*~P;#zeSn_3lP%#ao7lx$rNE8C@*W{Vg5CcqpC z*v>8R7@}&Lk;cK`Dk7L zWmH3Ykp%?br54TrEE2(WaDpxd6GR<=?|PJSTdO#Ga7I(bp%t30sYt3Y#;`m9Sy-k|G5E{FlzUkgwd69|(M$b*Wg(SA*s-;fs4u6c5 z@GL~O@8vovRVq%ZPEucC5+=eV-v(~*S%Dstm;$BbWWQN(f%&y^?Mc(=V?|bzXsN_K zX=dL0nq=D+DngS|5R20D2e7?2cM&bIs)f`mf+L9%#X*|K$uIxNclRMy1gD?HcB z48qJy1;?d;AXl21QZC_ro0pHUZYgvtDPrTyooXS7ikfdUUk>6@Daqv82KqWg@P_gup@G{UU$XK!!Fn|s{P<&nd=)8iifpp&Lh^MWc8$$8t8 zNE6)v`s4y)R(IPh?sw!L*RH9`Ym`}BC8YetAShIbbwUrH;>80=U>iw!KcS*|!Z!@B z-Bv7>86doNOs1E#_mPWSLYa>R(2d$WxspfyN?hN3BqR;R{VCAR6s`9f`e{GL8CNio zSSiX;lsx_HWb!JRJeYj&vP3wZRp3_CIhEpF_)Issh{6n3oGF9!_ug2|1;<48w>PZ- zE|S;Hhh;daY|ouhUuSG+Nf!e+mnJg!xxmlxDH5*35$H>V4kEi?IS*T4=mQxoA@lw^ zmg}U1`G`mL&C*G1pNCLN*BtgUZhUWz6;`mf?7c~3*h=b(73R*k?vKOg#E^A-%4f91 z_*8Ik%Z6YF=_Rl{{(Yp<)EgeE)IuSN=0b*HXI*Ffs?U{f*zM@U&1qrGPb5lNxTzoH z;17{lf8ITnSeK5{L~BYBBf@u-xrsstG#gYlq0J&`1FoPX>eIzAcnngDiC1))l;%lc zP*NI&g|HWOVpGmV*dFPAN|UQPv%<7CpmpfN83v6y>BGIgN25*{+9N~Kjm)bx8Q3$f zGDk{srZ!^5E}Z!IwYFoePi>2pL|qe*E&w8hu1S@mb%QZOx=1dk6IA!fFD?}RDdqBE z>GeL4e_2Wom;XJoQu=T60B+xbVIdjeP0X@R7(eLZ@?;4#xiYXPte2;h>liA7>a8W% zsQguEo4)BNQHi^kQ#-EdVACWCi@kdm;L5-#v_M5Ksn|gtKyRB~C!Ut+E#6 zF7mun%ha)OOOX>?;?6-@wDbGyd%5JjVd-!@=&iZZx&&tLp#DG9zRUC5`Gs)x)eXz$ z7s6+Z+TYfh?xX)@odxC``+GcHW1b-VtKCJ{znOv2gVEmxgWm6+dT=z=QDA#pJfSt2 zspoh0(>Ub*g@86+t(fTLR@PKCw$RCR!`{igbEo083X~!1M29Z?LZCF(IKFG)aRI}S zG`C{l9x2ynuCIyypgE6{zhGnWTdbfok;GNV6q9njHxsDoFhMnVpJv(e z(t20}BSJcU@kJU2hyg8xrTMdOadCDn3eQ@T4&&;ZO|%a(ev<(hSBSX0CMj?W>Fu(_ z%dOXoi|Xv$dQTMVkNT}Nua|W?MqLI6HmT9K16_onxIBe3uFv@wf}#H{ylc#6*P~+c?U^fbHreo2L`9pUi}Rl6QnDqjS;{SW zY?ExcI+BSv393V*SHm<#Gzp!CYmYa-tWkhk~6}DZnr~b3}Rbp`dm3j3$r1q(^uxm;yIyU?vmS! z1qCv)hsQ#ANWvJ@on4Ayuuy>RID7s=h&_?m<<}6t(tvs9(h3hZVOvH&qPlZCVo z;ZxiP-tm3iLB5_k{;?c%?D%si`<4Hm5DQPjPiQu!ceMZ38=R~jU8G+K33{KuzF5)u zyV-hNZrio3WH#^L29&Gw**;WJq`tDwoSfj{zKhlQv9ydRDymqwD> z?K`C5>@~~Dv2}TRNV-u~!6a!!L5IbQ8s8b4qYVe>ouMU1pvZ`{d#j(sx z1n&bb>(Txx^NyvC&^{hJen<4=r`skHkd);U9c5H!{I;%i7Fdz;)W%ugK0Q^c{JjhP zE2xMYctHg(1z7y-<8XYBmG&@b(DbqtrL%+^Jfz;V*-Ihx(o02DIhG?mF8(gYBNkOosINVyt;Y_zZ2y zxuY;a z>Ze2+5OIm`iqt+d$7l1o8*_?xVxF9c+i~KtdmTP+H+eUGA{#&iZ}4hBxG%<}68~*6 z4L|VSUi#>a3_g*~jD?h2{6sGX{#bUgtKZ{q3_j>uQEwC^_g4hKKonJPHY)#x>-S&b zfx|WIs(<0SDi6=8=wDoSH>br})wmT@mtM+CJk&~yz7z8szkZ9|VtTh4f3i*TE-`N~ z4=MOKpy2-Ve(|075ZCQn3l<$zHbUR>bQGI;4T@&%%`mo8bGceoAU^FzMMVOz+1?Nq z@i|SA-*}X152ri!^yBT|oo4zp6@$J5&l0N%s>;yQ&46NBBZ^u1r zd5OD-1z@VZDUyj%{}b`s3061B$L8LDr&!?Iz!N94qLCS1s&>?NiY4x&A?bP=SSLO& z+9HRuW)WLew+fCt5a^gE#O-?#kF-3j9`J%j!|On&-oQjpz53zaf2(u{(djJ`e-LeI9V(5ZJY zx$uLYK}|HTC?;vLzqNRHp+lm3dTHCPB)K*ZUR_F+&`#C!vpF^3kA0o+Vg3`k5D6ak z+{XlzYU|qYBgm(QdPv@{h5S7-ELT{$x+t*p`>TJJF{PBsz<{ zVmN4Ae`^M!>jy_{Z>L_}Y3ZbeX}ZdCF1D4S3Qj3Z5$5-M45~ zO-+}e8&yDAZxW25ks>QI*?#)u^JU_*h=N5G=eMBesXUsH zvKT_r48MzYTvg=M_YKA&ls};6ng(Cp!&fCzsb`#36~%uc(5iKqIQVbCz9RF23h9+M zK^JtRu?_|hGQe!U{ayMdqh;wBdFn_617^EDC8yTu4E2|HGv;s)To+a#i$iQd&*+7MK1N@mQ?Gi;FLoeUV+* zYYo*GmB?b?^Gf+$DEhyD3wxfB!;LL5aj)|!k6kC56W=^A+UD#UZ# zOsoBheY1g;47IJtX{(!-pg#imipAdoysUbs#m*QM*_MuHf|m*b+je*?qof=C z(o>`nIcIc%G22givD&(UjnOTgEyZE>Tt>z6A*Org1OeKum1rIaNwY^fvMR}i8b~6u z{kzZ><+F}=N}Z%FCGV5BZ;YDM<*K^f!G0CdX$a&wkcs1*Qo60N% zK5>#6(#`G-eY`>aNxDN3{G%UB5oxcCAb@2AC2l@_?2mb>#>GF+mYZ?+AfmXswtfgg? zMM%xZL-s_1ghh|~8*L}!4fJl}D#LB|Qhfi~wZ_BX+m2ypD~cAq%dMNdaO4*we=Hfh zf0d5)mjEf`2;UJSVp z3Fl!cJ*@72*=+WlaNA&+P+M^rtra(a49|C8J#X7*gkraEos4$EUf1@^qJE56H?5wR zufW5o2v~CoK?UmJ>WX_v>Uy^-l0+w1KMrz3SvUL4*F=8E8AoiInuf4A5SAbw$22~H z4Y*7?Uq%Y&=2Zv>5<}PA>D{d53y%rdpf-w zJ{dVtwr#<{t{pr{9DM;A$sNcRm64-XQr()~ptXo)0u~L-nL>K^`Nv z&+_M|e4DlJk)t`x@bLS#BUJ<0-_~eRRRl)Fh|dNZ=ph{F_GrGPp;+qB$G z`0{<@{H)nXqt^#Pv2cXv6Zzp&Q7vZNWUnKhi{kDDx$^t$kY3zG*H+4Q2Y4ai1FQd0 zk^bjv#dTS@hvY#c{2{i#ohlEC?c;;!{N+^n?_2)o-WY2-L#y4mZXcQk+_NE&QlfIv zr3Jt%%}cf*S}EGD5r-?`$VzWbWU!OzTvk$!^+**i&Sp_)4H)fsW}-0)YHt{l_n7c5 z`uY$aHdDuk`Txu3|FAk=Pf#dgv15sagfS*5^l&6``9IT(h>1R1;4?b9=;TUd%x&{l zG}iIHjLz@yw)?rz&fVUA#)JG{3Y?|aD*C&)4IMnv$|3E$;uvx^tlSS9_xS1x|Faf^-b zr2nS3R+E1<`}3%iHtV9JQUBt|tX1OYDqd68)jy%M$ekFY$vBQ4=>{1Lv`OgZyeCM8k>XS~>xE(yX*zIv zO4-f%*z*ZB&LLyoUkEkA_h0-^4H3iHc2bQ6EI8)TE;dlm0qkZd*|X9cvDnjGIzTW; z4F`kYzf9Ibb^IG;kGQNu16I*zd^iHiLjF^Fh#LeL$x+w}t;jUzURK5rY{ZPD*L~|_ zlJ|4Cwz6NRG>I^`mVUxE`wpU=rB27IXl~=JKRKp7slA;vrz_o;sp4du&DNZ$IT{;z z{N&E(BW{X@dT!X7l9`p6GLK!hin_Xrz*2{+xcs}L+ntv=ynZM7AFK6PjE_d==&N-i zi6mGn-UEGGaR)0rW;$HJXI8jC1DpTPGDjm|i0$CeZw(8b<%1DuOZsk5Yb*QNvVbzl1mRiI{pG(n_o+zu?Y`zsZt7=)vcXH~5C8QiN_9gf2 zI%N$N4**fecSJd1908oXxiBF7K71R3qrluB^bHcopdl%0UpA1C*gQ3R5GWEmy>#R@ zf*Hg)|EUVb7qcuX$ClxeAnF@<_Gb-Wl(iX&9Hws{R>%(Y10(kiJWY5RzbCzBuVZ1b z{v$aBKib5VB$@>Mca^NJmFR*=2qI-xSL%6Wvbx`hI})amu>t6SX*!y4n<6Og8c%tf zX}x{6{36ob*@?b+IGON-LP60&H>8%Nu_9(BKqXP>D`@nhU8PPnG{_G}N0<5{7Aupy~3 zDo1uIb|(2)qmN5B68<)rsjNTL1LIiv8Em1B+6U^wiHx7WN0RnL4dYh}keB)8XQTx~ zuy-PNv)}R%qTYgaBI|#p$Ib_LX1;T62`8)shzEvRlSq@ zuu|jlfiuhsJL#wu=Rp&Xh`62}PULG=LfBf_Slfe#rL>;jJAi)Y3YklA6hf;dO8LwD64_*IgsgxyCZYOB(}Iq*cJ25v#Qs={UYdY$}t4UOc#Z zv9FWohXzuJ%nx08HYwOPLZI756~Zx`DVfi~wY>>WBO`fAg*ej46DheH+_046QZ+I1 zsE8p~s}z!gi@mh2mf_*7iTf{Tuha*{Dv{#~IA2RP$C|h(Kz*tD55j6k)ZZSk@KzZH zAx~XzX7EeLr`?_=4a@2?TY@-o@gi()_Dt9bK_sBp3%-eGo9?vR_Z}us^lIPO`(r|> z^I}+fwM0%vpD{5rFmJdC7|Tj`FJoag=ej$~axZWbG?GHuhwvqyBol)HvL^H%C4{Q~ z-g(db!C`~b1kf;-;I2udeNMVsc{;{DeHb+~kOWEIouM`Yt7e!jP{;dgdh;x8$+9i< zk5Vu&5h>If6)n_RHgJ7W-lH$*)q} zP`N9TQ@rYM*@s}aC^PZk4-;lb;bPzTBkH}rCn<+k-KFUdYJKQMuQ16d4UIG9^4f}e zjM9E)_dIBBRZXO6d7BNy-MgJ6@#9ov*MdUR`AG#pO#&O%gk?MrXOx6bxH5GCQ?*K2sBEl7^wy<(vD8!) z+0HcFsnwg?qPD?B!eP#v@x%@a(KllYEK}C+b{!T#%pFD1-ZblNzEW;%YPwxAF_gIZ?N`Jy3Q#$&-9)3GctC0L z=D6BDyDL0kK&(;A{rVoPkRr{I&BJz;K?`NMI~i#llOm-iwXRcIyU92c;SdXx&mEke`Xv<=SHWR}eVf|4l$7nL1uVfnPKzQz-kk=ZoX!mJtXOc^ zDyWDl>oFqRgi6`;$1fwt=yIwc3$PeSQCZ1jLjYg@K zf5D_o!*6GuB^U{sRAp1$OiKhZ*bP3GaT^RJ;0tQU*q{=TJvrp&T(*=Nr8_?KRgD*E ziJyF~{6?~4Ih>URrn}1Cy?h!%OlC4gMKp-?I{I}(@zN9_8=nz<8(viYCAQ1*5r}R+ zN)v)-baTcIvjq*;?p0(0>#bXDw!sSjJ}ViaMDiPb zEURvsa?H9_W(7IfIst8KyUK1p$s4rQ5UuN;Ik~vnb#>n!#n(UN5^|^SEco^;HkU>| zue|_rpQV*udo+-t1QJOZ@nEGznr`DMFqiQqZqg-|sk-+&02mPC&*~O&Q^^wsGG+_I zLP?hB&{tK$9pm#X}VViJdhvYX~qAqiZk4|{)cq`LH_o- z|8e;T`G4$4tm#~S+$fN1c{f3nOq>l{+Fy2iIxqc5w?$puEWJi18hNTD3jB;0m~F?kHe$k9}Phsx4lh$*;{qu1H(Vazu+1kMoiwNB;#%aK2DlJ(g_zi zlK3T&quHa5NML)Rt>$N6GioR19Ta99eC*IS7927N=X@>!?rmTHMDY)xto;L-bNnOf z@ttWC)aFg&A_DUFzlGZu694-$6X_NHaR;$s{re7^7EMJ21jcArh^IbI9-;-B{KFBBoPn zX-p5HUkG2$)nPxsqxTr(adCTPm_H8?J2_ZorT+Lccz9z0t@wFw6PQ)f$Ck<*Re$_Q{>GsGTNRO@ z)_te%-V&-q`Tqs$HGTmS)+`zT5~VW%n*8cW8+45&xK>w3O|5L-q2xK zyMC?FaD0kA2Ty(!*H}dky5BQr4;0(Q*dSYnI&VxFmE=Y=*?inAc1}=^R|b=LzNKf@ zK`Kp7Kg@Jc1mt2f(;nauGE!3)NRZl6hg1uRQg4bNfEjnO4+N29pif zD4=U4VMl@a3IFIsI8u62f-YKtnnZp@n*4okq((k(uQ){;Pop78!u}`l$ zG8@LJQ^RH42|tyg3K~?*VJ6LJv(0_vGO2)^j;!5a8L~w=2$4<6TP|Z}AVzq1eQ7ig3(v41+=~S1j9RypwUQ{BiG`k5 zdW2Nc>-#nnjZVK}(5>W4l&UYZBaIJ2eI|ne1}aH)*9c=V$=iY824fXqNbS9!s64lc zECywT^--&;IBPtyCBI=g)5ZA|-zt??3kuB=ozoiM2+f3X!P%xfeq8gsjYpa6uDY9BvE&y$#_!tjTX11$rTd$p4L z?5c$@n&fL}Vi2wR)&+iUT4<{Evf+rzlj4eowLROVII2WMnM7?YE%VSTe$oCa3yH+h<%11TM(KOo5c1y0Lk-vrt~SR|x~}qLUwOH3 z=m(3Jj!Le)MKMmq=_3VP8Qa=6dzPx(#M2z@TAnKsU1 zHZ(175$Si-wV``GsN?MXVL3LwTsI=gaU?Tsyu|!&v)5~+KAtUIg4r+Cvt0PXCS zM)IaDo@e`~%i0SGLDZ6VOjQ?GqaY2w$+*>51twm}>tjWCu*=>jlIvy8`?_8wJ$7oQ z*GX9_5v{>}(}Ry(ZM)~Qzof~H53x?q>K*BPm_L&i<4a5_XPE+s`p;`Ila9pu@iAKF z#>R7WlPRb=dMr1sGo&?A4+{QkoC;%Y|lf%ISHGJ)@v6gu(N0_y6`s18hrSEGNRcTD7 zq7yC`*`{<{oRF`@7p=?eCX-NqCFj@0@u$&-|HK1t&bC-^kwt&O}Vr zwm3s2-afHhA~aEZolkRS*Hsc4g)IN1&m$nyeqTbtvmJahN(}zAt1J;aDUyZjc=ZHN zCEap|Y@wn|%&Nnzr~?$DVBLsSEKwF&3jLvHyS0Ejah|YHv+5?80}kpxj!u~^I^P*B zv&e!mosL?c)pr07;tU2D07AF!!gyrTj0?SftPYg}ilC2boveLjeSI%<_#Z<5VmBRMGkz4BLAhD&>7Dn~n3*gXhYc)L}0K zI&*v8a^rl!``$)Hs2CH@J9P7!hwuSUkx#dW?m<01l1z%6m`_4f6@ppR9H-is{^Cutg zC7e2pNw^DS({AN&`luO|45(9o96JCLMLOZC=2<*8j?;CGPY$+F=Z4_<3;CMK4}?ytyqEq&1XchOC506F@&>GYnXARIfG2??F|aY zax}xcgxd1PBPHyCcdnm~0xMeQrsNKMlJ|Eh^KDB>rXz?>SQiLODp$LZ;qW$QPRLSP7=I8*|Kak zg15p*Gv`$w#=XFWMS;!bB-ltn$7iCPlP|+*n9nV&FSg^xzFyW_i0b4bh3;Gd^>^JO z4E%wO7?&U_7zPmb1#grRpJuN|84o7ZLJAPu{ z5}F{ClGTnw+a^Zb(#bYZm?g4OYm}zkw6Rnz7>$-H!AiOfqoz!|@uOOnrg{E=EFSDj zzQjv68XS(J!PmbKi1dnL1Z$1?24{wb!y~@c73c6AE=Oe*%QLJ9)E2+Y!7zp653_(7 zJ0^oEpU@g}Xhh?jrIRF0P;ozYlC=>iXrP^>)-;o^+Iu7=mu42E+&jEQ+DzoM6eHm& zK-oa8)6iAWXmU|?oc}dLFD#dqw^-=KfHWqOcDGu*h6!yrSy-wBP*cUeC($VgA}YkR zN)?9n_ydL+D&sgi9d6B{h^CZ^oYnZqjub(yHNrCG=f+JN$RO7k%XCWItAa&H?Acvf zjQYsBZn5s++rd~X3ta3hOkD199@vT7K=V(v#kU26sYRYGL>A&v+b&g~)vDebmJYLu z!M@C#i+B3PkI_<@(&-uL2xd{f1kl$t1pfhrxH;vq8WXvhB~YE@3t zp@NEmdls9JX&@3(w5=%VR!jMmqJbEgNIs}&%5{6p>)QmAvlw;lIC7}S$B0Atj~rZv zbRU-NKe_3nKQ=`abcAd(10O$-f4S)&F8_Pv|BgFI-hkoCy2`#KE)`Qc`aIEa^n=po z1a&eSop)5`S_kJ!K4%OKkss|f8Jz;(l|YpGDUvl66l$zQCX|~h{t=pCpjW3~*+BFR zuld6R8TdQ2{cssXKzS(ZA_s?NP=o4t4EzPnz_|Y9@M`p5pc%qwG9+tPuHeuNtNc@7 zI5dO84dar?PxHCSoqqvV^*H^$jQIU@th#Vq>c^&MH0hdqJfBoz_tnqP zyh|il)$#K+*QJIzr!A=bAb$#zZn^dWM9=Y-uk4_zrD3+LysXj{j-)WBXAL*4ep^>M zf&W_Dllzr;@tYwCM;|hO=?TiR-&qIlj?8D!ANZP$uvLg-4u79i6^e~5Q+xM9y|1VK zvy-r5OS#qnPPkXYmL#^(%L+V>CD4-eUdlRXjpH1=r@+fA_zU5~#yE#K&;E{8bw2af z=sg{IxLWNMK^hwo$zEw~RNSNeU?*4oH__2G!2ZB9AJQzw=RH<9{A`@dd@_Yo>q&$K zo8)~0#H5V9Oa z^N~1w-JUBq^d-(oJ|_RtojC>N(fAY2gbW=MGq*y)wE9c)`-NL+F;BPqqt58dj#VdF zg_QOCf=_w*wD0=0aSkPUJmE{%88t7T7E6F5 zyrw|62|F2X){aTn?%AL!V_Hog%&kPXa>XV#VGl^pWKcP+Cvy-mihg+9XVYn-11jZ_ zTF|qZys?>9B70~9s`ML7;cwf3zWbeN{kLsE|LRWhw{1ZGa-%);m;cAN`44nZjtH^9 zJE`Tpw1cgTJu5fs2q#rkiK4(Q60QwCz#;^jP$_LA4O>v=%{(I#Xa%%VK1qb&#Mx6` z3W-V8fdumyUxSAQg;F7FZw==KWW!Tk|&nIO8XacVxtR~NA zHl8XB#mOd;Y^OmYW!EvCII`HD8rA1awocJmW@slY@$T@;foH{P)0Ns5uho3#NYx^* z=T#th|AW2vj%upg`h~F<1f@$?dRKZEmEH{y0-*_^g(@AXD$<(?gc6F0(i`w~PAz{kK~$vWna!{kMV!GKC{Er$^%{S8l{iZKmCfO7U_?WYA}zSk}LgQz5vY zOhy5&5)EZ$oV>r+Q$+8mXy+qs{7_~m{<=VD3p!j*=xjyaAIKY3l9&PnxF|C9svok( zsJefM@9dA|Q%dj5h~c$LSpHt>n)AT$ZeooGGrQ@t5w53ZO5CcEHSE00EKO%{9d_#x z$Ngy9zBkgO*il&ml1WV=AF>$z{%75r=eLY{Z=f!A6PU3)U6MMu@;sqK|8@uLJ2L3F z)rG2#NA=iIojz%lb&H7lR$ATtq+xEB=i;*;+CzK2@`;=|dJ&muuh$#Uwn=FLvGo3m zq6i)jU2BHK11)jxN5!9PKK1|}Jp9rcbSIRr>DE$QBrtSwG3O)PJ-JEGG$mKU1RBX> zPi_pgUn+x-2B4AgEO}C&lZSR3c!biGgKQxKiVjv;ES;PY+ER-G3=1u;56N+m&kN38O|v6D0^%VV zBhv${+X{1o!Z6k1CwKhg{WhFE$9b2P{WHmT#t$9n<5Nwwbf0VfMn(;} z=&q=+cyM90w+4Rw*2na?TIzJz_+1TGnP#jeyv-H!7fvG=uM>Et#Tz$AMSCgrdi_d| zSO0DNJzlf$Thda3nJ7Mw1#^G z1n5KbE|FqGk@efL&+Es<=Dh8g%A_hvZVvdby+01;@)t-ojARqDmPM*Pk1)E{@1hwG z*KOfUkB4xF>Fv^cN?-{9Yi_W$23{h(uFPwFf(nUx#XgaGywzKM|&l~b} zv7DoG_ZYUTGF6kG2y{9sS`3})c2h5|Poh>H{M$&Q+STMxtvjyvy+4X0@GE-wnySqp81- zi9DXWkMSq(#B~^``c$Xu&iKcCjyk!lRCh;ab|7WZvrXj7B^e{1;#s*Y_MGY|;lR-U zzgSHz^A`~F@35Z#5-j?qfuH&7#|$za;xBr$oWp>o}DzDB4DcsormQo;AM5$aeXM#NiH%*IL! z7IyxN-_JP5q~Xl3X-F0tUnvV_xwGuT$SWW?;UM9MciS{E#AQ^?4|_D%>DS`EA@6(8 zGWA1b+zXvaMs`~ew+;U!S{5?xcw^;v$Zuqc+~|N;Pp4M^IE@)4E;NY=bP*pmP?dI^~6G{(R@`4e>kl_^;A-9 z*@$HQ@LqLA=^sqp#3$RN`Nj+Lqzo(VOxRxcccw?JUpc&CykTEYZmPQed9dh`(m#nV z*=*76RBF0@*5~kMKljJFbCLRjTc0A9UCt_HMFkH+ELh|$o{^DV5l;+FlTjMJ2>p=~ zBV*ma_-UE)=I$e=5K`_7?r;AN3I2_?=l)*_0)jti11CQU3Dj6wPfoR$d0?!Z8{9To zv(DBYx)$NnXUKG$>>|j0XkP8#(@93 zgoJ;O!ZnJ+@kW(HRsL(R+%T%;S`jtre*>$WB?9Q|?D7k>S9 zg5I<@aiN!IugfX@CYn9vb71ys~=qH4s{X!@s4cZ#nUDc@nbReY4VVjE97yfagqD2pBl+- zto*0Lf2j0-F>CzJ)@1p+08r8ajSJ*3&4EVKa+A7JdgHnFV+j_`-V1QXawn9b1~jGI z3~!**Vji#}3|*LQdPJ!=5`YioZp?Tb0sr zE^Le`&UR(Dw5tVzMtbkjXc#FQNxXSF&n%TVp>L`!>x}{Ni3F9;LYviPZgOCr3r|)W z)2gQQZ-b&)aKffMFf^Lw>v>mMajWz0DYh3S@9DbMKVSo;vDt#?qVi#GraTi4Z$$Dp zo;O))U)@ww23(vuduJOr!}Hl%*7%J_ z5ibJ|k5Q=sW57bM+(nfFi5hk-3Zo+`xkyM^=3wCM>2gWFW@~C?eHdN!o&2C@uNK6` z+Oiw0buyp(mB!X7IK~_hjkn6;D*|EcHrH{w&H(d$v5rO$y?xFMiv!1k@Yp)1t@pxy zSiBQ<;V9N+I{K?InyLhFfUpP2^HB+O2~B$m-d=8d(3V-%9gq9KM`!k~{F0Q4Wn=qG zPY!Lk?cQoui=JYwLC(U+k6WEGjRRG@dkj61nFGU7o4M^f6t4+ltx0MiKd`UN4Bm4) zA+b7EbzqgOBLj6cQuO#rQhpj-(^uBg4Z%dKMK z{*RIOQ}P(5Uf*$t-^eiN2B)$B&dg;+T!x(~)3Rb^^tYKr%9N%uPtJwLna7TN!dEbL z#+yitLF1rPljr8fB_(Ils^11)v<Ew6l@E@#UIptzjhyv)9vJgF0 z6*98Ve>15lO*0410{mucYnw}U;i}2r<6&RMM>R{3jjlR~u+);=?RC(4HF(?)Q)^US zT3lt>LB9z;%L@CAEJXVQJuggG#O(|hTkXcpZZ*)6X2TDbH+>Ipr;?dut4cJ~C!X9c z7PGf=vwu8@NNa`kHql1sV@a)2nW9VSvJY=a_tK4vU7|B|EvwBEd0nuJi@D~|T|)J4 zCpaeljog#@U=ap!dy-$7Pg{t^V9NJAU&ZaHb^pUJjyZN-X6hSEC!KX0m10fPQ}dVE z0emIxZo6d#F(z92958g!Wr#m)k%3J+-~rWUj)=y!9ygSC&O$ZHt~v&mZ}iPMt+o+Gtyo^(cUM-dQ6u1K#9U6F0rH9fKP8rscuuEpFOArJWPgjT$5* zvlq@IG7!tFW@OxE=JV1a5Eh59Gg(Njvcqy--fR?^vai{i!l_CN&tT_ND`xwAMhlEU zs@_QiUV-0iV} z(I%S?UN5b+Aad}XBnx!zWi#xo-oztWq3NKV*Gn(pYImjYTP9?b=<{UfPc6@ul(|hAuud77Z{3oT?5sXLHBj12Nmz}oqbxgkaOIOLSDl33%S~^jS=dSd7h>G9 z41Jk1ek^4Vh~O%@;cJ2`h+2%Wx*&~31m2vPwF#hX?B)z`ku}XtlR^kc=@xOQO32g{ z6SJtc-;b-KGAGPeAsg(!;~th@bPztbj=v$)@7vvCvzei5OZ%fvcw9HH2g*fbq-9RX z@_zSGyuEKy417}VGlKIIHZQUe7}b?Lf)c?#X9Y}O*&nGW*jMy_Y=1rKw|*p1RMYen zkwJ9YfR1W;Z=fp5L$tg{bstzI=}HXgFW_ohxXqMEyY7=6FIeJtu*nEBzGa+T(m0`# z9#Es&yXjlFeYh*rx|IQy?EUqdEE!dfwyXQr!^KJBj*Pn0S;V(D?NVG7E zIz9s^`C_5@v+?YgX!Vt9JG3`N>FA~Uf$#*+=gZu`8}J<-iBGS^UKf4~^B3A2TYf0om!zjHA$bnAs zh=$f@uIOgaYu)vxq)Ic9cV4IWyiNtwP8L;!CS@RJppeFUqwSm1Ef#bH$7YH3fQ0HQ zjv1L{#R!?ATedp-isE@;dfi)+FOsNOszTPbzR0SkT?g1Sygi~_4(DZ#?$Bw&2ns%S z-v#Z9wO-n6v_S8SY^3cWGw>LGB`(;yaMUFEh9;|jmSx%XG30oxpp#6VI?nl-GxsYl zQ;Q2LdG>G}L#hv`kF2ObHy5qJtgqg>ko=&UGB@~bK|&>W`mCT?b%ocW%@#mi;8Qvy zhqb?(q;)kgvfApG{zx#}5`y}k?Qji7RYB4}l&ie-;J^01_@k+bVI2)lv&v3xbd;tt zUCuXZcsDn@?~Q=44M~UyU2yGUD?wc@qp2)6yU8mq^~9&jFy$@v0+glE(80|yRag0W z%ahG6#tYHc(bOsCI_T}{OVops=FY_ob&n*(g=j$%c75)wHUd#7(Ki;@uRaJSfr$WxsvsN|3dz3;o{&qO@)b!;&nP~tTB)h+>_iolI z$F-SJJds`H!DH$~e6<3`)^S1z>z|(|0ag`vFy*WuMPB_NdLxuF3tzR&?e9@4^wHhe zry!hBn=wIlmg1OD+6EvXRt{~|2=&v4!W08grb9>ul|3ix=|r%1`gU8Gjfs?e_@}Hm z#P;MzmmXHr(m}ci-xM$>IW~DJDnZO6;C$bvOCK}=UTaaDT)5+r^w88NZ>C@M&P>%soA&} zGsE6;<`bA4kaNk{p3H}Coz_SIOJb_n_sM0SL~^^#Q9AX4*?S;x46bCUKDpDeZp4Vy z4dLS;9c>8BNTn+%MIMQYxgqVR3L zH;sALkv&%jZEh^GO5{<0l%BKYS=DbG^(RPm^4muti6T}Kj}}>S9D4` z+pGN}ks?XWJqcG}7dMyN`^ec2)#9{VP+QiC4ZL0ns>taT{f$f@%Zjw`x5-d-f%h{x zO?P3cG}UipdAHXa9^6ix8sP2wmi)BGPJ${q)2%D4z6O9LIM=~?Egn=fEj$w%(&VN) zD<6yPqD9cgr*NngWE4_ZVeDspR(az?wyJ=-%xTl_=!ifGfEv3kHxOvtXs9bOdRtcV zFPRBTXBu-<_b<5#XW0d#7Cxu*nH3W;pv`Kc-%h3uRg2-5;ATkl?Bw$+_ozC;<=-ewtsplV3<7?cw zY?(52a%i@aR~~3aa);q_1^nU|bmcztwSlef#{nLGR5rev4#0wYa2;{rdltPLGldy6 zzf+`=G8nl3d`dd^3&OV%hlpx9^~g_OzZf;{y|Cq>Eqlz><>L|(inXLj*IeolTjLqh zLfCmdL;H@NgPNYxdgLQ^U2a&rI5jF&@9x_4Es{bnBD#A4;J2rX`njga!w1 z6nCc!GYH*X9Zx%T>KrU)B%Q0t!4JZCf;HcA7- ztIIrFFnKX7di?}@F9Ye3)^xY=CN&hu+M(bQ*w|^p#6k$;;lEgJhX$k{BCISxJ{YZL ztYihe@4hf{rFg8|i?vSp`k_V1+|op2Px__IPZ#euBGyo~(;_kCtSFhkes=Kfp(`Ew zL3Y^XT?o(Z@cc|u$OA?Qld)vrGzPt2i!-z{0fX21PdgD>8%X)Vb+1hvW zBng-rGlTY5({|HozV%x4erza&b8ci-jJe{TIn*kpJDLH*QM5>K2(Nbb5jH?Y7F>=hBjk%(1Y2yaZZcHXP2Uv47D?& zej^)>v;JU71!Z$7lq*wpMVvvNv|n}#d^M_i`#Z)*wyAdOiHb`yARn_w@)TB^jvnxk z6qu|D&*&!C25k%>ll}1txQ4gzk|zXf45PhSNU8VykTHS+-bv|BV)f|+HODuzVyjqom-KZWlGr6{`b^S-=RnG{= z%xb>TH-rt50CgsrY{qC5B)d>|k5%MSS)HiK$rx5l1yS?si(pDGHt zde6kk2swu)q})YR?egwn?|I9fb$Jc&?xg(_V+H8i$KXSfd_RHi-|fH$=(0&oZB zY^dq!w$o8Vd--=LrQYg;G-6Um^l5z6E=QN%)k76B4ZWK3KwVj{^5KAZ(Mrw4B*%ih zGb0*}T9wsy8pC4?R?~m6Z@1wK-B=e9TNgSSCv zkp&a7x5|~dE!7&Mwlu`Db9>#p*X+z9wepz}vrGbMcZ!s{rLAz(4mtzkjNTTK4?>e@ zzYFbKkz#qdJuIUec2^u!4~51cI?GQr8FZc+_?aZ$wST!wzt5|X!N@~Q7`AXfx~wZU zA$MR|s{KsL@w-7X2APc-XGaSZf&Fh2@<^i=^74Gwd_{(Lek1!)K38oV zi>)v`+cP9K1(i#pgBBP8&%!$uuvZTd(*m~TAa|=_S!jEb{c6k}E~L76>ykuuz$YB& zry#LbKyM(`9aK>#LM#A@0D(S0M1)W4CPLVivioNQaLF*+Qg3w%-pz!`NeBkyWw zHJg3$KH!!|z{vA9rg)dxJg}Q-bBS_lEhb>rXr{&s>;^DKjOsON)^=@Z@fT)amPBnp zN3sWYu3P-T_{u?x>^%1t>h4)oS|yQ6NmJ_$mMy}-aUgiT0*x&CjjZiAvJ6x9_k527 z$!yI3zACYiLCmBLUl23OZ|DgikAS>}TFoxX52masW4M`1}?Y!wc#k-k+;!(%+0p7yOhwpz0yQn=gC5H|0x z#9{d*r?srh_SF%1Q@9@QxWg#-7t8pF|@O`WRr=5_6X%m&(dUj5yEY4DbQS- zKIus^)-!z5I2QLK50c1TVUusA{Ixg}#IcTXGh4aYnU{Q|hC@s<_w2NS^@)=UY1UfAXkC-Y_Ag(AEp&~l^2ugple!dN~O1I>i0W#-c~(`gJbor zU!$YTM;Ru*o{UCI0=vZSSe7pF!aNP!ma%|4ja(~6s}3BoJ~OW^YjbZnyiOTXR(5+} z<*NDvOV7qTV#H8d?9ZJfzS%2w?kk_J0Ob{J$c-K9NBwEJ=ldEyiB%)Ty9?XO(D`ym z6Pzr>z)kYA3bs$u)7-k&qg+fyt{qmvi>2Vja2eaGRUH@3Ea7_^-d`zXbfz&W^emM` zJd0e|kxVqnMULBdU94(cqLa3l8=z5)dr$fZ8Y2xcY|v&OW0gLo38Uy*${DMaK#da5wdr=3&tb z3Mpm;K}(o4r>ms-klLEjm$!W#LVBhX7|Ovtx?briPr*+9kor8NNqIC6bt3D4#&)d# zF!w&PqwOA0++N;8BI8w z_bW5WUC2wH`fTPY!dkimAE|y7opiZk3p0bTa0Q*hKc(J$2@s#nM@7!pxpkwEZZ0u8 z2G`M&fRRQMSE(`Y!k5mtc2j1YkdT*U`K#t=mT`Q!-2?{)5O0KM=XLHm+Xo)LC8{?Y zQ)5}N4E9_6-N0`r;Y(*9Yl&37j!TOODxLOzUkB($=nf47?Ut6j!u6(ko$xij0zx^y zq{e%rR#0&_{13Td&pE_3$EVdwOPJ8*%T{0Wg8WHoQGfGL{q^^sRn`7?0&n{_GK8^z zzGLXWxUK(-o!na}?(M?BE+i|Hm_SCZ^iMPG)_g_tG0a+!l#yBEJ)_xS>*mts=zzCF z%OzluytP1W`Ds-Si<%&_w`9swiOy-8W!d9S8x8%wdaA0EfpnFSYKt$DcrQ?JX-rh% z-j|fJZ>BG&X^2SK`H(&o`A6*PQ77MqRf9k zr~J!{@;bM@4MD_rAR>wA9aD|vXQn7-DNE2T9!}cd#7Qj}pTSx&Q!k`UN^y~D4Fq8+ zoa(Y=*c9qM@E1wmP1N>+r%;AKhQ8~H4H8ErjcH6$fRry7aFvQC$Tr;V5>c1C?Y@t% zd4AryJajJ>OV}@ws$Hb$d@71Wb1B%ESMq3@&Vp!zMH+3Xf@`mnPhV zIdTiRin(PN;IPEW`VC^KT&pVdOytrN3-*m&1<}FQ#79J0{5aXFX@587nJ*yb?1hDZ_x8W z<4?7n5@a`0<0sxGAAZj+Z-GWKqm}r#zbKl8jNTq16BC4QSWf_(KH)Dxtaut3bX7g_ z1xs0Rl?P@*s6xMc?yzu&z)YFlygdqde)a?fQ$=E5g zO<$+t(5GZKUunax<^egPy*CuF}_@QQeS zy2W#@umtO%Ek^{#td4H|@22U0(&DeGBE;^-KpQ7UpG&6P_HA{6K$?#n*Z7eA=U)S{IKSQ0-bYTWUPsYLT zzdpPdG7Ap%t+hl4Kb~Ee<|mR0aUj=W&zM@mS?X4~N4kdYnH|D^eiOlJ6p#GB)Vt}uGpfGbhKR??B(f4YYUzkJto{K|8|Pjz@d?Q5UO zE++rO?f%u%Oa8t0@CKoUnYO3gHUR66Kb(5}TnY&courh185BDd@()RsY&na4f7rxP zA}oAPWtBt!%BN$tSeYxHqz$Up45AyzcXLLwi#*kBrnGd+yun-!Uy4f=L|HghG_DcB zrGG6m!u+kqXyorTM*kXq5E&)we;MhW{&ybbHQRX{tmOqXd|Ohr48rw8bUo&T;dV@| z8u?3IewBi$(0Ma4l_RsuF6N^@R&Y)+wWlW+m+$ifU+V#5m!g_gwqMsC-F^Mdn$3XU zEw(c$x{0}0cu_sp6bGc=O5?XytD$$DDMbo4xFMZUT4z2;vc$6aY_91Vr!bSj3uWoK zvN5n!-MtqXra1o>)L&Myg&kwf>>BLSjwL%5wwlmVwAAiTt^U%T!=O)Hd;q_ zZ$362$(3Lpx_E!ro*VN7E`*t+h8%_RfSvTYczjy@%(yG&YS_^6Y#P|qTme`y@VaWYcsq))sw+s+s(>5q`8e8N!mldyG>^>ed5D)N{tjNOIKa2Nz z?i-w^YB&XWFG%8)vVEz1`Y@GiXY-=E;cPkU{oa}h)Vj$2k6<(te|KMWgqu7O8K+7& zjzC1Kb_E@D52)~EYLo()nNK%su2e$o*(NJ(-meSEH;J|39&zSovl67meIkrYmjP!wmzz<~Mxc2Y23Vo0Y9txg zik~dU-(3)TwfJn>=_BafVYr|`Mi*cjNvD_frI7YL)}(a}FSNey>cdyQNnXcfPYq`& zCu|-&P#=IsD93;e{)LpG-ts(8JWdVnS2B*2H85-i-x?^fG_MmDgQQ=IQ}fa72HEN} zs1BdZxTNPf$ajS4JWaz(EQL^iZsu2A_pPVSUJrN2%%&ogy#g*jdA%cdWjqAHG1ik_ z;^gBi#3N&t&l*?vxG$&aD>m|WsfnYbYSoa6vAbavT(3Yli^sw`B%`;mlHWOm;LiVf zLsi%?Ns36%9@Vi^#F#!&_>*tS(2Yi9CVt{rhc>qu%?EHo8KrFyqRWR}CP`~E#dUnT zZYvh=%?x6|sOM$r+?(kx9W?C1Mdn48Wr4|bcbbh)rqg^7Q$bjEedhKQ0Rv@0K|X)0Kuw6Ha?(<2L&sEY*7CqKu@h`}j0$@<3U2-A3FzV#y7z12pv$ znQHNXt(U8N6Rch!zKd$#(S{65_0s!xntY6np^ng`3SX4q7`$PVz*C@26NmL;xlR=~ z)w6M&%2m$FkW_euqc(ZR;PtSDXP3$To(*K;YwGw1lTeJ*2MFRW}A`4IB(`Wr}zNHrao?7dfZYz*l^rt=Ao5% zdACbWZL5LFGH&Nw)4Ux^lsjJ=jkyP1(WAm8zB*8o!gt}IF~rS+l!PVU{6b6pL?g3< zghqmPmna&58)+@sxFV{uNDL+R`P$BOwfQXi*MRde^w; zk{K~|>;^|}-4tK{BM*L+kxhyMpy*AOtqYpyXz6SJ~jKN)LTbch}4J;89{+OHQ z%kJcB63vp}6BlqpYd2L$GSzoutQM|lVSn|u*J9+Y4_~Usd>o4+g6#R3rl~Zuen<%^ zv)ptNS?D#e#(dT#7(Z}2920+7r5O(P;fV9{mEI>! zICm_IVtdnp{Jo+_-*o~8LvdfC&%4K_PmGsECnYW!U6S!*W3Ef-dsm=t|9L_chb7nq zR@7ZMD)H%0x6w-aL_fUIV@d|@)eiP(2-y0#;atII>YH(SgZ701LlUPwCZ2}RIX@HZ z0<%r=SkpG@O*DHZHEB#l>egIaQ@NU?9Dvhi)>C@67QEYH7wU;!N6}0&J9K`MXTPXl z9N%h-!D)tfS;|6!`59lUtI;pey;XoFEToN=boVP=Z$l}6256Q_rA(DoD@Uj@VL|G4 zaqR;07>H{hC7Ld|g>}YP#{fjgg7|sd@4qYjurIBA(&8~^{18!R@YaGyrVFIwIgO-# z|25UmO;0vfEG43;VcX4imnE_@R}3IhGMP}X^)AkI>{vWC6%vuFpdn)J!R#kBMAgsI z0vz#Et~nQagyDD5Z?&UvO{uU~+aKL5mL+s3@h2&k9R*tg0y>FW21dd+f zHtYx1Y@7R=poC@6#GI+D4srn6FXVEKaaINYrFaRwZ*ec`I27NmR`Q{>_@{KvQbP2U zT>Cw~TS%!zEva>p*&+OU&cY8`GH+N!jE+>wzhvAU8j|-MKeJrllO!JbGPrzJKyC_4&-+k2p3RRiWal)xTL-Rtsi7_z1k=t6K3u2vtDm*F!j=T!B1#@f13$5 zoZIU!*L<8NqcM5o0ObVQ_!U6C)^B^r}B^MD3vtY(Q3Sa%tR5uAPB#SVh&t z@Xm;~#>mL%?zrd?*C;tZdND^NZ;T_zP}(Acd}5*D{yScqslMxiS%u3H&A_Un+D;U&GRt@g=0^8uem4H$gjcNsr}$ z#&<_Nw!wYU>((r|s|5e*s4|_zCVPu!93(X5R1LXFAhe7uC(!6eU8cz6=TyJ$0BmoG61T6C9d{7X*#h z1R9~`V3+thAzCWy>>9j$q0_L!-FVNeWj0Cage7v-xwVo(8T%h%d>t1TDb7dvMN6bL zyqbu$TfP|eHFdM%+go;nu?*N=I<=Hrb5?>lY++hcH??pC*$2vuk64WAebjjrzC;_2 zS`oPNCV4Q|(nwz;`ztF!BppEQgO^y|$@Ynq|wY2bNk zw9a%SHlvUGNKLp?U_lRCWh=_Tp3_jUaB)Y^L$fSB;6`#z7ORNEWJf9hbkUDNz4S)! zjPs`d@w(>g0yC}HNT#m%+X?bEsyY{YhfUBZ#oAW;yqO2@N<8?>hShmqy~qN?(RsZw zba3rEM$PI*ddv#WW&xk1?Dvk>8;k@#);yO2u+QFDMbWFH`B7hG`<=64@*WAi^ZgOd zROL8vsYSa=HMutvoiezy3>smTK^}wJ>93w>cFBU^OC=JpJoD~l#9}QGdun<9PFv^$VE~EZO74_Goqs+{XQ>#zPqAdR=e?*&>-| zAjpcEE7^&g1+AAvnqSs z#!DSBPVo<{;2FLvg|oCjlZSOBRtu3jdV8rw6ilv{Uu9S@yI?+1s?srVTAL0ZiIjcN z=F@CjT@;yytxAdR_ZI)o^BX@a+mo%E;LIMtx6ik_U8eYCEs z-qo&yYygZ@X8J~MsV%PkPbokb!II>ux|re>CfdclI2y}lQcS_XOeYGvqI7Gifa{3+ zo*Sl8ps#4hZl%jy;C0rS3-&_yw<*cUkitH`A_GE|Wo{&Q_a`y*dE3z94ywz>5Em3b zml>$3oT>AB=F_9lf+_d(eE8lty_fY+@dCH&Es{4asI1qg)KG`08`e09GfsBab@TQ%==Q2u{t~8kXz<<5|Ds+WiRJ3D^Am`yT#zB`XnIIw0=$mKIb{~3SsNZdZp7^DFBKuLBMnUR z=o5=(-Y3}^oH(b+B;|MLDL5ymt>*tmW{_Q+{1J&q7bekZ%Tbl+8^u6`&1WRX^;M>v zo{U9AA(3+uW9}IDV99!knf(j{4a+{NxtQmra+kca#GR&VBB$LJmaE%6fz5j3+Elu< zp6Wt3fli$(bEoo7(z6N1a=h~+%B=WuL;~exiRFnYy|3&yIYA@W@w$s;i2fk)8cFrM ziT4%SOhkqAeBNg49W9|aY^Qp%1I>tDgPw}hAiyjETc*H$sf$U!%^=kClgQx$q4f*%^0m!XOyg^=`VATa&M;1*aEaAS*7(dNZDlsF-MCC1fTh2LPDatrV}6K4^1DPjH@x2%7kS z23EnPogb=mIW70x%v_v@;R0x`--+4pRPQlK_7!&+;}xM3x)Its#!jK>I8TDAw}9wFH?Cb^gDEkW^iD@(I` z)E(^GfvXi~iSFrQuz>2dj=n}@OiVYZvLLO_vo#*&uvV|NM>i#|Te@1Bbys>?a5T7z zgp-w?J_||K`54V5Lh0zF#>G`t^{5dQZlv0Ax2@6%aaG+({2&zV@I7WW)jkeyXy^6W zaIxC{?Y)K5D2rgfh#q86OYHGopS?MbLMhLAh3pf8V7wPOeZH)cawH8Veq!|q8(VWG1IURMRR-^kO9iZxzHQu-76V4W7@brb6*p59 z`&=`P1Rjh^fxf*?gLG9)FPw3T+IQGVLw!heCkJ|(!vHc@TmDo;oUizdN>pc%3(e<7 z$Y#Hgo284=P8k&18bvDrhZ5em^(*zcdh-qt zaB?33VW)Al*QhqtujXRPSO2W<&UT4v+Ls2t2Y2`|zmZ+j`Q{R)1@KAR8a-ADCB5FU zT%CE8`#<=fNy13e2cf5`zP^#@9`pd3&bihiCG9`5`(aXJ<_@ z(V+*7+|6cxDn5$nbpIDlRw%YUJA0|Lw~nSwGo?4tvYiYC$L8EIgrHUmKtY|VzAn8B zRoyevQZv9?(XpH^t)fE;TG}jjo>4JgtFnhuNT=ITQcW-F&7dqPYUy?LN12~-%uwIx zPlnIBUC*x{u-wR(SijDHkSHiilXR7~)<3K5vha|72XJ82mr-QYPxcP7j|(rw4#rVy z)89Vb8+ycyJStE%S+L2=Ud8|(=VG?vyGuCjVi-gLfKB9cR$3}q)DGOV0J0b(sp1Wn z?6;A{8|lRU>D;RK$RSeH>;}Hf2BG=f1DVu+?m`o&1rl#gc;D zPJbg0#7dEZiUvYk9@*)iFo$4lRQanzPkFt-I-$ol;}z;#PUGqun;QZmbDboXAAt(; z@^NTgOo~}DtS7r|3e_@ZVOWR!j0t=uv5k9#=pdd z`PV!WE^l033FD>qaJU1F{w~dYH?t#QvF09D<;k&c`xd|EI1sw*q^_b8#hNfezMN3+ zjB{LI3Kzz#gZAzMrC6oc;u#NqEwLv~9?o zmAR~Kw4m`dc&(l8!|`>y>!#Hv7`j#a4wP?kZoI6Oyy&MC35&j)sT47$4BWOA9n3L} zjUV;}ktg_P&SllaGEbG04quiY2@TyWyiX1-6f*3IyB#+uKL*LTPY z%ErTGO>v=P&UwZax__!MxXI2({ki;}?c)-=|92zx+pmv*`#Z9KBJzLivdzmsA*nS? zD=|$Vp96s9QXS_P4k)LE9g&{ZS)d9!5xWO@41%?Y@)&6Tu#z|e1_P%CwEDl^|x$1TFBn(e?GpIeE>}uUh%20t@&vSFWraikm zNP2uz!btHu^&2vdxr?$tiytk=M83EF5UGT?sxbWI3_|W9S*-u9FFQF`kOFK+k>^=X zjqZF`RsJN(Pj+7TKacoDC|m0DH6JhKmsPhr(FZjcO7J|ZQ9GXW-8?wQe5IFw&3sel ziy!Nwl-_F|{#Vv7(m6|&A)h2?g?L|(Q9t=FOVNMr0^K5Ci=^f$ z(j)2cEJ_;r$J4Iwu%pd06F_F`($2j{KmGyfFd*_DFfc|a8J^SRM=P+0t^^2wzW1V# z?u2@~YiwOYC+1Uj%9Y%sz=Migx8}|tUbnUyzQ+=*SEI7w!)m@!-)q2?obM_V=fww; z3o4;eyGA2dxE$HfOC#YlUb?{`kz>tou+4E$0q~y9A3YEPc)qe1sQVpKqQeKJkY1y0 z=x<~tX{RPG1^V=FkJA%L39L$FH$PP)6u_Zt_DU(t{DvJxr-hKCr#C-bIN11GtB7Ac zFMK0A`SG__)_?Cv_VYp|!Dq5VCv4XE<^9uVutnx2QuAQkQ1~2Re)8Nq*xod8(b$jz6U^Y{ z&bKXP!y#|9(4{qFh`m?&bndRGc`6*5G}XbbjFwG5T15p)b(Vn$sfTG(>SL~SWVXWnCtok zSD_B7s>s({bH0WST9v0{c-iYx2Xki@hH9!{{5B1(99WY}7Dj&~GZg(bU-jC- zXU}^l`;5`^IL9U3p872E+d)Y#D3Gg7On)Lo+(sy>!wU<*`vH?sg3JX!YOcykMnt;H zZXaB}k9CiO?ZmTntiu$dlMK(x)niHaEC#B2Vz0bmBu^%R zItl@&X;z!%c3eYL-v`DbaZwP%jzl}*@#5S?)H!Yf7-QJqLN|s%Ai3T6=q#b$Bem#lLL+@;V_q0&o`s)3!ez#G-w`~gfC(Z zT6MmFR3H`>cef}R@`jZ?8L8E$O;+Yb^Rig>IFlMpmbA}5it3`%tHjks7G|DYkjYztPGBMN_nM!w=l1j){EAWz($tqUyRxA9PBgZrAiW7uIhS{t!k!U(Bj%BZQu?_P=~-;W~2=j~w+@A;HXl6v$@X!=~|3YfIX# z%|{!vY?4E%W0*~49eWpQy-S|;1VRUGF5ZwzG`7KWCvT1W&LIrrlsrDuLMXS276|LZ z*zbm%`7?QZ~nk zYmDfV`vsTuXENL~)VVxvgRV@VxL%$E$J$rhBAbm@+H9IS6>+iLM#EsJ5RIx6Z$XBu zs*RjPqDl73QZ~@L9Ci$XXnppkeviwkv3ChD6=>d=f@`E9mAyPt_yx6Dsthxvg$$(J zikS24GRuSVW~{RlValB>qVWs-=eUp+uB{0rTRa`<(bjQ9?wii=a<%10)mldjKSb5#27@z zcvCor3;0zf$3O^rEuu?N?C1mJ5c%S^3Q4jz&w*ngvtrc}J0PsvFw_Eu5-TVwy59bn zo+*A<@X(+@?6_|bC_*=^IW;QkW6#sEe7sPb<|BIkMBgUx zRPCl!Er$xkeU9f1J*@BXQ+eVv95827gfo6$+Z!jOCQmW}b}1w8OE)ubs2Yj*Rv!Fh zj(>c}xXdBRL`O6AzKHgM>+NEv2j7X`>hm8wYEK-N*8r~B@}(Q^wuAQox>8`G5M>u%$S|o3F zFvHPWe(*isO+(}rDr?evQnaYVF!a){`JWCr<_Z2pH9rcg@`ZbyM%e{S}tr?t9#c+CuS%)L>qET`G8m6uI|PtbYV)cA_qFW*ls%u|!m zI##UG)qS3)!ZTr^5p~He3w7|q06Nm5T-sk2OLl*}5-S;sYfX#o)43r_Fz}S4$n+$% znTbj1eWahuft7*M1}Pb5%Vud`5~1Hx9Ozh)ibq>W1!W>;qyZ)np;^CMbdU&mXO<)pUZ0bKn8*x#>l8nvKAVW*Q}RbeWCx^qC${7uy>goy4O_4trI+K2$ty)U%|2 z3EW}ogSZL*+Bvw4KXtr5B{b5)YlZG%5S0zdi7C52Xj6XdU@@quyIkm{P@V&7AXi^A zEZa<5wOZ@P_MluH}T6I?a%+N^aPbbNpq?6F$y zQv1Xg*cv+>Nq*X2Zm|;Yl)%ULhGZ)X&eRPTx}sa?WFM!mJ}@QG*+q1TEsb`E(ZtdY zW&ezdXFE+jf9oP3Pnk0`){#mz?YJ4?fq{xeQuykFC5?4s;K)@PPo@#_TocUv+#TM4 z;h!y~uEvMj2ZzNQnCnB&cHGr^F_5x$eH$>s(k`W&_I@pz#T>GV^_i+hs4(`up(vPT_(As3D8;LVgOAx(`yX@EQo;ZUM&FKhVw+$gZYSlS~ zP-yU)F?;2DRLZ5-6&9>4_t9c#ve;5WayjbHQto%js{q{QmzernUIoAbzUrxJfVa5W zU%_9YV_D0t-Y_1tI{e%+X%ce*K z;-NLDn`v(T&eXiQvwJ+(HrnuM9u9$CJM+@G{ub9GIb3U9LX=B-YEQ^9d10mVrc(4> zJ_I)*&m+!;ru1=mGWFvOu? zeD!B6!?!RI0DztSYq#n@j52?ivOx#0yEYoWHo9t%g45DfM$Ia6r$c!YJiquXdDs@xxd~Y&;MbRZ` z2Vviw8ha!Aw;t;b+35pVwf&EPZ<=-s;>AD6x*|hbR`kRv;eIh`^`TG9b3Dq#^xxip z61a)!h!ty5uvz36W6fNMBIU#3b8H1yY8ZK7(qfsEjc2DEmGfjPEZ{Qj3nkDR6YYFs z1O@+KzNKY}MW;Ormk7}M2kV~?;OEgpOUag*Km^-LBCH1emYtxahPNS2h^sMrIp6gdl`5@&RIg`;6+sR;ubK ztD1Gb$cw*9a}l@48%D6Hdr-B(zxI+#U8*hL?mPmwr@r!~>e9pR$o!Q&{)U=UQ!5>- zA}t_<505BoPGe$&cPv_gbe|8sD#rhQqq|4N;PlSwG=u@H)M2#|XQ?=Jy9~0RhI4W^ zjqpW7i;k1)?0`2jHeC6U*x}nmydy&?IsYXW)wAX`OqmsH#IU_xIIGy9>F^EWHx~c9 z&mu6Tb6u|&h+~pVFEBmRrP{ijW@D7zbwF~$B80#hPd{}9F!YfhGxF&9F-U`F_aQ7M z((FmMiQRy0d_bdX6XRx2u>DmD-_@8Y)W_;5u=o+pQY$!@>p?J&%g1x;LGEXi18>UX%(w&!eW)^(s zv%XpS7nFbF>8~BlZ#?~P@w5%_-3GwOToA@sm-J2=SdYMQx2?a}YTisVNQ8)}MGWmr zn>DxspS)s%wGoKHPEI7cjhfewco~>3CdK=gMA+`Tdw!E6JLX?2;qsSmOSG#G41DCS z5t_I!83^8jX1}JSg67G!pPsd}wGvsao_9Gs9sk}|s?Va~2~_U8IhP=+Tg?#6dop$> zm9-l_an)0b1|kdi?jKal-jaq&?`aj2l@$un;u*3s1_P zViX{3fJoUPFk_VyO~Wy5gGoqfox<9>jo%TlI`ef~-ni6_BkJnLT46I<0neF6CwlLT zu|6G44y49m>7@;6tO+-(N^29}y$#M>F^6hK0r}tb5LF zIe_L1R(_>>#td4v$gB&wc}nHNG~}E%k9eDWxGKCM_bvVUnvVcivks2&0uz(j53bd)i4W^n9`$c-5?Q!Y>UJuD9S6k-W#z<_NTDei^d*#This tool has only been tested on [Pji's](https://www.pji.net.cn/) Electronic Control Unit of RK3588. Other RK3588 development boards require independent testing. +3. Set the path of `boot.img` in **boot** and connect the RK3588 board. +4. Press the `Run` button to flash the image to the RK3588 board. + +![RKDevTool](./figures/RKDevTool3.3.png) \ No newline at end of file diff --git a/modules/axhal/build.rs b/modules/axhal/build.rs index 56061e6434..f2804abd18 100644 --- a/modules/axhal/build.rs +++ b/modules/axhal/build.rs @@ -5,6 +5,7 @@ const BUILTIN_PLATFORMS: &[&str] = &[ "aarch64-bsta1000b", "aarch64-qemu-virt", "aarch64-raspi4", + "aarch64-rk3588j", "riscv64-qemu-virt", "x86_64-pc-oslab", "x86_64-qemu-q35", @@ -15,6 +16,7 @@ const BUILTIN_PLATFORM_FAMILIES: &[&str] = &[ "aarch64-phytium-pi", "aarch64-qemu-virt", "aarch64-raspi", + "aarch64-rk3588j", "riscv64-qemu-virt", "x86-pc", ]; diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index e8dbab6f72..9f9039ae3a 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -276,6 +276,7 @@ unsafe extern "C" fn _start() -> ! { // PC = 0x8_0000 // X0 = dtb core::arch::asm!(" + mov x20, x0 // save DTB pointer // disable cache and MMU mrs x1, sctlr_el2 bic x1, x1, #0xf @@ -289,13 +290,12 @@ unsafe extern "C" fn _start() -> ! { mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id - mov x20, x0 // save DTB pointer adrp x8, {boot_stack} // setup boot stack add x8, x8, {boot_stack_size} mov sp, x8 - bl {switch_to_el2} // switch to EL1 + bl {switch_to_el2} // switch to EL2 bl {enable_fp} // enable fp/neon bl {init_boot_page_table} bl {init_mmu_el2} diff --git a/modules/axhal/src/platform/aarch64_common/mod.rs b/modules/axhal/src/platform/aarch64_common/mod.rs index c585541fe8..3cc33611fb 100644 --- a/modules/axhal/src/platform/aarch64_common/mod.rs +++ b/modules/axhal/src/platform/aarch64_common/mod.rs @@ -7,5 +7,8 @@ pub mod psci; #[cfg(feature = "irq")] pub mod gic; -#[cfg(not(platform_family = "aarch64-bsta1000b"))] +#[cfg(not(any( + platform_family = "aarch64-bsta1000b", + platform_family = "aarch64-rk3588j" +)))] pub mod pl011; diff --git a/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs new file mode 100644 index 0000000000..6d9de293be --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs @@ -0,0 +1,147 @@ +//! Definitions for snps,dw-apb-uart serial driver. +//! Uart snps,dw-apb-uart driver in Rust for BST A1000b FADA board. +use crate::mem::phys_to_virt; +use kspin::SpinNoIrq; +use memory_addr::PhysAddr; + +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_structs, + registers::{ReadOnly, ReadWrite}, +}; + +register_structs! { + DW8250Regs { + /// Get or Put Register. + (0x00 => rbr: ReadWrite), + (0x04 => ier: ReadWrite), + (0x08 => fcr: ReadWrite), + (0x0c => lcr: ReadWrite), + (0x10 => mcr: ReadWrite), + (0x14 => lsr: ReadOnly), + (0x18 => msr: ReadOnly), + (0x1c => scr: ReadWrite), + (0x20 => lpdll: ReadWrite), + (0x24 => _reserved0), + /// Uart Status Register. + (0x7c => usr: ReadOnly), + (0x80 => _reserved1), + (0xc0 => dlf: ReadWrite), + (0xc4 => @END), + } +} + +/// dw-apb-uart serial driver: DW8250 +pub struct DW8250 { + base_vaddr: usize, +} + +impl DW8250 { + /// New a DW8250 + pub const fn new(base_vaddr: usize) -> Self { + Self { base_vaddr } + } + + const fn regs(&self) -> &DW8250Regs { + unsafe { &*(self.base_vaddr as *const _) } + } + + /// DW8250 initialize + pub fn init(&mut self) { + const UART_SRC_CLK: u32 = 25000000; + const BST_UART_DLF_LEN: u32 = 6; + const BAUDRATE: u32 = 115200; + //const BAUDRATE: u32 = 38400; + + let get_baud_divider = |baudrate| (UART_SRC_CLK << (BST_UART_DLF_LEN - 4)) / baudrate; + let divider = get_baud_divider(BAUDRATE); + + // Waiting to be no USR_BUSY. + while self.regs().usr.get() & 0b1 != 0 {} + + // bst_serial_hw_init_clk_rst + + /* Disable interrupts and Enable FIFOs */ + self.regs().ier.set(0); + self.regs().fcr.set(1); + + /* Disable flow ctrl */ + self.regs().mcr.set(0); + + /* Clear MCR_RTS */ + self.regs().mcr.set(self.regs().mcr.get() | (1 << 1)); + + /* Enable access DLL & DLH. Set LCR_DLAB */ + self.regs().lcr.set(self.regs().lcr.get() | (1 << 7)); + + /* Set baud rate. Set DLL, DLH, DLF */ + self.regs().rbr.set((divider >> BST_UART_DLF_LEN) & 0xff); + self.regs() + .ier + .set((divider >> (BST_UART_DLF_LEN + 8)) & 0xff); + self.regs().dlf.set(divider & ((1 << BST_UART_DLF_LEN) - 1)); + + /* Clear DLAB bit */ + self.regs().lcr.set(self.regs().lcr.get() & !(1 << 7)); + + /* Set data length to 8 bit, 1 stop bit, no parity. Set LCR_WLS1 | LCR_WLS0 */ + self.regs().lcr.set(self.regs().lcr.get() | 0b11); + } + + /// DW8250 serial output + pub fn putchar(&mut self, c: u8) { + // Check LSR_TEMT + // Wait for last character to go. + while self.regs().lsr.get() & (1 << 6) == 0 {} + self.regs().rbr.set(c as u32); + } + + /// DW8250 serial input + pub fn getchar(&mut self) -> Option { + // Check LSR_DR + // Wait for a character to arrive. + if self.regs().lsr.get() & 0b1 != 0 { + Some((self.regs().rbr.get() & 0xff) as u8) + } else { + None + } + } + + /// DW8250 serial interrupt enable or disable + pub fn set_ier(&mut self, enable: bool) { + if enable { + // Enable interrupts + self.regs().ier.set(1); + } else { + // Disable interrupts + self.regs().ier.set(0); + } + } +} + +const UART_BASE: PhysAddr = pa!(axconfig::UART_PADDR); + +pub static UART: SpinNoIrq = + SpinNoIrq::new(DW8250::new(phys_to_virt(UART_BASE).as_usize())); + +/// Writes a byte to the console. +pub fn putchar(c: u8) { + let mut uart = UART.lock(); + match c { + b'\r' | b'\n' => { + uart.putchar(b'\r'); + uart.putchar(b'\n'); + } + c => uart.putchar(c), + } +} + +/// Reads a byte from the console, or returns [`None`] if no input is available. +pub fn getchar() -> Option { + UART.lock().getchar() +} + +/// UART simply initialize +pub fn init_early() { + UART.lock().init(); +} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs new file mode 100644 index 0000000000..d7cf3415ee --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs @@ -0,0 +1,68 @@ +use crate::mem::{MemRegion, MemRegionFlags, PhysAddr}; +use page_table_entry::{aarch64::A64PTE, GenericPTE, MappingFlags}; + +/// Returns (rk3588j only) memory regions. +pub(crate) fn default_rk3588j_regions() -> impl Iterator { + [ + MemRegion { + paddr: PhysAddr::from(0x9400000), + size: 0xe6c00000, + flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "free memory", + }, + MemRegion { + paddr: PhysAddr::from(0x1f0000000), + size: 0x10000000, + flags: MemRegionFlags::FREE | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "free memory", + }, + ] + .into_iter() +} + +/// Returns platform-specific memory regions. +pub(crate) fn platform_regions() -> impl Iterator { + crate::mem::default_free_regions() + .chain(default_rk3588j_regions()) + .chain(crate::mem::default_mmio_regions()) +} + +pub(crate) unsafe fn init_boot_page_table( + boot_pt_l0: *mut [A64PTE; 512], + boot_pt_l1: *mut [A64PTE; 512], +) { + let boot_pt_l0 = &mut *boot_pt_l0; + let boot_pt_l1 = &mut *boot_pt_l1; + boot_pt_l0[0] = A64PTE::new_table(PhysAddr::from(boot_pt_l1.as_ptr() as usize)); + + boot_pt_l1[0] = A64PTE::new_page( + PhysAddr::from(0), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[1] = A64PTE::new_page( + PhysAddr::from(0x4000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[2] = A64PTE::new_page( + PhysAddr::from(0x8000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[3] = A64PTE::new_page( + PhysAddr::from(0xC000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::DEVICE, + true, + ); + boot_pt_l1[7] = A64PTE::new_page( + PhysAddr::from(0x1_C000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[8] = A64PTE::new_page( + PhysAddr::from(0x1_F000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); +} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mod.rs b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs new file mode 100644 index 0000000000..c59047bd78 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs @@ -0,0 +1,66 @@ +pub mod dw_apb_uart; +pub mod mem; + +#[cfg(feature = "smp")] +pub mod mp; + +#[cfg(feature = "irq")] +pub mod irq { + pub use crate::platform::aarch64_common::gic::*; +} + +pub mod console { + pub use super::dw_apb_uart::*; +} + +pub mod time { + pub use crate::platform::aarch64_common::generic_timer::*; +} + +pub mod misc { + pub use crate::platform::aarch64_common::psci::system_off as terminate; +} + +extern "C" { + fn exception_vector_base(); + fn rust_main(cpu_id: usize, dtb: usize); + #[cfg(feature = "smp")] + fn rust_main_secondary(cpu_id: usize); +} + +pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { + crate::mem::clear_bss(); + crate::arch::set_exception_vector_base(exception_vector_base as usize); + #[cfg(not(feature = "hv"))] + crate::arch::write_page_table_root0(0.into()); // disable low address access + crate::cpu::init_primary(cpu_id); + super::dw_apb_uart::init_early(); + super::aarch64_common::generic_timer::init_early(); + rust_main(cpu_id, dtb); +} + +#[cfg(feature = "smp")] +#[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. +pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { + crate::arch::set_exception_vector_base(exception_vector_base as usize); + crate::arch::write_page_table_root0(0.into()); // disable low address access + crate::cpu::init_secondary(cpu_id); + rust_main_secondary(cpu_id); +} + +/// Initializes the platform devices for the primary CPU. +/// +/// For example, the interrupt controller and the timer. +pub fn platform_init() { + #[cfg(feature = "irq")] + super::aarch64_common::gic::init_primary(); + super::aarch64_common::generic_timer::init_percpu(); +} + +/// Initializes the platform devices for secondary CPUs. +#[cfg(feature = "smp")] +pub fn platform_init_secondary() { + #[cfg(feature = "irq")] + super::aarch64_common::gic::init_secondary(); + super::aarch64_common::generic_timer::init_percpu(); +} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mp.rs b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs new file mode 100644 index 0000000000..01f84fc4c1 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs @@ -0,0 +1,20 @@ +use crate::mem::{virt_to_phys, PhysAddr, VirtAddr}; + +/// Hart number of rk3588 board +pub const MAX_HARTS: usize = 8; +/// CPU HWID from cpu device tree nodes with "reg" property +pub const CPU_HWID: [usize; MAX_HARTS] = [0x00, 0x100, 0x200, 0x300, 0x400, 0x500, 0x600, 0x700]; + +/// Starts the given secondary CPU with its boot stack. +pub fn start_secondary_cpu(cpu_id: usize, stack_top: PhysAddr) { + assert!(cpu_id < MAX_HARTS, "No support for rk3588 core {}", cpu_id); + extern "C" { + fn _start_secondary(); + } + let entry = virt_to_phys(VirtAddr::from(_start_secondary as usize)); + crate::platform::aarch64_common::psci::cpu_on( + CPU_HWID[cpu_id], + entry.as_usize(), + stack_top.as_usize(), + ); +} diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 349aac9ae0..57431a0e05 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -25,6 +25,9 @@ cfg_if::cfg_if! { } else if #[cfg(all(target_arch = "aarch64", platform_family = "aarch64-phytium-pi"))] { mod aarch64_phytium_pi; pub use self::aarch64_phytium_pi::*; + } else if #[cfg(all(target_arch = "aarch64", platform_family = "aarch64-rk3588j"))] { + mod aarch64_rk3588j; + pub use self::aarch64_rk3588j::*; } else { mod dummy; pub use self::dummy::*; diff --git a/platforms/aarch64-rk3588j.toml b/platforms/aarch64-rk3588j.toml new file mode 100644 index 0000000000..61d0793234 --- /dev/null +++ b/platforms/aarch64-rk3588j.toml @@ -0,0 +1,68 @@ +# Architecture identifier. +arch = "aarch64" +# Platform identifier. +platform = "aarch64-rk3588j" +# Platform family. +family = "aarch64-rk3588j" + +# Base address of the whole physical memory. +phys-memory-base = "0x20_0000" +# Size of the whole physical memory. +phys-memory-size = "0x800_0000" # 128M +# Base physical address of the kernel image. +kernel-base-paddr = "0x48_0000" +# Base virtual address of the kernel image. +kernel-base-vaddr = "0x0000_0000_0048_0000" +# Linear mapping offset, for quick conversions between physical and virtual +# addresses. +phys-virt-offset = "0xffff_0000_0000_0000" +# Kernel address space base. +kernel-aspace-base = "0xffff_0000_0000_0000" +# Kernel address space size. +kernel-aspace-size = "0x0000_ffff_ffff_f000" +# MMIO regions with format (`base_paddr`, `size`). +mmio-regions = [ + # ["0x0900_0000", "0x1000"], # PL011 UART + # ["0x0800_0000", "0x5_0000"], # GICv2 with Virtualization (GICV@0x0803_0000, GICH@0x0804_0000) + # ["0x0a00_0000", "0x4000"], # VirtIO + # ["0x1000_0000", "0x2eff_0000"], # PCI memory ranges (ranges 1: 32-bit MMIO space) + # ["0x40_1000_0000", "0x1000_0000"], # PCI config space + ["0xfeb50000", "0x1000"], # uart8250 UART0 + ["0xfe600000", "0x10000"], # gic-v3 gicd + ["0xfe680000", "0x100000"], # gic-v3 gicr + ["0xa41000000", "0x400000"], + ["0xa40c00000", "0x400000"], + ["0xf4000000","0x1000000"], + ["0xf3000000","0x1000000"], +] +# VirtIO MMIO regions with format (`base_paddr`, `size`). +virtio-mmio-regions = [] + +# Base physical address of the PCIe ECAM space. +pci-ecam-base = "0xf4000000" +# End PCI bus number (`bus-range` property in device tree). +pci-bus-end = "0xff" +# PCI device memory ranges (`ranges` property in device tree). +pci-ranges = [] +# UART Address +uart-paddr = "0xfeb5_0000" +uart-irq = "0x14d" + +# GICC Address +gicd-paddr = "0xfe600000" +# GICR Address +gicc-paddr = "0xfe680000" +gicr-paddr = "0xfe680000" + +# PSCI +psci-method = "smc" + +# pl031@9010000 { +# clock-names = "apb_pclk"; +# clocks = <0x8000>; +# interrupts = <0x00 0x02 0x04>; +# reg = <0x00 0x9010000 0x00 0x1000>; +# compatible = "arm,pl031\0arm,primecell"; +# }; +# RTC (PL031) Address +# rtc-paddr = "0x901_0000" \ No newline at end of file diff --git a/scripts/make/rk3588.mk b/scripts/make/rk3588.mk new file mode 100644 index 0000000000..49c0a44434 --- /dev/null +++ b/scripts/make/rk3588.mk @@ -0,0 +1,13 @@ +RK3588_GITHUB_URL = https://github.com/arceos-hypervisor/platform_tools/releases/download/latest/rk3588.zip +RK3588_MKIMG_FILE = ./tools/rk3588/mkimg +check-download: +ifeq ("$(wildcard $(RK3588_MKIMG_FILE))","") + @echo "file not found, downloading from $(RK3588_GITHUB_URL)..."; + wget $(RK3588_GITHUB_URL); + unzip -o rk3588.zip -d tools; + rm rk3588.zip; +endif + +kernel: check-download build + $(RK3588_MKIMG_FILE) --dtb rk3588-firefly-itx-3588j.dtb --img $(OUT_BIN) + @echo 'Built the FIT-uImage boot.img' From f95d7e733ad29e714455ee70c965ae09e8096aea Mon Sep 17 00:00:00 2001 From: hky1999 Date: Wed, 23 Oct 2024 16:19:02 +0800 Subject: [PATCH 03/48] [refactor] aarch64 el2 support (#31) --- .../axhal/src/platform/aarch64_common/boot.rs | 151 ------------- .../src/platform/aarch64_common/boot_el2.rs | 198 ++++++++++++++++++ .../axhal/src/platform/aarch64_common/mod.rs | 4 + 3 files changed, 202 insertions(+), 151 deletions(-) create mode 100644 modules/axhal/src/platform/aarch64_common/boot_el2.rs diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index 9f9039ae3a..1140c17b67 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -1,8 +1,5 @@ use aarch64_cpu::{asm, asm::barrier, registers::*}; use core::ptr::addr_of_mut; -use memory_addr::PhysAddr; -//Todo: remove this, when hv is enabled, `MemAttr` is not used. -#[cfg_attr(feature = "hv", allow(unused_imports))] use page_table_entry::aarch64::{MemAttr, A64PTE}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; @@ -17,7 +14,6 @@ static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; #[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; -#[cfg(not(feature = "hv"))] unsafe fn switch_to_el1() { SPSel.write(SPSel::SP::ELx); SP_EL0.set(0); @@ -61,7 +57,6 @@ unsafe fn switch_to_el1() { } } -#[cfg(not(feature = "hv"))] unsafe fn init_mmu() { MAIR_EL1.set(MemAttr::MAIR_VALUE); @@ -106,7 +101,6 @@ unsafe fn init_boot_page_table() { } /// The earliest entry point for the primary CPU. -#[cfg(not(feature = "hv"))] #[naked] #[unsafe(no_mangle)] #[unsafe(link_section = ".text.boot")] @@ -147,7 +141,6 @@ unsafe extern "C" fn _start() -> ! { } /// The earliest entry point for the secondary CPUs. -#[cfg(not(feature = "hv"))] #[cfg(feature = "smp")] #[naked] #[unsafe(no_mangle)] @@ -176,147 +169,3 @@ unsafe extern "C" fn _start_secondary() -> ! { entry = sym crate::platform::rust_entry_secondary, ) } - -#[cfg(feature = "hv")] -unsafe fn switch_to_el2() { - SPSel.write(SPSel::SP::ELx); - let current_el = CurrentEL.read(CurrentEL::EL); - - if current_el == 3 { - SCR_EL3.write( - SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64, - ); - SPSR_EL3.write( - SPSR_EL3::M::EL2h - + SPSR_EL3::D::Masked - + SPSR_EL3::A::Masked - + SPSR_EL3::I::Masked - + SPSR_EL3::F::Masked, - ); - ELR_EL3.set(LR.get()); - SP_EL1.set(BOOT_STACK.as_ptr_range().end as u64); - // This should be SP_EL2. To - asm::eret(); - } -} - -#[cfg(feature = "hv")] -unsafe fn init_mmu_el2() { - // Set EL1 to 64bit. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Device-nGnRE memory - let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; - // Normal memory - let attr1 = MAIR_EL2::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc - + MAIR_EL2::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc; - MAIR_EL2.write(attr0 + attr1); // 0xff_04 - - // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 40 bits. - let tcr_flags0 = TCR_EL2::TG0::KiB_4 - + TCR_EL2::SH0::Inner - + TCR_EL2::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL2::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL2::T0SZ.val(16); - TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); - barrier::isb(barrier::SY); - - let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; - TTBR0_EL2.set(root_paddr); - - // Flush the entire TLB - crate::arch::flush_tlb(None); - - // Enable the MMU and turn on I-cache and D-cache - SCTLR_EL2.modify(SCTLR_EL2::M::Enable + SCTLR_EL2::C::Cacheable + SCTLR_EL2::I::Cacheable); - barrier::isb(barrier::SY); -} - -#[cfg(feature = "hv")] -unsafe fn cache_invalidate(cache_level: usize) { - core::arch::asm!( - r#" - msr csselr_el1, {0} - mrs x4, ccsidr_el1 // read cache size id. - and x1, x4, #0x7 - add x1, x1, #0x4 // x1 = cache line size. - ldr x3, =0x7fff - and x2, x3, x4, lsr #13 // x2 = cache set number – 1. - ldr x3, =0x3ff - and x3, x3, x4, lsr #3 // x3 = cache associativity number – 1. - clz w4, w3 // x4 = way position in the cisw instruction. - mov x5, #0 // x5 = way counter way_loop. - // way_loop: - 1: - mov x6, #0 // x6 = set counter set_loop. - // set_loop: - 2: - lsl x7, x5, x4 - orr x7, {0}, x7 // set way. - lsl x8, x6, x1 - orr x7, x7, x8 // set set. - dc cisw, x7 // clean and invalidate cache line. - add x6, x6, #1 // increment set counter. - cmp x6, x2 // last set reached yet? - ble 2b // if not, iterate set_loop, - add x5, x5, #1 // else, next way. - cmp x5, x3 // last way reached yet? - ble 1b // if not, iterate way_loop - "#, - in(reg) cache_level, - options(nostack) - ); -} - -#[cfg(feature = "hv")] -#[naked] -#[no_mangle] -#[link_section = ".text.boot"] -unsafe extern "C" fn _start() -> ! { - // PC = 0x8_0000 - // X0 = dtb - core::arch::asm!(" - mov x20, x0 // save DTB pointer - // disable cache and MMU - mrs x1, sctlr_el2 - bic x1, x1, #0xf - msr sctlr_el2, x1 - - // cache_invalidate(0): clear dl1$ - mov x0, #0 - bl {cache_invalidate} - mov x0, #2 - bl {cache_invalidate} - - mrs x19, mpidr_el1 - and x19, x19, #0xffffff // get current CPU id - - adrp x8, {boot_stack} // setup boot stack - add x8, x8, {boot_stack_size} - mov sp, x8 - - bl {switch_to_el2} // switch to EL2 - bl {enable_fp} // enable fp/neon - bl {init_boot_page_table} - bl {init_mmu_el2} - - mov x8, {phys_virt_offset} // set SP to the high address - add sp, sp, x8 - - mov x0, x19 // call rust_entry(cpu_id, dtb) - mov x1, x20 - ldr x8, ={entry} - blr x8 - b .", - cache_invalidate = sym cache_invalidate, - init_boot_page_table = sym init_boot_page_table, - init_mmu_el2 = sym init_mmu_el2, - switch_to_el2 = sym switch_to_el2, - enable_fp = sym enable_fp, - boot_stack = sym BOOT_STACK, - boot_stack_size = const TASK_STACK_SIZE, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry, - options(noreturn), - ); -} diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs new file mode 100644 index 0000000000..d8b6bc0cf1 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -0,0 +1,198 @@ +use core::ptr::addr_of_mut; + +use aarch64_cpu::{asm, asm::barrier, registers::*}; +use memory_addr::PhysAddr; +use page_table_entry::aarch64::A64PTE; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +use axconfig::TASK_STACK_SIZE; + +#[link_section = ".bss.stack"] +static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; + +#[link_section = ".data.boot_page_table"] +static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; + +#[link_section = ".data.boot_page_table"] +static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; + +unsafe fn switch_to_el2() { + SPSel.write(SPSel::SP::ELx); + let current_el = CurrentEL.read(CurrentEL::EL); + + if current_el == 3 { + SCR_EL3.write( + SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64, + ); + SPSR_EL3.write( + SPSR_EL3::M::EL2h + + SPSR_EL3::D::Masked + + SPSR_EL3::A::Masked + + SPSR_EL3::I::Masked + + SPSR_EL3::F::Masked, + ); + ELR_EL3.set(LR.get()); + asm::eret(); + } +} + +unsafe fn init_mmu_el2() { + // Set EL1 to 64bit. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Device-nGnRE memory + let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; + // Normal memory + let attr1 = MAIR_EL2::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL2::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc; + MAIR_EL2.write(attr0 + attr1); // 0xff_04 + + // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 40 bits. + let tcr_flags0 = TCR_EL2::TG0::KiB_4 + + TCR_EL2::SH0::Inner + + TCR_EL2::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::T0SZ.val(16); + TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); + barrier::isb(barrier::SY); + + let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + TTBR0_EL2.set(root_paddr); + + // Flush the entire TLB + crate::arch::flush_tlb(None); + + // Enable the MMU and turn on I-cache and D-cache + SCTLR_EL2.modify(SCTLR_EL2::M::Enable + SCTLR_EL2::C::Cacheable + SCTLR_EL2::I::Cacheable); + barrier::isb(barrier::SY); +} + +unsafe fn enable_fp() { + if cfg!(feature = "fp_simd") { + CPACR_EL1.write(CPACR_EL1::FPEN::TrapNothing); + barrier::isb(barrier::SY); + } +} + +unsafe fn init_boot_page_table() { + crate::platform::mem::init_boot_page_table(addr_of_mut!(BOOT_PT_L0), addr_of_mut!(BOOT_PT_L1)); +} + +unsafe fn cache_invalidate(cache_level: usize) { + core::arch::asm!( + r#" + msr csselr_el1, {0} + mrs x4, ccsidr_el1 // read cache size id. + and x1, x4, #0x7 + add x1, x1, #0x4 // x1 = cache line size. + ldr x3, =0x7fff + and x2, x3, x4, lsr #13 // x2 = cache set number – 1. + ldr x3, =0x3ff + and x3, x3, x4, lsr #3 // x3 = cache associativity number – 1. + clz w4, w3 // x4 = way position in the cisw instruction. + mov x5, #0 // x5 = way counter way_loop. + // way_loop: + 1: + mov x6, #0 // x6 = set counter set_loop. + // set_loop: + 2: + lsl x7, x5, x4 + orr x7, {0}, x7 // set way. + lsl x8, x6, x1 + orr x7, x7, x8 // set set. + dc cisw, x7 // clean and invalidate cache line. + add x6, x6, #1 // increment set counter. + cmp x6, x2 // last set reached yet? + ble 2b // if not, iterate set_loop, + add x5, x5, #1 // else, next way. + cmp x5, x3 // last way reached yet? + ble 1b // if not, iterate way_loop + "#, + in(reg) cache_level, + options(nostack) + ); +} + +/// The earliest entry point for the primary CPU. +#[naked] +#[no_mangle] +#[link_section = ".text.boot"] +unsafe extern "C" fn _start() -> ! { + // PC = 0x8_0000 + // X0 = dtb + core::arch::asm!(" + // disable cache and MMU + mrs x1, sctlr_el2 + bic x1, x1, #0xf + msr sctlr_el2, x1 + + // cache_invalidate(0): clear dl1$ + mov x0, #0 + bl {cache_invalidate} + mov x0, #2 + bl {cache_invalidate} + + mrs x19, mpidr_el1 + and x19, x19, #0xffffff // get current CPU id + mov x20, x0 // save DTB pointer + + adrp x8, {boot_stack} // setup boot stack + add x8, x8, {boot_stack_size} + mov sp, x8 + + bl {switch_to_el2} // switch to EL2 + bl {enable_fp} // enable fp/neon + bl {init_boot_page_table} + bl {init_mmu_el2} + + mov x8, {phys_virt_offset} // set SP to the high address + add sp, sp, x8 + + mov x0, x19 // call rust_entry(cpu_id, dtb) + mov x1, x20 + ldr x8, ={entry} + blr x8 + b .", + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + options(noreturn), + ); +} + +/// The earliest entry point for the secondary CPUs. +#[cfg(feature = "smp")] +#[naked] +#[no_mangle] +#[link_section = ".text.boot"] +unsafe extern "C" fn _start_secondary() -> ! { + core::arch::asm!(" + mrs x19, mpidr_el1 + and x19, x19, #0xffffff // get current CPU id + + mov sp, x0 + bl {switch_to_el2} + bl {init_mmu_el2} + bl {enable_fp} + + mov x8, {phys_virt_offset} // set SP to the high address + add sp, sp, x8 + + mov x0, x19 // call rust_entry_secondary(cpu_id) + ldr x8, ={entry} + blr x8 + b .", + switch_to_el2 = sym switch_to_el2, + init_mmu_el2 = sym init_mmu_el2, + enable_fp = sym enable_fp, + phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry_secondary, + options(noreturn), + ) +} diff --git a/modules/axhal/src/platform/aarch64_common/mod.rs b/modules/axhal/src/platform/aarch64_common/mod.rs index 3cc33611fb..8a2a450930 100644 --- a/modules/axhal/src/platform/aarch64_common/mod.rs +++ b/modules/axhal/src/platform/aarch64_common/mod.rs @@ -1,4 +1,8 @@ +#[cfg(not(feature = "hv"))] mod boot; +#[cfg(feature = "hv")] +// Todo: maybe we can enter el2 in arm_vcpu? +mod boot_el2; pub mod generic_timer; #[cfg(not(platform_family = "aarch64-raspi"))] From 230e4319efbc66375225fbbad0d1dc90fe7df2bd Mon Sep 17 00:00:00 2001 From: hky1999 Date: Wed, 20 Nov 2024 22:32:30 +0800 Subject: [PATCH 04/48] [feat] support irq handling routine from arceos-umhv (#33) --- Cargo.lock | 3 +- modules/axhal/Cargo.toml | 2 +- modules/axhal/src/irq.rs | 10 +- .../src/platform/aarch64_common/boot_el2.rs | 13 +- .../platform/aarch64_common/generic_timer.rs | 10 +- .../axhal/src/platform/aarch64_common/gic.rs | 153 ++---------------- .../src/platform/aarch64_qemu_virt/mod.rs | 1 + modules/axhal/src/platform/dummy/mod.rs | 5 + 8 files changed, 47 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 272bbf376d..bb680752a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,8 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arm_gicv2?rev=7acc0dd#7acc0dd1e1fefa90b4cd45342072d2350a74b071" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2#dfe5f164b94cdd07081c2fe74a0cfe4bef2852c9" dependencies = [ - "crate_interface", "tock-registers", ] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index d2a5b8d38e..39a01bb433 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -55,7 +55,7 @@ riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.4" tock-registers = "0.8" -arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", rev = "7acc0dd" } +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2" } crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 9498bcb002..2d6b1247c6 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -7,6 +7,9 @@ use crate::trap::{IRQ, register_trap_handler}; pub use crate::platform::irq::{register_handler, set_enable}; +#[cfg(target_arch = "aarch64")] +pub use crate::platform::irq::fetch_irq; + /// The type if an IRQ handler. pub type IrqHandler = handler_table::Handler; @@ -35,8 +38,13 @@ pub(crate) fn register_handler_common(irq_num: usize, handler: IrqHandler) -> bo false } +/// Core IRQ handling routine, registered at `axhal::trap::IRQ`, +/// which dispatches IRQs to registered handlers. +/// +/// Note: this function is denoted as public here because it'll be called by the +/// hypervisor for hypervisor reserved IRQ handling. #[register_trap_handler(IRQ)] -fn handler_irq(irq_num: usize) -> bool { +pub fn handler_irq(irq_num: usize) -> bool { let guard = kernel_guard::NoPreempt::new(); dispatch_irq(irq_num); drop(guard); // rescheduling may occur when preemption is re-enabled. diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index d8b6bc0cf1..fd15e6ac36 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -38,7 +38,18 @@ unsafe fn switch_to_el2() { unsafe fn init_mmu_el2() { // Set EL1 to 64bit. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + // Enable `IMO` and `FMO` to make sure that: + // * Physical IRQ interrupts are taken to EL2; + // * Virtual IRQ interrupts are enabled; + // * Physical FIQ interrupts are taken to EL2; + // * Virtual FIQ interrupts are enabled. + HCR_EL2.modify( + HCR_EL2::VM::Enable + + HCR_EL2::RW::EL1IsAarch64 + + HCR_EL2::IMO::EnableVirtualIRQ // Physical IRQ Routing. + + HCR_EL2::FMO::EnableVirtualFIQ // Physical FIQ Routing. + + HCR_EL2::TSC::EnableTrapEl1SmcToEl2, + ); // Device-nGnRE memory let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index 8af7ffc728..127db32c39 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -101,12 +101,12 @@ pub(crate) fn init_percpu() { } #[cfg(all(feature = "irq", feature = "hv"))] { - // ENABLE, bit [0]， Enables the timer. - let ctl = 1; - let tval = 0; unsafe { - core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) ctl); - core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) tval); + // ENABLE, bit [0], Enables the timer. + // * 0b0: Timer disabled. + // * 0b1: Timer enabled. + core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); + core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); } } #[cfg(feature = "irq")] diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 5fdf37eb2b..dad3172cd3 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -42,13 +42,24 @@ pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { crate::irq::register_handler_common(irq_num, handler) } +/// Fetches the IRQ number. +pub fn fetch_irq() -> usize { + GICC.iar() as usize +} + /// Dispatches the IRQ. /// /// This function is called by the common interrupt handler. It looks /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. -pub fn dispatch_irq(_unused: usize) { - GICC.handle_irq(|irq_num| crate::irq::dispatch_irq_common(irq_num as _)); +pub fn dispatch_irq(irq_no: usize) { + if irq_no == 0 { + GICC.handle_irq(|irq_num| crate::irq::dispatch_irq_common(irq_num as _)); + } else { + crate::irq::dispatch_irq_common(irq_no as _); + GICC.eoi(irq_no as _); + GICC.dir(irq_no as _); + } } /// Initializes GICD, GICC on the primary CPU. @@ -63,141 +74,3 @@ pub(crate) fn init_primary() { pub(crate) fn init_secondary() { GICC.init(); } - -#[cfg(feature = "hv")] -mod gic_if { - - use super::{GICC, GICD}; - use arm_gicv2::GicTrait; - struct GicIfImpl; - - #[crate_interface::impl_interface] - impl GicTrait for GicIfImpl { - /// Sets the enable status of a specific interrupt in the GICD (Distributor). - /// - /// # Parameters - /// - `vector`: The interrupt vector number, identifying the interrupt to be enabled or disabled. - /// - `enable`: A boolean value indicating whether to enable the interrupt. `true` enables the interrupt, `false` disables it. - /// - /// This function locks and accesses the GICD controller, then sets the enable status of the specified interrupt vector based on the `enable` parameter. - /// It provides a mechanism for controlling whether interrupts can trigger CPU responses, used for interrupt management. - fn set_enable(vector: usize, enable: bool) { - GICD.lock().set_enable(vector, enable); - } - - /// Retrieves the enable status of a specified interrupt vector from the GICD. - /// - /// # Parameters - /// - `vector`: The index of the interrupt vector, used to identify a specific interrupt source. - /// - /// # Returns - /// - `bool`: Indicates whether the specified interrupt vector is enabled. `true` means the interrupt vector is enabled, `false` means it is not enabled. - fn get_enable(vector: usize) -> bool { - GICD.lock().get_enable(vector) - } - - /// Get the type of the GICD register - /// - /// This function locks the GICD and calls its internal `get_typer` method to retrieve the type of the GICD register - /// - /// # Returns - /// * `u32` - The type of the GICD register - fn get_typer() -> u32 { - GICD.lock().get_typer() - } - - /// Get the Implementer ID Register (IIDR) of the interrupt controller - /// - /// This function locks the GICD (interrupt controller) and calls its `get_iidr` method to retrieve the value of the Implementer ID Register. - /// This register can be used to identify the specific hardware implementer and version. - fn get_iidr() -> u32 { - GICD.lock().get_iidr() - } - - /// Set the state of an interrupt source - /// - /// This function updates the state of a specific interrupt source in the GICD (Interrupt Controller). - /// It first locks the GICD and then updates the interrupt source state using the provided interrupt ID (`int_id`), - /// new state value (`state`), and current CPU ID (`current_cpu_id`). - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `state`: The new state value for the interrupt source. - /// - `current_cpu_id`: The ID of the current CPU. - fn set_state(int_id: usize, state: usize, current_cpu_id: usize) { - GICD.lock().set_state(int_id, state, current_cpu_id); - } - - /// Get the state of an interrupt source - /// - /// This function retrieves the current state of a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - /// Returns: - /// - The current state value. - fn get_state(int_id: usize) -> usize { - GICD.lock().get_state(int_id) - } - - /// Set the ICFGR (Interrupt Configuration and Control Register) - /// - /// This function sets the configuration of a specific interrupt source in the ICFGR. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `cfg`: The new configuration value. - fn set_icfgr(int_id: usize, cfg: u8) { - GICD.lock().set_icfgr(int_id, cfg); - } - - /// Get the target CPU for an interrupt source - /// - /// This function retrieves the target CPU for a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - /// Returns: - /// - The target CPU ID. - fn get_target_cpu(int_id: usize) -> usize { - GICD.lock().get_target_cpu(int_id) - } - - /// Set the target CPU for an interrupt source - /// - /// This function sets the target CPU for a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `target`: The new target CPU value. - fn set_target_cpu(int_id: usize, target: u8) { - GICD.lock().set_target_cpu(int_id, target); - } - - /// Get the priority of an interrupt source - /// - /// This function retrieves the priority of a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - /// Returns: - /// - The priority value. - fn get_priority(int_id: usize) -> usize { - GICD.lock().get_priority(int_id) - } - - /// Set the priority of an interrupt source - /// - /// This function sets the priority of a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `priority`: The new priority value. - fn set_priority(int_id: usize, priority: u8) { - GICD.lock().set_priority(int_id, priority); - } - } -} diff --git a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs index 1d50d10d0b..4bb16d8450 100644 --- a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs @@ -42,6 +42,7 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { #[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { crate::arch::set_exception_vector_base(exception_vector_base as usize); + #[cfg(not(feature = "hv"))] crate::arch::write_page_table_root0(0.into()); // disable low address access crate::cpu::init_secondary(cpu_id); rust_main_secondary(cpu_id); diff --git a/modules/axhal/src/platform/dummy/mod.rs b/modules/axhal/src/platform/dummy/mod.rs index e0d229ff3a..616ca47e7d 100644 --- a/modules/axhal/src/platform/dummy/mod.rs +++ b/modules/axhal/src/platform/dummy/mod.rs @@ -83,6 +83,11 @@ pub mod irq { /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. pub fn dispatch_irq(irq_num: usize) {} + + /// Fetches the IRQ number. + pub fn fetch_irq() -> usize { + 0 + } } /// Initializes the platform devices for the primary CPU. From 46c3a88e6edd86f06ac0f58080716a0d37346e4c Mon Sep 17 00:00:00 2001 From: hky1999 Date: Tue, 17 Dec 2024 19:51:13 +0800 Subject: [PATCH 05/48] [feat] update allocator version to support alloc_pages_at (#34) * [feat] update allocator version to support alloc_pages_at * [feat] set phys-virt-offset as 0xffff_8000_0000_0000 for x86_64 qemu --- Cargo.toml | 2 ++ modules/axalloc/src/lib.rs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 2924a174af..87e7f85b09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,5 +63,7 @@ axsync = { path = "modules/axsync" } axtask = { path = "modules/axtask" } axdma = { path = "modules/axdma" } +allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } + [profile.release] lto = true diff --git a/modules/axalloc/src/lib.rs b/modules/axalloc/src/lib.rs index ef6f0f6c94..c9a2db6dea 100644 --- a/modules/axalloc/src/lib.rs +++ b/modules/axalloc/src/lib.rs @@ -146,6 +146,24 @@ impl GlobalAllocator { self.palloc.lock().alloc_pages(num_pages, align_pow2) } + /// Allocates contiguous pages starting from the given address. + /// + /// It allocates `num_pages` pages from the page allocator starting from the + /// given address. + /// + /// `align_pow2` must be a power of 2, and the returned region bound will be + /// aligned to it. + pub fn alloc_pages_at( + &self, + start: usize, + num_pages: usize, + align_pow2: usize, + ) -> AllocResult { + self.palloc + .lock() + .alloc_pages_at(start, num_pages, align_pow2) + } + /// Gives back the allocated pages starts from `pos` to the page allocator. /// /// The pages should be allocated by [`alloc_pages`], and `align_pow2` From 1ce8e38555ce834385ccc615da78029cbb9781f0 Mon Sep 17 00:00:00 2001 From: bhxh <32200913+buhenxihuan@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:12:23 +0800 Subject: [PATCH 06/48] adapt hypervisor for a1000b and fix the bug of smp startup on a1000 (#37) * [FIX] Fix the bug of smp startup on a1000 * [feat] adapt hypervisor for a1000b --- .../src/platform/aarch64_bsta1000b/mem.rs | 31 +++++++++++++++++-- .../src/platform/aarch64_bsta1000b/mod.rs | 19 ++++++++++-- .../src/platform/aarch64_common/boot_el2.rs | 4 ++- .../axhal/src/platform/aarch64_rk3588j/mem.rs | 2 +- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs b/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs index 63b0349b91..f122e28f77 100644 --- a/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs +++ b/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs @@ -1,9 +1,21 @@ -use crate::mem::MemRegion; +use crate::mem::{MemRegion, MemRegionFlags}; use page_table_entry::{GenericPTE, MappingFlags, aarch64::A64PTE}; +/// Returns (a1000b only) memory regions. +pub(crate) fn default_a1000b_regions() -> impl Iterator { + [MemRegion { + paddr: pa!(0x80000000), + size: 0x70000000, + flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "reserved memory", + }] + .into_iter() +} /// Returns platform-specific memory regions. pub(crate) fn platform_regions() -> impl Iterator { - crate::mem::default_free_regions().chain(crate::mem::default_mmio_regions()) + crate::mem::default_free_regions() + .chain(default_a1000b_regions()) + .chain(crate::mem::default_mmio_regions()) } pub(crate) unsafe fn init_boot_page_table( @@ -32,4 +44,19 @@ pub(crate) unsafe fn init_boot_page_table( MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, true, ); + boot_pt_l1[3] = A64PTE::new_page( + pa!(0xc0000000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[6] = A64PTE::new_page( + pa!(0x180000000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[7] = A64PTE::new_page( + pa!(0x1C0000000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); } diff --git a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs index cc9cbed108..ad0d0df152 100644 --- a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs +++ b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs @@ -19,7 +19,10 @@ pub mod time { pub use crate::platform::aarch64_common::generic_timer::*; } -unsafe extern "C" { +use crate::mp::CPU_HWID; +use crate::mp::MAX_HARTS; + +extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] @@ -36,7 +39,19 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { } #[cfg(feature = "smp")] -pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { +pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_hwid: usize) { + let mut cpu_id = cpu_hwid; + let mut map_success = false; + for index in 0..MAX_HARTS { + if cpu_id == CPU_HWID[index] { + cpu_id = index; + map_success = true; + break; + } + } + if !map_success { + panic!("CPU{} not found", cpu_id); + } crate::arch::set_exception_vector_base(exception_vector_base as usize); crate::cpu::init_secondary(cpu_id); rust_main_secondary(cpu_id); diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index fd15e6ac36..25045b1920 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -132,6 +132,9 @@ unsafe extern "C" fn _start() -> ! { // PC = 0x8_0000 // X0 = dtb core::arch::asm!(" + // save DTB pointer + mov x20, x0 + // disable cache and MMU mrs x1, sctlr_el2 bic x1, x1, #0xf @@ -145,7 +148,6 @@ unsafe extern "C" fn _start() -> ! { mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id - mov x20, x0 // save DTB pointer adrp x8, {boot_stack} // setup boot stack add x8, x8, {boot_stack_size} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs index d7cf3415ee..4c0a970aba 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs @@ -8,7 +8,7 @@ pub(crate) fn default_rk3588j_regions() -> impl Iterator { paddr: PhysAddr::from(0x9400000), size: 0xe6c00000, flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::WRITE, - name: "free memory", + name: "reserved memory", }, MemRegion { paddr: PhysAddr::from(0x1f0000000), From aa2dacf8038cfdbc1ce98b43f067c9865f08d44a Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 1 Jan 2025 19:34:43 +0800 Subject: [PATCH 07/48] Rebase to arceos-org/arceos main --- .../platforms}/aarch64-rk3588j.toml | 0 configs/platforms/x86_64-qemu-q35.toml | 1 + .../src/platform/aarch64_bsta1000b/mod.rs | 2 +- .../axhal/src/platform/aarch64_common/boot.rs | 2 +- .../src/platform/aarch64_common/boot_el2.rs | 60 ++++++++++--------- .../platform/aarch64_rk3588j/dw_apb_uart.rs | 24 +++++++- .../axhal/src/platform/aarch64_rk3588j/mem.rs | 2 +- .../axhal/src/platform/aarch64_rk3588j/mod.rs | 2 +- .../axhal/src/platform/aarch64_rk3588j/mp.rs | 4 +- 9 files changed, 61 insertions(+), 36 deletions(-) rename {platforms => configs/platforms}/aarch64-rk3588j.toml (100%) diff --git a/platforms/aarch64-rk3588j.toml b/configs/platforms/aarch64-rk3588j.toml similarity index 100% rename from platforms/aarch64-rk3588j.toml rename to configs/platforms/aarch64-rk3588j.toml diff --git a/configs/platforms/x86_64-qemu-q35.toml b/configs/platforms/x86_64-qemu-q35.toml index a44e98cf93..b68a10873a 100644 --- a/configs/platforms/x86_64-qemu-q35.toml +++ b/configs/platforms/x86_64-qemu-q35.toml @@ -40,6 +40,7 @@ mmio-regions = [ [0xfec0_0000, 0x1000], # IO APIC [0xfed0_0000, 0x1000], # HPET [0xfee0_0000, 0x1000], # Local APIC + [0x380000000000, 0x4000] # PCI devices ] # [(uint, uint)] # VirtIO MMIO regions with format (`base_paddr`, `size`). virtio-mmio-regions = [] # [(uint, uint)] diff --git a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs index ad0d0df152..8f1a582580 100644 --- a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs +++ b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs @@ -22,7 +22,7 @@ pub mod time { use crate::mp::CPU_HWID; use crate::mp::MAX_HARTS; -extern "C" { +unsafe extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index 1140c17b67..31a5e222db 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -1,6 +1,6 @@ use aarch64_cpu::{asm, asm::barrier, registers::*}; use core::ptr::addr_of_mut; -use page_table_entry::aarch64::{MemAttr, A64PTE}; +use page_table_entry::aarch64::{A64PTE, MemAttr}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::{TASK_STACK_SIZE, plat::PHYS_VIRT_OFFSET}; diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index 25045b1920..ca06e24e0d 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -7,13 +7,13 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::TASK_STACK_SIZE; -#[link_section = ".bss.stack"] +#[unsafe(link_section = ".bss.stack")] static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; -#[link_section = ".data.boot_page_table"] +#[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; -#[link_section = ".data.boot_page_table"] +#[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; unsafe fn switch_to_el2() { @@ -126,12 +126,13 @@ unsafe fn cache_invalidate(cache_level: usize) { /// The earliest entry point for the primary CPU. #[naked] -#[no_mangle] -#[link_section = ".text.boot"] +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.boot")] unsafe extern "C" fn _start() -> ! { - // PC = 0x8_0000 - // X0 = dtb - core::arch::asm!(" + unsafe { + // PC = 0x8_0000 + // X0 = dtb + core::arch::naked_asm!(" // save DTB pointer mov x20, x0 @@ -166,26 +167,27 @@ unsafe extern "C" fn _start() -> ! { ldr x8, ={entry} blr x8 b .", - cache_invalidate = sym cache_invalidate, - init_boot_page_table = sym init_boot_page_table, - init_mmu_el2 = sym init_mmu_el2, - switch_to_el2 = sym switch_to_el2, - enable_fp = sym enable_fp, - boot_stack = sym BOOT_STACK, - boot_stack_size = const TASK_STACK_SIZE, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry, - options(noreturn), - ); + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::plat::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + ); + } } /// The earliest entry point for the secondary CPUs. #[cfg(feature = "smp")] #[naked] -#[no_mangle] -#[link_section = ".text.boot"] +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.boot")] unsafe extern "C" fn _start_secondary() -> ! { - core::arch::asm!(" + unsafe { + core::arch::naked_asm!(" mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id @@ -201,11 +203,11 @@ unsafe extern "C" fn _start_secondary() -> ! { ldr x8, ={entry} blr x8 b .", - switch_to_el2 = sym switch_to_el2, - init_mmu_el2 = sym init_mmu_el2, - enable_fp = sym enable_fp, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry_secondary, - options(noreturn), - ) + switch_to_el2 = sym switch_to_el2, + init_mmu_el2 = sym init_mmu_el2, + enable_fp = sym enable_fp, + phys_virt_offset = const axconfig::plat::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry_secondary, + ) + } } diff --git a/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs index 6d9de293be..5d34bc9b0f 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs @@ -119,11 +119,33 @@ impl DW8250 { } } -const UART_BASE: PhysAddr = pa!(axconfig::UART_PADDR); +const UART_BASE: PhysAddr = pa!(axconfig::devices::UART_PADDR); pub static UART: SpinNoIrq = SpinNoIrq::new(DW8250::new(phys_to_virt(UART_BASE).as_usize())); +/// Write a slice of bytes to the console. +pub fn write_bytes(bytes: &[u8]) { + for c in bytes { + putchar(*c); + } +} + +/// Reads bytes from the console into the given mutable slice. +/// Returns the number of bytes read. +pub fn read_bytes(bytes: &mut [u8]) -> usize { + let mut read_len = 0; + while read_len < bytes.len() { + if let Some(c) = getchar() { + bytes[read_len] = c; + } else { + break; + } + read_len += 1; + } + read_len +} + /// Writes a byte to the console. pub fn putchar(c: u8) { let mut uart = UART.lock(); diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs index 4c0a970aba..5c2fd2d609 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs @@ -1,5 +1,5 @@ use crate::mem::{MemRegion, MemRegionFlags, PhysAddr}; -use page_table_entry::{aarch64::A64PTE, GenericPTE, MappingFlags}; +use page_table_entry::{GenericPTE, MappingFlags, aarch64::A64PTE}; /// Returns (rk3588j only) memory regions. pub(crate) fn default_rk3588j_regions() -> impl Iterator { diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mod.rs b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs index c59047bd78..873cde06d3 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mod.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs @@ -21,7 +21,7 @@ pub mod misc { pub use crate::platform::aarch64_common::psci::system_off as terminate; } -extern "C" { +unsafe extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mp.rs b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs index 01f84fc4c1..70d8cebecb 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mp.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs @@ -1,4 +1,4 @@ -use crate::mem::{virt_to_phys, PhysAddr, VirtAddr}; +use crate::mem::{PhysAddr, VirtAddr, virt_to_phys}; /// Hart number of rk3588 board pub const MAX_HARTS: usize = 8; @@ -8,7 +8,7 @@ pub const CPU_HWID: [usize; MAX_HARTS] = [0x00, 0x100, 0x200, 0x300, 0x400, 0x50 /// Starts the given secondary CPU with its boot stack. pub fn start_secondary_cpu(cpu_id: usize, stack_top: PhysAddr) { assert!(cpu_id < MAX_HARTS, "No support for rk3588 core {}", cpu_id); - extern "C" { + unsafe extern "C" { fn _start_secondary(); } let entry = virt_to_phys(VirtAddr::from(_start_secondary as usize)); From 39848356f3bb43c1c0a5b02a6188bd3235e789a5 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 1 Jan 2025 22:36:01 +0800 Subject: [PATCH 08/48] [fix] clippy error in aarch64 boot_el2 --- modules/axhal/src/platform/aarch64_common/boot_el2.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index ca06e24e0d..931a00e991 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -67,7 +67,7 @@ unsafe fn init_mmu_el2() { TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); barrier::isb(barrier::SY); - let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + let root_paddr = PhysAddr::from(&raw const BOOT_PT_L0 as usize).as_usize() as _; TTBR0_EL2.set(root_paddr); // Flush the entire TLB From 698f81c646c167a58955240a18730edba4af9168 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 11 Dec 2024 14:13:29 +0800 Subject: [PATCH 09/48] [wip] add cpu_id in TaskInner, expose on_cpu api --- modules/axtask/src/run_queue.rs | 1 + modules/axtask/src/task.rs | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/modules/axtask/src/run_queue.rs b/modules/axtask/src/run_queue.rs index 0c5873ca73..c9ebe17f63 100644 --- a/modules/axtask/src/run_queue.rs +++ b/modules/axtask/src/run_queue.rs @@ -494,6 +494,7 @@ impl AxRunQueue { } } // TODO: priority + task.set_cpu_id(self.cpu_id as _); self.scheduler.lock().put_prev_task(task, preempt); true } else { diff --git a/modules/axtask/src/task.rs b/modules/axtask/src/task.rs index 2b9a5bfd8c..c36f70d105 100644 --- a/modules/axtask/src/task.rs +++ b/modules/axtask/src/task.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, string::String, sync::Arc}; use core::ops::Deref; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU64, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicU8, Ordering}; use core::{alloc::Layout, cell::UnsafeCell, fmt, ptr::NonNull}; #[cfg(feature = "preempt")] @@ -51,6 +51,9 @@ pub struct TaskInner { /// Mark whether the task is in the wait queue. in_wait_queue: AtomicBool, + /// Used to indicate the CPU ID where the task is running or will run. + #[cfg(feature = "smp")] + cpu_id: AtomicU32, /// Used to indicate whether the task is running on a CPU. #[cfg(feature = "smp")] on_cpu: AtomicBool, @@ -227,6 +230,8 @@ impl TaskInner { #[cfg(feature = "irq")] timer_ticket_id: AtomicU64::new(0), #[cfg(feature = "smp")] + cpu_id: AtomicU32::new(0), + #[cfg(feature = "smp")] on_cpu: AtomicBool::new(false), #[cfg(feature = "preempt")] need_resched: AtomicBool::new(false), @@ -398,6 +403,19 @@ impl TaskInner { self.ctx.get() } + #[cfg(feature = "smp")] + pub fn set_cpu_id(&self, cpu_id: u32) { + self.cpu_id.store(cpu_id, Ordering::Release); + } + + /// Returns the CPU ID where the task is running or will run. + /// + /// Note: the task may not be running on the CPU, it just exists in the run queue. + #[cfg(feature = "smp")] + pub fn cpu_id(&self) -> u32 { + self.cpu_id.load(Ordering::Acquire) + } + /// Returns whether the task is running on a CPU. /// /// It is used to protect the task from being moved to a different run queue @@ -405,8 +423,7 @@ impl TaskInner { /// The `on_cpu field is set to `true` when the task is preparing to run on a CPU, /// and it is set to `false` when the task has finished its scheduling process in `clear_prev_task_on_cpu()`. #[cfg(feature = "smp")] - #[inline] - pub(crate) fn on_cpu(&self) -> bool { + pub fn on_cpu(&self) -> bool { self.on_cpu.load(Ordering::Acquire) } From 853870b7f03687f77dc7ac793ad3a60733082cbb Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 11 Dec 2024 16:42:43 +0800 Subject: [PATCH 10/48] [wip] add ipi for aarch64 --- Cargo.lock | 397 ++++++++++-------- Cargo.toml | 2 + api/axfeat/Cargo.toml | 1 + modules/axhal/Cargo.toml | 3 +- modules/axhal/src/irq.rs | 2 +- .../axhal/src/platform/aarch64_common/gic.rs | 9 + modules/axhal/src/platform/mod.rs | 6 +- modules/axipi/Cargo.toml | 22 + modules/axipi/src/lib.rs | 39 ++ modules/axipi/src/queue.rs | 77 ++++ modules/axruntime/Cargo.toml | 4 +- modules/axruntime/src/lib.rs | 3 + modules/axtask/Cargo.toml | 2 +- 13 files changed, 380 insertions(+), 187 deletions(-) create mode 100644 modules/axipi/Cargo.toml create mode 100644 modules/axipi/src/lib.rs create mode 100644 modules/axipi/src/queue.rs diff --git a/Cargo.lock b/Cargo.lock index bb680752a9..8099f2aaf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -83,7 +83,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -93,7 +93,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -200,18 +200,18 @@ dependencies = [ [[package]] name = "atomic-polyfill" -version = "0.1.11" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ff7eb3f316534d83a8a2c3d1674ace8a5a71198eba31e2e2b597833f699b28" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" dependencies = [ "critical-section", ] [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axalloc" @@ -251,7 +251,7 @@ dependencies = [ "axconfig-gen", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] @@ -446,6 +446,7 @@ dependencies = [ "bitflags 2.6.0", "cfg-if", "cortex-a", + "cpumask", "crate_interface", "dw_apb_uart", "handler_table", @@ -459,7 +460,7 @@ dependencies = [ "page_table_entry", "page_table_multiarch", "percpu", - "raw-cpuid 11.1.0", + "raw-cpuid 11.2.0", "riscv", "riscv_goldfish", "sbi-rt", @@ -480,6 +481,17 @@ dependencies = [ "axerrno", ] +[[package]] +name = "axipi" +version = "0.1.0" +dependencies = [ + "axhal", + "kspin", + "lazyinit", + "log", + "percpu", +] + [[package]] name = "axlibc" version = "0.1.0" @@ -544,6 +556,7 @@ dependencies = [ "axdriver", "axfs", "axhal", + "axipi", "axlog", "axmm", "axnet", @@ -615,9 +628,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -632,7 +645,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.70", + "syn 2.0.90", "which", ] @@ -683,15 +696,15 @@ checksum = "a7913f22349ffcfc6ca0ca9a656ec26cfbba538ed49c31a273dff2c5d1ea83d9" [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cap_access" @@ -704,11 +717,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ - "libc", + "shlex", ] [[package]] @@ -728,9 +741,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -742,9 +755,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -782,7 +795,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] @@ -805,9 +818,9 @@ checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core_detect" @@ -841,20 +854,20 @@ checksum = "6af24c4862260a825484470f5526a91ad1031e04ab899be62478241231f62b46" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] name = "critical-section" -version = "1.1.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "defmt" -version = "0.3.5" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2d011b2fee29fb7d659b83c43fce9a2cb4df453e16d441a51448e448f3f98" +checksum = "86f6162c53f659f65d00619fe31f14556a6e9f8752ccc4a41bd177ffcf3d6130" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -862,22 +875,22 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "0.3.6" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54f0216f6c5acb5ae1a47050a6645024e6edafc2ee32d421955eccfef12ef92e" +checksum = "9d135dd939bad62d7490b0002602d35b358dce5fd9233a709d3c1ef467d4bde6" dependencies = [ "defmt-parser", - "proc-macro-error", + "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] name = "defmt-parser" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269924c02afd7f94bc4cecbfa5c379f6ffcf9766b3408fe63d22c728654eccd0" +checksum = "3983b127f13995e68c1e29071e5d115cd96f215ccb5e6812e3728cd6f92653b3" dependencies = [ "thiserror", ] @@ -893,9 +906,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "embedded-hal" @@ -909,6 +922,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "fatfs" version = "0.4.0" @@ -929,9 +952,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -961,15 +984,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heapless" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" dependencies = [ "atomic-polyfill", "hash32", @@ -984,11 +1007,20 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1009,9 +1041,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown", @@ -1031,9 +1063,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.10.5" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -1052,10 +1084,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1102,18 +1135,18 @@ checksum = "3861aac8febbb038673bf945ee47ac67940ca741b94d1bb3ff6066af2a181338" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "winapi", + "windows-targets", ] [[package]] @@ -1138,14 +1171,20 @@ checksum = "edbe595006d355eaf9ae11db92707d4338cd2384d16866131cc1afdbdd35d8d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1165,9 +1204,9 @@ checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memory_addr" @@ -1193,18 +1232,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "page_table_entry" @@ -1258,63 +1297,64 @@ checksum = "f56d145a4b532545c4932380a01bde738cc2f0bf2e18578358b2937e727231f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "prettyplease" -version = "0.2.12" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "proc-macro-error-attr2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ - "proc-macro-error-attr", "proc-macro2", "quote", - "syn 1.0.109", - "version_check", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "proc-macro-error2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" dependencies = [ + "proc-macro-error-attr2", "proc-macro2", "quote", - "version_check", + "syn 2.0.90", ] [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1360,18 +1400,18 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.9.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1381,9 +1421,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1392,9 +1432,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "riscv" @@ -1432,18 +1472,31 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "sbi-rt" @@ -1476,15 +1529,15 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.18" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "shlex" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "slab_allocator" @@ -1561,9 +1614,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.70" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -1572,22 +1625,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.47" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] [[package]] @@ -1610,9 +1663,9 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "toml_datetime", @@ -1621,15 +1674,15 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "utf8parse" @@ -1637,12 +1690,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "virtio-drivers" version = "0.7.4" @@ -1680,34 +1727,34 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1715,79 +1762,67 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-targets", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" +name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ "windows-targets", ] [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1801,57 +1836,57 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -1932,5 +1967,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.90", ] diff --git a/Cargo.toml b/Cargo.toml index 2924a174af..25cfe78ecf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "modules/axruntime", "modules/axsync", "modules/axtask", + "modules/axipi", "api/axfeat", "api/arceos_api", @@ -62,6 +63,7 @@ axruntime = { path = "modules/axruntime" } axsync = { path = "modules/axsync" } axtask = { path = "modules/axtask" } axdma = { path = "modules/axdma" } +axipi = { path = "modules/axipi" } [profile.release] lto = true diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index 804531f671..8a9e0ae391 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -20,6 +20,7 @@ fp_simd = ["axhal/fp_simd"] # Interrupts irq = ["axhal/irq", "axruntime/irq", "axtask?/irq"] +ipi = ["axruntime/ipi"] # Memory alloc = ["axalloc", "axruntime/alloc"] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index 39a01bb433..2acf059a66 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/arceos-org/arceos/tree/main/modules/axhal" documentation = "https://arceos-org.github.io/arceos/axhal/index.html" [features] -smp = [] +smp = ["dep:cpumask"] alloc = [] fp_simd = [] paging = ["axalloc", "page_table_multiarch"] @@ -39,6 +39,7 @@ axlog = { workspace = true } axconfig = { workspace = true } axalloc = { workspace = true, optional = true } cortex-a = { version = "8.1.1", optional = true } +cpumask = { version = "0.1", optional = true } [target.'cfg(target_arch = "x86_64")'.dependencies] x86 = "0.52" diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 2d6b1247c6..4e0149196b 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -5,7 +5,7 @@ use handler_table::HandlerTable; use crate::platform::irq::{MAX_IRQ_COUNT, dispatch_irq}; use crate::trap::{IRQ, register_trap_handler}; -pub use crate::platform::irq::{register_handler, set_enable}; +pub use crate::platform::irq::{register_handler, set_enable, IPI_IRQ_NUM}; #[cfg(target_arch = "aarch64")] pub use crate::platform::irq::fetch_irq; diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index dad3172cd3..f79d56eff3 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -18,6 +18,9 @@ pub const TIMER_IRQ_NUM: usize = translate_irq(10, InterruptType::PPI).unwrap(); /// The UART IRQ number. pub const UART_IRQ_NUM: usize = translate_irq(UART_IRQ, InterruptType::SPI).unwrap(); +/// The IPI IRQ number. +pub const IPI_IRQ_NUM: usize = translate_irq(1, InterruptType::SGI).unwrap(); + const GICD_BASE: PhysAddr = pa!(GICD_PADDR); const GICC_BASE: PhysAddr = pa!(GICC_PADDR); @@ -42,6 +45,12 @@ pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { crate::irq::register_handler_common(irq_num, handler) } +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given CPUs. +#[cfg(feature = "smp")] +pub fn send_sgi(cpu_if: usize, irq_num: usize) { + GICD.lock().send_sgi(cpu_if, irq_num); +} + /// Fetches the IRQ number. pub fn fetch_irq() -> usize { GICC.iar() as usize diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 57431a0e05..d824d01f34 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -29,7 +29,9 @@ cfg_if::cfg_if! { mod aarch64_rk3588j; pub use self::aarch64_rk3588j::*; } else { - mod dummy; - pub use self::dummy::*; + // mod dummy; + // pub use self::dummy::*; + mod aarch64_qemu_virt; + pub use self::aarch64_qemu_virt::*; } } diff --git a/modules/axipi/Cargo.toml b/modules/axipi/Cargo.toml new file mode 100644 index 0000000000..073abd27c3 --- /dev/null +++ b/modules/axipi/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "axipi" +edition = "2021" +version.workspace = true +authors = ["Keyang Hu "] +description = "ArceOS IPI management module" +license.workspace = true +homepage.workspace = true +# repository = "https://github.com/arceos-org/arceos/tree/main/modules/axipi" +# documentation = "https://arceos-org.github.io/arceos/axipi/index.html" + +[features] +default = [] + +[dependencies] +log = "=0.4.21" +lazyinit = { version = "0.2" } + +percpu = { version = "0.1.5" } +kspin = { version = "0.1" } + +axhal = { workspace = true } diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs new file mode 100644 index 0000000000..1d646c3c63 --- /dev/null +++ b/modules/axipi/src/lib.rs @@ -0,0 +1,39 @@ +extern crate alloc; + +use lazyinit::LazyInit; + +use kspin::SpinNoIrq; + +use axhal::cpu::this_cpu_id; +use axhal::irq::IPI_IRQ_NUM; + +mod queue; + +use queue::IPIEventQueue; + +pub use queue::IPIEventFn; + +#[percpu::def_percpu] +static IPI_MSG_QUEUE: LazyInit>> = LazyInit::new(); + +pub fn init() { + IPI_MSG_QUEUE.with_current(|ipi_queue| { + ipi_queue.init_once(SpinNoIrq::new(IPIEventQueue::default())); + }); + axhal::irq::register_handler(IPI_IRQ_NUM, ipi_handler); +} + +pub fn send_ipi_event(target_cpu: usize, event: IPIEventFn) { + unsafe { IPI_MSG_QUEUE.remote_ref_raw(target_cpu) } + .lock() + .push(this_cpu_id(), event); +} + +fn ipi_handler() { + while let Some(event) = unsafe { IPI_MSG_QUEUE.current_ref_mut_raw() } + .lock() + .pop_one() + { + event(); + } +} diff --git a/modules/axipi/src/queue.rs b/modules/axipi/src/queue.rs new file mode 100644 index 0000000000..80c2970ada --- /dev/null +++ b/modules/axipi/src/queue.rs @@ -0,0 +1,77 @@ +use alloc::collections::VecDeque; + +/// A queue of IPI events. +/// +/// It internally uses a `VecDeque` to store the events, make it +/// possible to pop these events using FIFO order. +pub struct IPIEventQueue { + events: VecDeque>, +} + +/// A trait that all events must implement. +pub trait IPIEvent: 'static { + /// Callback function that will be called when the event is triggered. + fn callback(self); +} + +struct IPIEventWrapper { + src_cpu_id: u32, + event: E, +} + +impl IPIEventQueue { + /// Creates a new empty timer list. + pub fn new() -> Self { + Self { + events: VecDeque::new(), + } + } + + /// Whether there is no event. + #[inline] + pub fn is_empty(&self) -> bool { + self.events.is_empty() + } + + pub fn push(&mut self, src_cpu_id: u32, event: E) { + self.events.push_back(IPIEventWrapper { src_cpu_id, event }); + } + + /// Try to pop the latest event that exists in the queue. + /// + /// Returns `None` if no event is available. + pub fn pop_one(&mut self) -> Option { + if let Some(e) = self.events.pop_front() { + Some(e.event) + } else { + None + } + } +} + +impl Default for IPIEventQueue { + fn default() -> Self { + Self::new() + } +} + +/// A simple wrapper of a closure that implements the [`IPIEvent`] trait. +/// +/// So that it can be used as in the [`IPIEventQueue`]. +pub struct IPIEventFn(Box); + +impl IPIEventFn { + /// Constructs a new [`IPIEventFn`] from a closure. + pub fn new(f: F) -> Self + where + F: FnOnce() + 'static, + { + Self(Box::new(f)) + } +} + +impl IPIEvent for IPIEventFn { + fn callback(self) { + (self.0)() + } +} diff --git a/modules/axruntime/Cargo.toml b/modules/axruntime/Cargo.toml index 3f76289949..8238f2d390 100644 --- a/modules/axruntime/Cargo.toml +++ b/modules/axruntime/Cargo.toml @@ -17,6 +17,7 @@ irq = ["axhal/irq", "axtask?/irq", "percpu", "kernel_guard"] tls = ["axhal/tls", "axtask?/tls"] alloc = ["axalloc"] paging = ["axhal/paging", "axmm"] +ipi = ["dep:axipi"] multitask = ["axtask/multitask"] fs = ["axdriver", "axfs"] @@ -35,9 +36,10 @@ axfs = { workspace = true, optional = true } axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } axtask = { workspace = true, optional = true } +axipi = { workspace = true, optional = true } crate_interface = "0.1" -percpu = { version = "0.1.4", optional = true } +percpu = { version = "0.1", optional = true } kernel_guard = { version = "0.1", optional = true } chrono = { version = "0.4.38", default-features = false } diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 114db35cc0..2fabe3caee 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -258,6 +258,9 @@ fn init_interrupt() { axtask::on_timer_tick(); }); + #[cfg(feature = "ipi")] + axipi::init(); + // Enable IRQs before starting app axhal::arch::enable_irqs(); } diff --git a/modules/axtask/Cargo.toml b/modules/axtask/Cargo.toml index f5d0ad73aa..6232c42f9d 100644 --- a/modules/axtask/Cargo.toml +++ b/modules/axtask/Cargo.toml @@ -40,7 +40,7 @@ cfg-if = "1.0" log = "=0.4.21" axhal = { workspace = true } axconfig = { workspace = true, optional = true } -percpu = { version = "0.1.4", optional = true } +percpu = { version = "0.1", optional = true } kspin = { version = "0.1", optional = true } lazyinit = { version = "0.2", optional = true } memory_addr = { version = "0.3", optional = true } From be145dc4b3a8b2a2b9423ab70386bb1bb30dd487 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 12 Dec 2024 15:12:26 +0800 Subject: [PATCH 11/48] [wip] add ipi for x86_64 --- Cargo.lock | 4 +- api/arceos_api/Cargo.toml | 1 + api/arceos_api/src/lib.rs | 1 + api/axfeat/Cargo.toml | 4 +- modules/axhal/Cargo.toml | 5 ++- modules/axhal/src/irq.rs | 5 ++- .../axhal/src/platform/aarch64_common/gic.rs | 14 ++++--- modules/axhal/src/platform/mod.rs | 8 ++-- modules/axhal/src/platform/x86_pc/apic.rs | 19 +++++++++ modules/axipi/Cargo.toml | 1 + modules/axipi/src/lib.rs | 42 +++++++++++++++---- modules/axipi/src/queue.rs | 16 +++---- modules/axruntime/src/lib.rs | 5 ++- modules/axruntime/src/mp.rs | 3 ++ modules/axtask/src/run_queue.rs | 1 + modules/axtask/src/task.rs | 5 ++- ulib/axstd/Cargo.toml | 2 +- 17 files changed, 102 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8099f2aaf3..2b0856fb9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,6 +141,7 @@ dependencies = [ "axfs", "axhal", "axio", + "axipi", "axlog", "axmm", "axnet", @@ -175,7 +176,7 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arm_gicv2#dfe5f164b94cdd07081c2fe74a0cfe4bef2852c9" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2?branch=sgi#956665adcdc6b09a28078db09ad3f09ed41af80b" dependencies = [ "tock-registers", ] @@ -485,6 +486,7 @@ dependencies = [ name = "axipi" version = "0.1.0" dependencies = [ + "axconfig", "axhal", "kspin", "lazyinit", diff --git a/api/arceos_api/Cargo.toml b/api/arceos_api/Cargo.toml index 9f679d830f..f73902ae9b 100644 --- a/api/arceos_api/Cargo.toml +++ b/api/arceos_api/Cargo.toml @@ -43,3 +43,4 @@ axdriver = { workspace = true, optional = true } axfs = { workspace = true, optional = true } axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } +axipi = { workspace = true } diff --git a/api/arceos_api/src/lib.rs b/api/arceos_api/src/lib.rs index 3e6d291870..e29c4d3be9 100644 --- a/api/arceos_api/src/lib.rs +++ b/api/arceos_api/src/lib.rs @@ -391,6 +391,7 @@ pub mod modules { pub use axlog; pub use axruntime; pub use axsync; + pub use axipi; #[cfg(feature = "alloc")] pub use axalloc; diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index 8a9e0ae391..b7ee539bac 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -20,7 +20,7 @@ fp_simd = ["axhal/fp_simd"] # Interrupts irq = ["axhal/irq", "axruntime/irq", "axtask?/irq"] -ipi = ["axruntime/ipi"] +ipi = ["axhal/ipi", "axruntime/ipi"] # Memory alloc = ["axalloc", "axruntime/alloc"] @@ -60,7 +60,7 @@ driver-ixgbe = ["axdriver?/ixgbe"] driver-bcm2835-sdhci = ["axdriver?/bcm2835-sdhci"] #Hypervisor support -hv = ["axhal/hv"] +hv = ["axhal/hv", "ipi"] # Logging log-level-off = ["axlog/log-level-off"] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index 2acf059a66..0b70862e36 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/arceos-org/arceos/tree/main/modules/axhal" documentation = "https://arceos-org.github.io/arceos/axhal/index.html" [features] -smp = ["dep:cpumask"] +smp = [] alloc = [] fp_simd = [] paging = ["axalloc", "page_table_multiarch"] @@ -19,6 +19,7 @@ tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] default = [] hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "arm_gicv2/el2", "dep:crate_interface"] +ipi = [] [dependencies] log = "=0.4.21" @@ -56,7 +57,7 @@ riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.4" tock-registers = "0.8" -arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2" } +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", branch = "sgi" } crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 4e0149196b..3b8f2d4bce 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -5,7 +5,10 @@ use handler_table::HandlerTable; use crate::platform::irq::{MAX_IRQ_COUNT, dispatch_irq}; use crate::trap::{IRQ, register_trap_handler}; -pub use crate::platform::irq::{register_handler, set_enable, IPI_IRQ_NUM}; +pub use crate::platform::irq::{register_handler, set_enable}; + +#[cfg(feature = "ipi")] +pub use crate::platform::irq::{send_sgi_all, send_sgi_one, IPI_IRQ_NUM}; #[cfg(target_arch = "aarch64")] pub use crate::platform::irq::fetch_irq; diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index f79d56eff3..632e534d65 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -41,14 +41,18 @@ pub fn set_enable(irq_num: usize, enabled: bool) { /// It also enables the IRQ if the registration succeeds. It returns `false` if /// the registration failed. pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { - trace!("register handler irq {}", irq_num); + warn!("register handler irq {}", irq_num); crate::irq::register_handler_common(irq_num, handler) } -/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given CPUs. -#[cfg(feature = "smp")] -pub fn send_sgi(cpu_if: usize, irq_num: usize) { - GICD.lock().send_sgi(cpu_if, irq_num); +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { + GICD.lock().send_sgi(dest_cpu_id, irq_num); +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_sgi_all(irq_num: usize) { + GICD.lock().send_sgi_all_except_self(irq_num); } /// Fetches the IRQ number. diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index d824d01f34..9de869edb7 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -29,9 +29,9 @@ cfg_if::cfg_if! { mod aarch64_rk3588j; pub use self::aarch64_rk3588j::*; } else { - // mod dummy; - // pub use self::dummy::*; - mod aarch64_qemu_virt; - pub use self::aarch64_qemu_virt::*; + mod dummy; + pub use self::dummy::*; + // mod aarch64_qemu_virt; + // pub use self::aarch64_qemu_virt::*; } } diff --git a/modules/axhal/src/platform/x86_pc/apic.rs b/modules/axhal/src/platform/x86_pc/apic.rs index ca2a86c36b..835919b0f9 100644 --- a/modules/axhal/src/platform/x86_pc/apic.rs +++ b/modules/axhal/src/platform/x86_pc/apic.rs @@ -16,6 +16,7 @@ pub(super) mod vectors { pub const APIC_TIMER_VECTOR: u8 = 0xf0; pub const APIC_SPURIOUS_VECTOR: u8 = 0xf1; pub const APIC_ERROR_VECTOR: u8 = 0xf2; + pub const APIC_IPI_VECTOR: u8 = 0xf2; } /// The maximum number of IRQs. @@ -24,6 +25,9 @@ pub const MAX_IRQ_COUNT: usize = 256; /// The timer IRQ number. pub const TIMER_IRQ_NUM: usize = APIC_TIMER_VECTOR as usize; +/// The IPI IRQ number. +pub const IPI_IRQ_NUM: usize = APIC_IPI_VECTOR as usize; + const IO_APIC_BASE: PhysAddr = pa!(0xFEC0_0000); static LOCAL_APIC: SyncUnsafeCell> = @@ -55,6 +59,21 @@ pub fn register_handler(vector: usize, handler: crate::irq::IrqHandler) -> bool crate::irq::register_handler_common(vector, handler) } +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { + unsafe { + local_apic().send_ipi(irq_num as _, dest_cpu_id as _); + }; +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_sgi_all(irq_num: usize) { + use x2apic::lapic::IpiAllShorthand; + unsafe { + local_apic().send_ipi_all(irq_num as _, IpiAllShorthand::AllExcludingSelf); + }; +} + /// Dispatches the IRQ. /// /// This function is called by the common interrupt handler. It looks diff --git a/modules/axipi/Cargo.toml b/modules/axipi/Cargo.toml index 073abd27c3..3823e549d6 100644 --- a/modules/axipi/Cargo.toml +++ b/modules/axipi/Cargo.toml @@ -20,3 +20,4 @@ percpu = { version = "0.1.5" } kspin = { version = "0.1" } axhal = { workspace = true } +axconfig = { workspace = true } \ No newline at end of file diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs index 1d646c3c63..04a851d6a9 100644 --- a/modules/axipi/src/lib.rs +++ b/modules/axipi/src/lib.rs @@ -1,3 +1,9 @@ +//! [ArceOS](https://github.com/arceos-org/arceos) Inter-Processor Interrupt (IPI) primitives. + +#![cfg_attr(not(test), no_std)] + +#[macro_use] +extern crate log; extern crate alloc; use lazyinit::LazyInit; @@ -11,29 +17,47 @@ mod queue; use queue::IPIEventQueue; -pub use queue::IPIEventFn; +pub use queue::{IPIEvent, IPIEventFn}; #[percpu::def_percpu] -static IPI_MSG_QUEUE: LazyInit>> = LazyInit::new(); +static IPI_EVENT_QUEUE: LazyInit>> = LazyInit::new(); +/// Initialize the per-CPU IPI event queue. pub fn init() { - IPI_MSG_QUEUE.with_current(|ipi_queue| { + IPI_EVENT_QUEUE.with_current(|ipi_queue| { ipi_queue.init_once(SpinNoIrq::new(IPIEventQueue::default())); }); - axhal::irq::register_handler(IPI_IRQ_NUM, ipi_handler); } -pub fn send_ipi_event(target_cpu: usize, event: IPIEventFn) { - unsafe { IPI_MSG_QUEUE.remote_ref_raw(target_cpu) } +/// Sends an IPI event to the processor(s) specified by `dest_cpu`. +pub fn send_ipi_event_to_one(dest_cpu: usize, event: IPIEventFn) { + warn!("Send IPI event to CPU {}", dest_cpu); + + unsafe { IPI_EVENT_QUEUE.remote_ref_raw(dest_cpu) } .lock() .push(this_cpu_id(), event); + axhal::irq::send_sgi_one(dest_cpu, IPI_IRQ_NUM); +} + +/// Sends an IPI event to all processors except the current one. +pub fn send_ipi_event_to_all(event: IPIEventFn) { + let current_cpu_id = this_cpu_id(); + for cpu_id in 0..axconfig::SMP { + if cpu_id != current_cpu_id { + unsafe { IPI_EVENT_QUEUE.remote_ref_raw(cpu_id) } + .lock() + .push(current_cpu_id, event.clone()); + } + } + axhal::irq::send_sgi_all(IPI_IRQ_NUM); } -fn ipi_handler() { - while let Some(event) = unsafe { IPI_MSG_QUEUE.current_ref_mut_raw() } +pub fn ipi_handler() { + while let Some((src_cpu_id, event)) = unsafe { IPI_EVENT_QUEUE.current_ref_mut_raw() } .lock() .pop_one() { - event(); + warn!("Received IPI event from CPU {}", src_cpu_id); + event.callback(); } } diff --git a/modules/axipi/src/queue.rs b/modules/axipi/src/queue.rs index 80c2970ada..724754e50f 100644 --- a/modules/axipi/src/queue.rs +++ b/modules/axipi/src/queue.rs @@ -1,3 +1,4 @@ +use alloc::boxed::Box; use alloc::collections::VecDeque; /// A queue of IPI events. @@ -15,7 +16,7 @@ pub trait IPIEvent: 'static { } struct IPIEventWrapper { - src_cpu_id: u32, + src_cpu_id: usize, event: E, } @@ -33,16 +34,16 @@ impl IPIEventQueue { self.events.is_empty() } - pub fn push(&mut self, src_cpu_id: u32, event: E) { + pub fn push(&mut self, src_cpu_id: usize, event: E) { self.events.push_back(IPIEventWrapper { src_cpu_id, event }); } /// Try to pop the latest event that exists in the queue. /// /// Returns `None` if no event is available. - pub fn pop_one(&mut self) -> Option { + pub fn pop_one(&mut self) -> Option<(usize, E)> { if let Some(e) = self.events.pop_front() { - Some(e.event) + Some((e.src_cpu_id, e.event)) } else { None } @@ -58,13 +59,14 @@ impl Default for IPIEventQueue { /// A simple wrapper of a closure that implements the [`IPIEvent`] trait. /// /// So that it can be used as in the [`IPIEventQueue`]. -pub struct IPIEventFn(Box); +#[derive(Clone)] +pub struct IPIEventFn(Box<&'static dyn Fn()>); impl IPIEventFn { /// Constructs a new [`IPIEventFn`] from a closure. - pub fn new(f: F) -> Self + pub fn new(f: &'static F) -> Self where - F: FnOnce() + 'static, + F: Fn(), { Self(Box::new(f)) } diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 2fabe3caee..a40fd04232 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -259,7 +259,10 @@ fn init_interrupt() { }); #[cfg(feature = "ipi")] - axipi::init(); + { + axipi::init(); + axhal::irq::register_handler(axhal::irq::IPI_IRQ_NUM, axipi::ipi_handler); + } // Enable IRQs before starting app axhal::arch::enable_irqs(); diff --git a/modules/axruntime/src/mp.rs b/modules/axruntime/src/mp.rs index 5e1baab4b9..b97ba0a665 100644 --- a/modules/axruntime/src/mp.rs +++ b/modules/axruntime/src/mp.rs @@ -44,6 +44,9 @@ pub extern "C" fn rust_main_secondary(cpu_id: usize) -> ! { #[cfg(feature = "multitask")] axtask::init_scheduler_secondary(); + #[cfg(feature = "ipi")] + axipi::init(); + info!("Secondary CPU {:x} init OK.", cpu_id); super::INITED_CPUS.fetch_add(1, Ordering::Relaxed); diff --git a/modules/axtask/src/run_queue.rs b/modules/axtask/src/run_queue.rs index c9ebe17f63..e781d97cf6 100644 --- a/modules/axtask/src/run_queue.rs +++ b/modules/axtask/src/run_queue.rs @@ -494,6 +494,7 @@ impl AxRunQueue { } } // TODO: priority + #[cfg(feature = "smp")] task.set_cpu_id(self.cpu_id as _); self.scheduler.lock().put_prev_task(task, preempt); true diff --git a/modules/axtask/src/task.rs b/modules/axtask/src/task.rs index c36f70d105..2c0d05222d 100644 --- a/modules/axtask/src/task.rs +++ b/modules/axtask/src/task.rs @@ -1,11 +1,14 @@ use alloc::{boxed::Box, string::String, sync::Arc}; use core::ops::Deref; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicU8, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, AtomicU8, Ordering}; use core::{alloc::Layout, cell::UnsafeCell, fmt, ptr::NonNull}; #[cfg(feature = "preempt")] use core::sync::atomic::AtomicUsize; +#[cfg(feature = "smp")] +use core::sync::atomic::AtomicU32; + use kspin::SpinNoIrq; use memory_addr::{VirtAddr, align_up_4k}; diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index 9801bc7c9c..2f3ce0d41d 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -67,7 +67,7 @@ driver-ixgbe = ["axfeat/driver-ixgbe"] driver-bcm2835-sdhci = ["axfeat/driver-bcm2835-sdhci"] # Hypervisor support -hv = ["axfeat/hv"] +hv = ["axfeat/hv", "axfeat/ipi"] # Logging log-level-off = ["axfeat/log-level-off"] From 3d0409dc7c821660743e079a2408694b157f9ecd Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 18 Dec 2024 14:12:55 +0800 Subject: [PATCH 12/48] [feat] add ipi support for riscv64, introduce riscv_plic --- Cargo.lock | 11 + api/arceos_api/Cargo.toml | 3 +- api/arceos_api/src/lib.rs | 3 +- api/axfeat/Cargo.toml | 3 +- examples/helloworld/Cargo.toml | 2 +- modules/axhal/Cargo.toml | 1 + .../axhal/src/platform/aarch64_common/gic.rs | 7 + modules/axhal/src/platform/mod.rs | 8 +- .../src/platform/riscv64_qemu_virt/irq.rs | 106 ++++++++- modules/axipi/src/queue.rs | 10 +- qemu.dts | 211 ++++++++++++++++++ ulib/axstd/Cargo.toml | 2 +- 12 files changed, 347 insertions(+), 20 deletions(-) create mode 100644 qemu.dts diff --git a/Cargo.lock b/Cargo.lock index 2b0856fb9a..384a758a70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,7 @@ dependencies = [ "axdriver", "axfs", "axhal", + "axipi", "axlog", "axnet", "axruntime", @@ -464,6 +465,7 @@ dependencies = [ "raw-cpuid 11.2.0", "riscv", "riscv_goldfish", + "riscv_plic", "sbi-rt", "static_assertions", "tock-registers", @@ -1454,6 +1456,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07aac72f95e774476db82916d79f2d303191310393830573c1ab5c821b21660a" +[[package]] +name = "riscv_plic" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daae4c8e29d4d8c36fbda4c318ffa1755645766fefab8fcff61a8ecd577bb822" +dependencies = [ + "tock-registers", +] + [[package]] name = "rlsf" version = "0.2.1" diff --git a/api/arceos_api/Cargo.toml b/api/arceos_api/Cargo.toml index f73902ae9b..e7db29ee11 100644 --- a/api/arceos_api/Cargo.toml +++ b/api/arceos_api/Cargo.toml @@ -13,6 +13,7 @@ documentation = "https://arceos-org.github.io/arceos/arceos_api/index.html" default = [] irq = ["axfeat/irq"] +ipi = ["dep:axipi", "axfeat/ipi"] alloc = ["dep:axalloc", "axfeat/alloc"] paging = ["dep:axmm", "axfeat/paging"] dma = ["dep:axdma", "axfeat/dma"] @@ -43,4 +44,4 @@ axdriver = { workspace = true, optional = true } axfs = { workspace = true, optional = true } axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } -axipi = { workspace = true } +axipi = { workspace = true, optional = true } diff --git a/api/arceos_api/src/lib.rs b/api/arceos_api/src/lib.rs index e29c4d3be9..7ec14731bb 100644 --- a/api/arceos_api/src/lib.rs +++ b/api/arceos_api/src/lib.rs @@ -391,8 +391,9 @@ pub mod modules { pub use axlog; pub use axruntime; pub use axsync; - pub use axipi; + #[cfg(feature = "ipi")] + pub use axipi; #[cfg(feature = "alloc")] pub use axalloc; #[cfg(feature = "display")] diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index b7ee539bac..d380218482 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -20,7 +20,7 @@ fp_simd = ["axhal/fp_simd"] # Interrupts irq = ["axhal/irq", "axruntime/irq", "axtask?/irq"] -ipi = ["axhal/ipi", "axruntime/ipi"] +ipi = ["irq", "dep:axipi", "axhal/ipi", "axruntime/ipi"] # Memory alloc = ["axalloc", "axruntime/alloc"] @@ -81,4 +81,5 @@ axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } axsync = { workspace = true, optional = true } axtask = { workspace = true, optional = true } +axipi = { workspace = true, optional = true } kspin = { version = "0.1", optional = true } diff --git a/examples/helloworld/Cargo.toml b/examples/helloworld/Cargo.toml index 31806aa9bb..8db0ea2300 100644 --- a/examples/helloworld/Cargo.toml +++ b/examples/helloworld/Cargo.toml @@ -7,4 +7,4 @@ authors = ["Yuekai Jia "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axstd = { workspace = true, optional = true } +axstd = { workspace = true,features = ["irq"], optional = true } diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index 0b70862e36..4f03c2515e 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -52,6 +52,7 @@ x86_rtc = { version = "0.1", optional = true } [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] riscv = "0.11" sbi-rt = { version = "0.0.3", features = ["legacy"] } +riscv_plic = { version = "0.1.0" } riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 632e534d65..44cc865b1e 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -66,6 +66,13 @@ pub fn fetch_irq() -> usize { /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. pub fn dispatch_irq(irq_no: usize) { + // I know, `irq_no == 0` seems very strange here. + // The truth is, the previous design of ArceOS in aarch64 DO NOT fetch the IRQ number from GICC, + // until the function closure passed to `GICC.handle_irq`. + // So, the `irq_no` is always 0 when `dispatch_irq` is called by `handler_irq` from inside ArceOS. + // + // However, when `handler_irq` is called by the arceos-vmm app, the `irq_no` has been fetched from GICC, so we can not use the `handler_irq` directly. + // Instead, we call `GICC.eoi` and `GICC.dir` manually. if irq_no == 0 { GICC.handle_irq(|irq_num| crate::irq::dispatch_irq_common(irq_num as _)); } else { diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 9de869edb7..411d018a4e 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -29,9 +29,9 @@ cfg_if::cfg_if! { mod aarch64_rk3588j; pub use self::aarch64_rk3588j::*; } else { - mod dummy; - pub use self::dummy::*; - // mod aarch64_qemu_virt; - // pub use self::aarch64_qemu_virt::*; + // mod dummy; + // pub use self::dummy::*; + mod riscv64_qemu_virt; + pub use self::riscv64_qemu_virt::*; } } diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index d46dfb3015..d3b7192c0c 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -1,8 +1,43 @@ //! TODO: PLIC -use crate::irq::IrqHandler; +use kspin::SpinNoIrq; use lazyinit::LazyInit; + use riscv::register::sie; +use riscv_plic::{HartContext, InterruptSource, Plic}; +use sbi_rt::{send_ipi, HartMask}; + +use memory_addr::{pa, PhysAddr}; + +use crate::cpu::this_cpu_id; +use crate::irq::IrqHandler; +use crate::mem::phys_to_virt; + +struct Context { + hart_id: usize, +} + +impl HartContext for Context { + fn index(self) -> usize { + self.hart_id * 2 + 1 + } +} + +impl Context { + const fn new(hart_id: usize) -> Self { + Self { hart_id } + } +} + +struct Interrupt { + irq_num: usize, +} + +impl InterruptSource for Interrupt { + fn id(self) -> core::num::NonZeroU32 { + core::num::NonZeroU32::new(self.irq_num as u32).unwrap() + } +} /// `Interrupt` bit in `scause` pub(super) const INTC_IRQ_BASE: usize = 1 << (usize::BITS - 1); @@ -19,26 +54,45 @@ pub(super) const S_EXT: usize = INTC_IRQ_BASE + 9; static TIMER_HANDLER: LazyInit = LazyInit::new(); +static IPI_HANDLER: LazyInit = LazyInit::new(); + /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; /// The timer IRQ number (supervisor timer interrupt in `scause`). pub const TIMER_IRQ_NUM: usize = S_TIMER; +/// The IPI IRQ number (supervisor software interrupt in `scause`). +pub const IPI_IRQ_NUM: usize = S_SOFT; + macro_rules! with_cause { - ($cause: expr, @TIMER => $timer_op: expr, @EXT => $ext_op: expr $(,)?) => { + ($cause: expr, + @TIMER => $timer_op: expr, + @IPI => $ipi_op: expr, + @EXT => $ext_op: expr $(,)?) => { match $cause { S_TIMER => $timer_op, + S_SOFT => $ipi_op, S_EXT => $ext_op, _ => panic!("invalid trap cause: {:#x}", $cause), } }; } +const PLIC_BASE: PhysAddr = pa!(axconfig::PLIC_PADDR); + +static PLIC: SpinNoIrq = SpinNoIrq::new(Plic::new(phys_to_virt(PLIC_BASE).as_mut_ptr())); + /// Enables or disables the given IRQ. -pub fn set_enable(scause: usize, _enabled: bool) { - if scause == S_EXT { - // TODO: set enable in PLIC +pub fn set_enable(irq_num: usize, enabled: bool) { + // TODO: set enable in PLIC + let source = Interrupt { irq_num }; + let context = Context::new(this_cpu_id()); + + if enabled { + PLIC.lock().enable(source, context); + } else { + PLIC.lock().disable(source, context); } } @@ -55,10 +109,37 @@ pub fn register_handler(scause: usize, handler: IrqHandler) -> bool { } else { false }, + @IPI => if !IPI_HANDLER.is_inited() { + IPI_HANDLER.init_once(handler); + true + } else { + false + }, @EXT => crate::irq::register_handler_common(scause & !INTC_IRQ_BASE, handler), ) } +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_sgi_one(dest_cpu_id: usize, _irq_num: usize) { + let res = send_ipi(HartMask::from_mask_base(1, dest_cpu_id)); + if res.is_err() { + warn!("send_sgi_one failed: {:?}", res); + } +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_sgi_all(_irq_num: usize) { + for i in 0..axconfig::SMP { + if i != this_cpu_id() { + let res = send_ipi(HartMask::from_mask_base(1, i)); + if res.is_err() { + warn!("send_sgi_all failed: {:?}", res); + break; + } + } + } +} + /// Dispatches the IRQ. /// /// This function is called by the common interrupt handler. It looks @@ -71,11 +152,24 @@ pub fn dispatch_irq(scause: usize) { trace!("IRQ: timer"); TIMER_HANDLER(); }, - @EXT => crate::irq::dispatch_irq_common(0), // TODO: get IRQ number from PLIC + @IPI => { + trace!("IRQ: IPI"); + IPI_HANDLER(); + }, + @EXT => { + if let Some(irq_num) = PLIC.lock().claim(Context::new(this_cpu_id())) { + let irq_num = irq_num.get() as usize; + trace!("IRQ: external {}", irq_num); + crate::irq::dispatch_irq_common(irq_num); + PLIC.lock().complete(Context::new(this_cpu_id()), Interrupt { irq_num }); + } + }, ); } pub(super) fn init_percpu() { + PLIC.lock().init_by_context(Context::new(this_cpu_id())); + // enable soft interrupts, timer interrupts, and external interrupts unsafe { sie::set_ssoft(); diff --git a/modules/axipi/src/queue.rs b/modules/axipi/src/queue.rs index 724754e50f..d926fe5107 100644 --- a/modules/axipi/src/queue.rs +++ b/modules/axipi/src/queue.rs @@ -10,7 +10,7 @@ pub struct IPIEventQueue { } /// A trait that all events must implement. -pub trait IPIEvent: 'static { +pub trait IPIEvent: Sized { /// Callback function that will be called when the event is triggered. fn callback(self); } @@ -60,11 +60,11 @@ impl Default for IPIEventQueue { /// /// So that it can be used as in the [`IPIEventQueue`]. #[derive(Clone)] -pub struct IPIEventFn(Box<&'static dyn Fn()>); +pub struct IPIEventFn<'a>(Box<&'a dyn Fn()>); -impl IPIEventFn { +impl<'a> IPIEventFn<'a> { /// Constructs a new [`IPIEventFn`] from a closure. - pub fn new(f: &'static F) -> Self + pub fn new(f: &'a F) -> Self where F: Fn(), { @@ -72,7 +72,7 @@ impl IPIEventFn { } } -impl IPIEvent for IPIEventFn { +impl<'a> IPIEvent for IPIEventFn<'a> { fn callback(self) { (self.0)() } diff --git a/qemu.dts b/qemu.dts new file mode 100644 index 0000000000..f1efae6d64 --- /dev/null +++ b/qemu.dts @@ -0,0 +1,211 @@ +/dts-v1/; + +/ { + #address-cells = <0x02>; + #size-cells = <0x02>; + compatible = "riscv-virtio"; + model = "riscv-virtio,qemu"; + + poweroff { + value = <0x5555>; + offset = <0x00>; + regmap = <0x04>; + compatible = "syscon-poweroff"; + }; + + reboot { + value = <0x7777>; + offset = <0x00>; + regmap = <0x04>; + compatible = "syscon-reboot"; + }; + + platform-bus@4000000 { + interrupt-parent = <0x03>; + ranges = <0x00 0x00 0x4000000 0x2000000>; + #address-cells = <0x01>; + #size-cells = <0x01>; + compatible = "qemu,platform\0simple-bus"; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x00 0x80000000 0x00 0x80000000>; + }; + + cpus { + #address-cells = <0x01>; + #size-cells = <0x00>; + timebase-frequency = <0x989680>; + + cpu@0 { + phandle = <0x01>; + device_type = "cpu"; + reg = <0x00>; + status = "okay"; + compatible = "riscv"; + riscv,cbop-block-size = <0x40>; + riscv,cboz-block-size = <0x40>; + riscv,cbom-block-size = <0x40>; + riscv,isa-extensions = "i\0m\0a\0f\0d\0c\0h\0zic64b\0zicbom\0zicbop\0zicboz\0ziccamoa\0ziccif\0zicclsm\0ziccrse\0zicntr\0zicsr\0zifencei\0zihintntl\0zihintpause\0zihpm\0za64rs\0zawrs\0zfa\0zca\0zcd\0zba\0zbb\0zbc\0zbs\0ssccptr\0sscounterenw\0sstc\0sstvala\0sstvecd\0svadu"; + riscv,isa-base = "rv64i"; + riscv,isa = "rv64imafdch_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zicsr_zifencei_zihintntl_zihintpause_zihpm_za64rs_zawrs_zfa_zca_zcd_zba_zbb_zbc_zbs_ssccptr_sscounterenw_sstc_sstvala_sstvecd_svadu"; + mmu-type = "riscv,sv57"; + + interrupt-controller { + #interrupt-cells = <0x01>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + phandle = <0x02>; + }; + }; + + cpu-map { + + cluster0 { + + core0 { + cpu = <0x01>; + }; + }; + }; + }; + + pmu { + riscv,event-to-mhpmcounters = <0x01 0x01 0x7fff9 0x02 0x02 0x7fffc 0x10019 0x10019 0x7fff8 0x1001b 0x1001b 0x7fff8 0x10021 0x10021 0x7fff8>; + compatible = "riscv,pmu"; + }; + + fw-cfg@10100000 { + dma-coherent; + reg = <0x00 0x10100000 0x00 0x18>; + compatible = "qemu,fw-cfg-mmio"; + }; + + flash@20000000 { + bank-width = <0x04>; + reg = <0x00 0x20000000 0x00 0x2000000 0x00 0x22000000 0x00 0x2000000>; + compatible = "cfi-flash"; + }; + + chosen { + stdout-path = "/soc/serial@10000000"; + rng-seed = <0xf57e47f3 0x8a852b2a 0xa547abb4 0x830202d1 0xab9aef61 0xa2c692fe 0xb55bb2c5 0x63aea82d>; + }; + + soc { + #address-cells = <0x02>; + #size-cells = <0x02>; + compatible = "simple-bus"; + ranges; + + rtc@101000 { + interrupts = <0x0b>; + interrupt-parent = <0x03>; + reg = <0x00 0x101000 0x00 0x1000>; + compatible = "google,goldfish-rtc"; + }; + + serial@10000000 { + interrupts = <0x0a>; + interrupt-parent = <0x03>; + clock-frequency = "\08@"; + reg = <0x00 0x10000000 0x00 0x100>; + compatible = "ns16550a"; + }; + + test@100000 { + phandle = <0x04>; + reg = <0x00 0x100000 0x00 0x1000>; + compatible = "sifive,test1\0sifive,test0\0syscon"; + }; + + virtio_mmio@10008000 { + interrupts = <0x08>; + interrupt-parent = <0x03>; + reg = <0x00 0x10008000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10007000 { + interrupts = <0x07>; + interrupt-parent = <0x03>; + reg = <0x00 0x10007000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10006000 { + interrupts = <0x06>; + interrupt-parent = <0x03>; + reg = <0x00 0x10006000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10005000 { + interrupts = <0x05>; + interrupt-parent = <0x03>; + reg = <0x00 0x10005000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10004000 { + interrupts = <0x04>; + interrupt-parent = <0x03>; + reg = <0x00 0x10004000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10003000 { + interrupts = <0x03>; + interrupt-parent = <0x03>; + reg = <0x00 0x10003000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10002000 { + interrupts = <0x02>; + interrupt-parent = <0x03>; + reg = <0x00 0x10002000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10001000 { + interrupts = <0x01>; + interrupt-parent = <0x03>; + reg = <0x00 0x10001000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + plic@c000000 { + phandle = <0x03>; + riscv,ndev = <0x5f>; + reg = <0x00 0xc000000 0x00 0x600000>; + interrupts-extended = <0x02 0x0b 0x02 0x09>; + interrupt-controller; + compatible = "sifive,plic-1.0.0\0riscv,plic0"; + #address-cells = <0x00>; + #interrupt-cells = <0x01>; + }; + + clint@2000000 { + interrupts-extended = <0x02 0x03 0x02 0x07>; + reg = <0x00 0x2000000 0x00 0x10000>; + compatible = "sifive,clint0\0riscv,clint0"; + }; + + pci@30000000 { + interrupt-map-mask = <0x1800 0x00 0x00 0x07>; + interrupt-map = <0x00 0x00 0x00 0x01 0x03 0x20 0x00 0x00 0x00 0x02 0x03 0x21 0x00 0x00 0x00 0x03 0x03 0x22 0x00 0x00 0x00 0x04 0x03 0x23 0x800 0x00 0x00 0x01 0x03 0x21 0x800 0x00 0x00 0x02 0x03 0x22 0x800 0x00 0x00 0x03 0x03 0x23 0x800 0x00 0x00 0x04 0x03 0x20 0x1000 0x00 0x00 0x01 0x03 0x22 0x1000 0x00 0x00 0x02 0x03 0x23 0x1000 0x00 0x00 0x03 0x03 0x20 0x1000 0x00 0x00 0x04 0x03 0x21 0x1800 0x00 0x00 0x01 0x03 0x23 0x1800 0x00 0x00 0x02 0x03 0x20 0x1800 0x00 0x00 0x03 0x03 0x21 0x1800 0x00 0x00 0x04 0x03 0x22>; + ranges = <0x1000000 0x00 0x00 0x00 0x3000000 0x00 0x10000 0x2000000 0x00 0x40000000 0x00 0x40000000 0x00 0x40000000 0x3000000 0x04 0x00 0x04 0x00 0x04 0x00>; + reg = <0x00 0x30000000 0x00 0x10000000>; + dma-coherent; + bus-range = <0x00 0xff>; + linux,pci-domain = <0x00>; + device_type = "pci"; + compatible = "pci-host-ecam-generic"; + #size-cells = <0x02>; + #interrupt-cells = <0x01>; + #address-cells = <0x03>; + }; + }; +}; diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index 2f3ce0d41d..d9e504f632 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -67,7 +67,7 @@ driver-ixgbe = ["axfeat/driver-ixgbe"] driver-bcm2835-sdhci = ["axfeat/driver-bcm2835-sdhci"] # Hypervisor support -hv = ["axfeat/hv", "axfeat/ipi"] +hv = ["axfeat/hv", "axfeat/ipi", "arceos_api/ipi"] # Logging log-level-off = ["axfeat/log-level-off"] From 1389f01b3248764c7d92ee14d309691f283bb554 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 17 Dec 2024 22:43:08 +0800 Subject: [PATCH 13/48] fix x86 ipi vector --- modules/axhal/src/platform/x86_pc/apic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/axhal/src/platform/x86_pc/apic.rs b/modules/axhal/src/platform/x86_pc/apic.rs index 835919b0f9..1836ce3bec 100644 --- a/modules/axhal/src/platform/x86_pc/apic.rs +++ b/modules/axhal/src/platform/x86_pc/apic.rs @@ -16,7 +16,7 @@ pub(super) mod vectors { pub const APIC_TIMER_VECTOR: u8 = 0xf0; pub const APIC_SPURIOUS_VECTOR: u8 = 0xf1; pub const APIC_ERROR_VECTOR: u8 = 0xf2; - pub const APIC_IPI_VECTOR: u8 = 0xf2; + pub const APIC_IPI_VECTOR: u8 = 0xf3; } /// The maximum number of IRQs. From 56bf973188e2b0535d3c5f35f196e7e9451b4233 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 18 Dec 2024 12:28:04 +0800 Subject: [PATCH 14/48] modify axipi to support more callback functions, add "ipi" features to necessary crates --- Cargo.lock | 16 +++++------ api/arceos_api/Cargo.toml | 1 + api/arceos_api/src/lib.rs | 2 ++ api/axfeat/Cargo.toml | 1 + api/axfeat/src/lib.rs | 1 + modules/axhal/Cargo.toml | 2 +- modules/axipi/src/event.rs | 52 +++++++++++++++++++++++++++++++++ modules/axipi/src/lib.rs | 19 ++++++------ modules/axipi/src/queue.rs | 59 ++++++++++---------------------------- ulib/axstd/Cargo.toml | 1 + 10 files changed, 92 insertions(+), 62 deletions(-) create mode 100644 modules/axipi/src/event.rs diff --git a/Cargo.lock b/Cargo.lock index 384a758a70..c4ad3199b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -471,7 +471,7 @@ dependencies = [ "tock-registers", "x2apic", "x86", - "x86_64 0.15.2", + "x86_64 0.15.1", "x86_rtc", ] @@ -1258,7 +1258,7 @@ dependencies = [ "aarch64-cpu", "bitflags 2.6.0", "memory_addr", - "x86_64 0.15.2", + "x86_64 0.15.1", ] [[package]] @@ -1914,7 +1914,7 @@ dependencies = [ "bitflags 1.3.2", "paste", "raw-cpuid 10.7.0", - "x86_64 0.14.13", + "x86_64 0.14.12", ] [[package]] @@ -1930,9 +1930,9 @@ dependencies = [ [[package]] name = "x86_64" -version = "0.14.13" +version = "0.14.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c101112411baafbb4bf8d33e4c4a80ab5b02d74d2612331c61e8192fc9710491" +checksum = "96cb6fd45bfeab6a5055c5bffdb08768bd0c069f1d946debe585bbb380a7c062" dependencies = [ "bit_field", "bitflags 2.6.0", @@ -1942,9 +1942,9 @@ dependencies = [ [[package]] name = "x86_64" -version = "0.15.2" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f042214de98141e9c8706e8192b73f56494087cc55ebec28ce10f26c5c364ae" +checksum = "4bc79523af8abf92fb1a970c3e086c5a343f6bcc1a0eb890f575cbb3b45743df" dependencies = [ "bit_field", "bitflags 2.6.0", @@ -1959,7 +1959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1a42420da20c01d82e5d42231570efa3b9e16a5515eaaf9ee4e964f49cc1313" dependencies = [ "cfg-if", - "x86_64 0.15.2", + "x86_64 0.15.1", ] [[package]] diff --git a/api/arceos_api/Cargo.toml b/api/arceos_api/Cargo.toml index e7db29ee11..549a4ae3a3 100644 --- a/api/arceos_api/Cargo.toml +++ b/api/arceos_api/Cargo.toml @@ -21,6 +21,7 @@ multitask = ["axtask/multitask", "axsync/multitask", "axfeat/multitask"] fs = ["dep:axfs", "dep:axdriver", "axfeat/fs"] net = ["dep:axnet", "dep:axdriver", "axfeat/net"] display = ["dep:axdisplay", "dep:axdriver", "axfeat/display"] +ipi = ["dep:axipi", "axfeat/ipi"] myfs = ["axfeat/myfs"] diff --git a/api/arceos_api/src/lib.rs b/api/arceos_api/src/lib.rs index 7ec14731bb..adb762fcc3 100644 --- a/api/arceos_api/src/lib.rs +++ b/api/arceos_api/src/lib.rs @@ -410,4 +410,6 @@ pub mod modules { pub use axnet; #[cfg(feature = "multitask")] pub use axtask; + #[cfg(feature = "ipi")] + pub use axipi; } diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index d380218482..c11e156021 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -83,3 +83,4 @@ axsync = { workspace = true, optional = true } axtask = { workspace = true, optional = true } axipi = { workspace = true, optional = true } kspin = { version = "0.1", optional = true } +axipi = { workspace = true, optional = true } diff --git a/api/axfeat/src/lib.rs b/api/axfeat/src/lib.rs index 85fc2abb26..44a1ba57dc 100644 --- a/api/axfeat/src/lib.rs +++ b/api/axfeat/src/lib.rs @@ -7,6 +7,7 @@ //! - `fp_simd`: Enable floating point and SIMD support. //! - Interrupts: //! - `irq`: Enable interrupt handling support. +//! - `ipi`: Enable Inter-Processor Interrupts (IPIs). //! - Memory //! - `alloc`: Enable dynamic memory allocation. //! - `alloc-tlsf`: Use the TLSF allocator. diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index 4f03c2515e..e16c550a1f 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -19,7 +19,7 @@ tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] default = [] hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "arm_gicv2/el2", "dep:crate_interface"] -ipi = [] +ipi = ["irq"] [dependencies] log = "=0.4.21" diff --git a/modules/axipi/src/event.rs b/modules/axipi/src/event.rs new file mode 100644 index 0000000000..c9a8e1e49d --- /dev/null +++ b/modules/axipi/src/event.rs @@ -0,0 +1,52 @@ +use alloc::{boxed::Box, sync::Arc}; + +/// A callback function that will be called when an [`IPIEvent`] is received and handled. +pub struct Callback(Box); + +impl Callback { + /// Create a new [`Callback`] with the given function. + pub fn new(callback: F) -> Self { + Self(Box::new(callback)) + } + + /// Call the callback function. + pub fn call(self) { + (self.0)() + } +} + +impl From for Callback { + fn from(callback: T) -> Self { + Self::new(callback) + } +} + +/// A [`Callback`] that can be called multiple times. It's used for multicast IPI events. +#[derive(Clone)] +pub struct MulticastCallback(Arc); + +impl MulticastCallback { + /// Create a new [`MulticastCallback`] with the given function. + pub fn new(callback: F) -> Self { + Self(Arc::new(callback)) + } + + /// Convert the [`MulticastCallback`] into a [`Callback`]. + pub fn into_unicast(self) -> Callback { + Callback(Box::new(move || { + (self.0)() + })) + } +} + +impl From for MulticastCallback { + fn from(callback: T) -> Self { + Self::new(callback) + } +} + +/// An IPI event that is sent from a source CPU to the target CPU. +pub struct IPIEvent { + pub src_cpu_id: usize, + pub callback: Callback, +} \ No newline at end of file diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs index 04a851d6a9..8e3add058b 100644 --- a/modules/axipi/src/lib.rs +++ b/modules/axipi/src/lib.rs @@ -13,14 +13,14 @@ use kspin::SpinNoIrq; use axhal::cpu::this_cpu_id; use axhal::irq::IPI_IRQ_NUM; +mod event; mod queue; use queue::IPIEventQueue; - -pub use queue::{IPIEvent, IPIEventFn}; +pub use event::*; #[percpu::def_percpu] -static IPI_EVENT_QUEUE: LazyInit>> = LazyInit::new(); +static IPI_EVENT_QUEUE: LazyInit> = LazyInit::new(); /// Initialize the per-CPU IPI event queue. pub fn init() { @@ -30,34 +30,35 @@ pub fn init() { } /// Sends an IPI event to the processor(s) specified by `dest_cpu`. -pub fn send_ipi_event_to_one(dest_cpu: usize, event: IPIEventFn) { +pub fn send_ipi_event_to_one>(dest_cpu: usize, callback: T) { warn!("Send IPI event to CPU {}", dest_cpu); unsafe { IPI_EVENT_QUEUE.remote_ref_raw(dest_cpu) } .lock() - .push(this_cpu_id(), event); + .push(this_cpu_id(), callback.into()); axhal::irq::send_sgi_one(dest_cpu, IPI_IRQ_NUM); } /// Sends an IPI event to all processors except the current one. -pub fn send_ipi_event_to_all(event: IPIEventFn) { +pub fn send_ipi_event_to_all>(callback: T) { let current_cpu_id = this_cpu_id(); + let callback = callback.into(); for cpu_id in 0..axconfig::SMP { if cpu_id != current_cpu_id { unsafe { IPI_EVENT_QUEUE.remote_ref_raw(cpu_id) } .lock() - .push(current_cpu_id, event.clone()); + .push(current_cpu_id, callback.clone().into_unicast()); } } axhal::irq::send_sgi_all(IPI_IRQ_NUM); } pub fn ipi_handler() { - while let Some((src_cpu_id, event)) = unsafe { IPI_EVENT_QUEUE.current_ref_mut_raw() } + while let Some((src_cpu_id, callback)) = unsafe { IPI_EVENT_QUEUE.current_ref_mut_raw() } .lock() .pop_one() { warn!("Received IPI event from CPU {}", src_cpu_id); - event.callback(); + callback.call(); } } diff --git a/modules/axipi/src/queue.rs b/modules/axipi/src/queue.rs index d926fe5107..720bbaf03d 100644 --- a/modules/axipi/src/queue.rs +++ b/modules/axipi/src/queue.rs @@ -1,27 +1,17 @@ -use alloc::boxed::Box; use alloc::collections::VecDeque; +use crate::{Callback, IPIEvent}; + /// A queue of IPI events. /// /// It internally uses a `VecDeque` to store the events, make it /// possible to pop these events using FIFO order. -pub struct IPIEventQueue { - events: VecDeque>, -} - -/// A trait that all events must implement. -pub trait IPIEvent: Sized { - /// Callback function that will be called when the event is triggered. - fn callback(self); +pub struct IPIEventQueue { + events: VecDeque, } -struct IPIEventWrapper { - src_cpu_id: usize, - event: E, -} - -impl IPIEventQueue { - /// Creates a new empty timer list. +impl IPIEventQueue { + /// Create a new empty timer list. pub fn new() -> Self { Self { events: VecDeque::new(), @@ -29,51 +19,32 @@ impl IPIEventQueue { } /// Whether there is no event. + #[allow(dead_code)] #[inline] pub fn is_empty(&self) -> bool { self.events.is_empty() } - pub fn push(&mut self, src_cpu_id: usize, event: E) { - self.events.push_back(IPIEventWrapper { src_cpu_id, event }); + /// Push a new event into the queue. + pub fn push(&mut self, src_cpu_id: usize, callback: Callback) { + self.events.push_back(IPIEvent { src_cpu_id, callback }); } /// Try to pop the latest event that exists in the queue. /// - /// Returns `None` if no event is available. - pub fn pop_one(&mut self) -> Option<(usize, E)> { + /// Return `None` if no event is available. + #[must_use] + pub fn pop_one(&mut self) -> Option<(usize, Callback)> { if let Some(e) = self.events.pop_front() { - Some((e.src_cpu_id, e.event)) + Some((e.src_cpu_id, e.callback)) } else { None } } } -impl Default for IPIEventQueue { +impl Default for IPIEventQueue { fn default() -> Self { Self::new() } } - -/// A simple wrapper of a closure that implements the [`IPIEvent`] trait. -/// -/// So that it can be used as in the [`IPIEventQueue`]. -#[derive(Clone)] -pub struct IPIEventFn<'a>(Box<&'a dyn Fn()>); - -impl<'a> IPIEventFn<'a> { - /// Constructs a new [`IPIEventFn`] from a closure. - pub fn new(f: &'a F) -> Self - where - F: Fn(), - { - Self(Box::new(f)) - } -} - -impl<'a> IPIEvent for IPIEventFn<'a> { - fn callback(self) { - (self.0)() - } -} diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index d9e504f632..18221d71d9 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -27,6 +27,7 @@ fp_simd = ["axfeat/fp_simd"] # Interrupts irq = ["arceos_api/irq", "axfeat/irq"] +ipi = ["arceos_api/ipi", "axfeat/ipi"] # Memory alloc = ["arceos_api/alloc", "axfeat/alloc", "axio/alloc"] From e0dab0a215a2b08028c4b3b93ee7c9734044b9f4 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 18 Dec 2024 12:31:08 +0800 Subject: [PATCH 15/48] update `percpu` --- modules/axipi/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/axipi/Cargo.toml b/modules/axipi/Cargo.toml index 3823e549d6..9a51db4f7e 100644 --- a/modules/axipi/Cargo.toml +++ b/modules/axipi/Cargo.toml @@ -16,7 +16,7 @@ default = [] log = "=0.4.21" lazyinit = { version = "0.2" } -percpu = { version = "0.1.5" } +percpu = { version = "0.1.6" } kspin = { version = "0.1" } axhal = { workspace = true } From e00b8d9a01b1da92d9251ebef55563e9b25eac87 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 18 Dec 2024 12:46:20 +0800 Subject: [PATCH 16/48] add ipi-related stubs to dummy platform --- modules/axhal/src/platform/dummy/mod.rs | 9 +++++++++ modules/axipi/Cargo.toml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/axhal/src/platform/dummy/mod.rs b/modules/axhal/src/platform/dummy/mod.rs index 616ca47e7d..13ab8b0200 100644 --- a/modules/axhal/src/platform/dummy/mod.rs +++ b/modules/axhal/src/platform/dummy/mod.rs @@ -88,6 +88,15 @@ pub mod irq { pub fn fetch_irq() -> usize { 0 } + + #[cfg(feature = "ipi")] + pub const IPI_IRQ_NUM: usize = 0; + + #[cfg(feature = "ipi")] + pub fn send_sgi_one(dest_cpu: usize, irq_num: usize) {} + + #[cfg(feature = "ipi")] + pub fn send_sgi_all(irq_num: usize) {} } /// Initializes the platform devices for the primary CPU. diff --git a/modules/axipi/Cargo.toml b/modules/axipi/Cargo.toml index 9a51db4f7e..3ba21b1149 100644 --- a/modules/axipi/Cargo.toml +++ b/modules/axipi/Cargo.toml @@ -19,5 +19,5 @@ lazyinit = { version = "0.2" } percpu = { version = "0.1.6" } kspin = { version = "0.1" } -axhal = { workspace = true } +axhal = { workspace = true, features = ["ipi"] } axconfig = { workspace = true } \ No newline at end of file From 3ad223ce467bf3dcaa89d1dd5a19120c76b4dabb Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 18 Dec 2024 12:52:28 +0800 Subject: [PATCH 17/48] add stubs for riscv64 --- .../axhal/src/platform/riscv64_qemu_virt/irq.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index d3b7192c0c..634a7f02de 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -177,3 +177,19 @@ pub(super) fn init_percpu() { sie::set_sext(); } } + +#[cfg(feature = "ipi")] +mod ipi { + pub const IPI_IRQ_NUM: usize = 0; + + pub fn send_sgi_one(_dest_cpu: usize, _irq_num: usize) { + unimplemented!() + } + + pub fn send_sgi_all(_irq_num: usize) { + unimplemented!() + } +} + +#[cfg(feature = "ipi")] +pub use ipi::*; From e850d785c5048555164e71a963bd49bdcac693f6 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 18 Dec 2024 12:55:11 +0800 Subject: [PATCH 18/48] formatted --- api/arceos_api/src/lib.rs | 4 ++-- modules/axipi/src/event.rs | 6 ++---- modules/axipi/src/lib.rs | 2 +- modules/axipi/src/queue.rs | 5 ++++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/arceos_api/src/lib.rs b/api/arceos_api/src/lib.rs index adb762fcc3..9f2a571ea0 100644 --- a/api/arceos_api/src/lib.rs +++ b/api/arceos_api/src/lib.rs @@ -404,12 +404,12 @@ pub mod modules { pub use axdriver; #[cfg(feature = "fs")] pub use axfs; + #[cfg(feature = "ipi")] + pub use axipi; #[cfg(feature = "paging")] pub use axmm; #[cfg(feature = "net")] pub use axnet; #[cfg(feature = "multitask")] pub use axtask; - #[cfg(feature = "ipi")] - pub use axipi; } diff --git a/modules/axipi/src/event.rs b/modules/axipi/src/event.rs index c9a8e1e49d..ff51ebad4d 100644 --- a/modules/axipi/src/event.rs +++ b/modules/axipi/src/event.rs @@ -33,9 +33,7 @@ impl MulticastCallback { /// Convert the [`MulticastCallback`] into a [`Callback`]. pub fn into_unicast(self) -> Callback { - Callback(Box::new(move || { - (self.0)() - })) + Callback(Box::new(move || (self.0)())) } } @@ -49,4 +47,4 @@ impl From for MulticastCallback { pub struct IPIEvent { pub src_cpu_id: usize, pub callback: Callback, -} \ No newline at end of file +} diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs index 8e3add058b..ebfdaa0723 100644 --- a/modules/axipi/src/lib.rs +++ b/modules/axipi/src/lib.rs @@ -16,8 +16,8 @@ use axhal::irq::IPI_IRQ_NUM; mod event; mod queue; -use queue::IPIEventQueue; pub use event::*; +use queue::IPIEventQueue; #[percpu::def_percpu] static IPI_EVENT_QUEUE: LazyInit> = LazyInit::new(); diff --git a/modules/axipi/src/queue.rs b/modules/axipi/src/queue.rs index 720bbaf03d..ddd300d618 100644 --- a/modules/axipi/src/queue.rs +++ b/modules/axipi/src/queue.rs @@ -27,7 +27,10 @@ impl IPIEventQueue { /// Push a new event into the queue. pub fn push(&mut self, src_cpu_id: usize, callback: Callback) { - self.events.push_back(IPIEvent { src_cpu_id, callback }); + self.events.push_back(IPIEvent { + src_cpu_id, + callback, + }); } /// Try to pop the latest event that exists in the queue. From fb252567bd3890e6d72d18c47682e5ae8f1e27fd Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 18 Dec 2024 16:45:01 +0800 Subject: [PATCH 19/48] fixes after cherry-pick --- api/arceos_api/Cargo.toml | 1 - api/arceos_api/src/lib.rs | 2 -- api/axfeat/Cargo.toml | 1 - examples/helloworld/Cargo.toml | 2 +- modules/axhal/src/platform/mod.rs | 8 +++---- .../src/platform/riscv64_qemu_virt/irq.rs | 22 +++---------------- 6 files changed, 8 insertions(+), 28 deletions(-) diff --git a/api/arceos_api/Cargo.toml b/api/arceos_api/Cargo.toml index 549a4ae3a3..e7db29ee11 100644 --- a/api/arceos_api/Cargo.toml +++ b/api/arceos_api/Cargo.toml @@ -21,7 +21,6 @@ multitask = ["axtask/multitask", "axsync/multitask", "axfeat/multitask"] fs = ["dep:axfs", "dep:axdriver", "axfeat/fs"] net = ["dep:axnet", "dep:axdriver", "axfeat/net"] display = ["dep:axdisplay", "dep:axdriver", "axfeat/display"] -ipi = ["dep:axipi", "axfeat/ipi"] myfs = ["axfeat/myfs"] diff --git a/api/arceos_api/src/lib.rs b/api/arceos_api/src/lib.rs index 9f2a571ea0..95658a17e6 100644 --- a/api/arceos_api/src/lib.rs +++ b/api/arceos_api/src/lib.rs @@ -392,8 +392,6 @@ pub mod modules { pub use axruntime; pub use axsync; - #[cfg(feature = "ipi")] - pub use axipi; #[cfg(feature = "alloc")] pub use axalloc; #[cfg(feature = "display")] diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index c11e156021..d380218482 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -83,4 +83,3 @@ axsync = { workspace = true, optional = true } axtask = { workspace = true, optional = true } axipi = { workspace = true, optional = true } kspin = { version = "0.1", optional = true } -axipi = { workspace = true, optional = true } diff --git a/examples/helloworld/Cargo.toml b/examples/helloworld/Cargo.toml index 8db0ea2300..30ccb889d3 100644 --- a/examples/helloworld/Cargo.toml +++ b/examples/helloworld/Cargo.toml @@ -7,4 +7,4 @@ authors = ["Yuekai Jia "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axstd = { workspace = true,features = ["irq"], optional = true } +axstd = { workspace = true, features = ["irq"], optional = true } diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 411d018a4e..16663b0dca 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -29,9 +29,9 @@ cfg_if::cfg_if! { mod aarch64_rk3588j; pub use self::aarch64_rk3588j::*; } else { - // mod dummy; - // pub use self::dummy::*; - mod riscv64_qemu_virt; - pub use self::riscv64_qemu_virt::*; + mod dummy; + pub use self::dummy::*; + // mod riscv64_qemu_virt; + // pub use self::riscv64_qemu_virt::*; } } diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index 634a7f02de..feaeabaeae 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -66,9 +66,9 @@ pub const TIMER_IRQ_NUM: usize = S_TIMER; pub const IPI_IRQ_NUM: usize = S_SOFT; macro_rules! with_cause { - ($cause: expr, - @TIMER => $timer_op: expr, - @IPI => $ipi_op: expr, + ($cause: expr, + @TIMER => $timer_op: expr, + @IPI => $ipi_op: expr, @EXT => $ext_op: expr $(,)?) => { match $cause { S_TIMER => $timer_op, @@ -177,19 +177,3 @@ pub(super) fn init_percpu() { sie::set_sext(); } } - -#[cfg(feature = "ipi")] -mod ipi { - pub const IPI_IRQ_NUM: usize = 0; - - pub fn send_sgi_one(_dest_cpu: usize, _irq_num: usize) { - unimplemented!() - } - - pub fn send_sgi_all(_irq_num: usize) { - unimplemented!() - } -} - -#[cfg(feature = "ipi")] -pub use ipi::*; From 00d43115395f428ce8b1724f2ed9c692dfda600c Mon Sep 17 00:00:00 2001 From: aarkegz Date: Thu, 9 Jan 2025 15:00:20 +0800 Subject: [PATCH 20/48] fixes after rebase --- Cargo.lock | 16 ++--- .../src/platform/aarch64_common/boot_el2.rs | 66 ++++++++++--------- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4ad3199b7..384a758a70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -471,7 +471,7 @@ dependencies = [ "tock-registers", "x2apic", "x86", - "x86_64 0.15.1", + "x86_64 0.15.2", "x86_rtc", ] @@ -1258,7 +1258,7 @@ dependencies = [ "aarch64-cpu", "bitflags 2.6.0", "memory_addr", - "x86_64 0.15.1", + "x86_64 0.15.2", ] [[package]] @@ -1914,7 +1914,7 @@ dependencies = [ "bitflags 1.3.2", "paste", "raw-cpuid 10.7.0", - "x86_64 0.14.12", + "x86_64 0.14.13", ] [[package]] @@ -1930,9 +1930,9 @@ dependencies = [ [[package]] name = "x86_64" -version = "0.14.12" +version = "0.14.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96cb6fd45bfeab6a5055c5bffdb08768bd0c069f1d946debe585bbb380a7c062" +checksum = "c101112411baafbb4bf8d33e4c4a80ab5b02d74d2612331c61e8192fc9710491" dependencies = [ "bit_field", "bitflags 2.6.0", @@ -1942,9 +1942,9 @@ dependencies = [ [[package]] name = "x86_64" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc79523af8abf92fb1a970c3e086c5a343f6bcc1a0eb890f575cbb3b45743df" +checksum = "0f042214de98141e9c8706e8192b73f56494087cc55ebec28ce10f26c5c364ae" dependencies = [ "bit_field", "bitflags 2.6.0", @@ -1959,7 +1959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1a42420da20c01d82e5d42231570efa3b9e16a5515eaaf9ee4e964f49cc1313" dependencies = [ "cfg-if", - "x86_64 0.15.1", + "x86_64 0.15.2", ] [[package]] diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index fd15e6ac36..931a00e991 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -7,13 +7,13 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::TASK_STACK_SIZE; -#[link_section = ".bss.stack"] +#[unsafe(link_section = ".bss.stack")] static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; -#[link_section = ".data.boot_page_table"] +#[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; -#[link_section = ".data.boot_page_table"] +#[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; unsafe fn switch_to_el2() { @@ -67,7 +67,7 @@ unsafe fn init_mmu_el2() { TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); barrier::isb(barrier::SY); - let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + let root_paddr = PhysAddr::from(&raw const BOOT_PT_L0 as usize).as_usize() as _; TTBR0_EL2.set(root_paddr); // Flush the entire TLB @@ -126,12 +126,16 @@ unsafe fn cache_invalidate(cache_level: usize) { /// The earliest entry point for the primary CPU. #[naked] -#[no_mangle] -#[link_section = ".text.boot"] +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.boot")] unsafe extern "C" fn _start() -> ! { - // PC = 0x8_0000 - // X0 = dtb - core::arch::asm!(" + unsafe { + // PC = 0x8_0000 + // X0 = dtb + core::arch::naked_asm!(" + // save DTB pointer + mov x20, x0 + // disable cache and MMU mrs x1, sctlr_el2 bic x1, x1, #0xf @@ -145,7 +149,6 @@ unsafe extern "C" fn _start() -> ! { mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id - mov x20, x0 // save DTB pointer adrp x8, {boot_stack} // setup boot stack add x8, x8, {boot_stack_size} @@ -164,26 +167,27 @@ unsafe extern "C" fn _start() -> ! { ldr x8, ={entry} blr x8 b .", - cache_invalidate = sym cache_invalidate, - init_boot_page_table = sym init_boot_page_table, - init_mmu_el2 = sym init_mmu_el2, - switch_to_el2 = sym switch_to_el2, - enable_fp = sym enable_fp, - boot_stack = sym BOOT_STACK, - boot_stack_size = const TASK_STACK_SIZE, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry, - options(noreturn), - ); + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::plat::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + ); + } } /// The earliest entry point for the secondary CPUs. #[cfg(feature = "smp")] #[naked] -#[no_mangle] -#[link_section = ".text.boot"] +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.boot")] unsafe extern "C" fn _start_secondary() -> ! { - core::arch::asm!(" + unsafe { + core::arch::naked_asm!(" mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id @@ -199,11 +203,11 @@ unsafe extern "C" fn _start_secondary() -> ! { ldr x8, ={entry} blr x8 b .", - switch_to_el2 = sym switch_to_el2, - init_mmu_el2 = sym init_mmu_el2, - enable_fp = sym enable_fp, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry_secondary, - options(noreturn), - ) + switch_to_el2 = sym switch_to_el2, + init_mmu_el2 = sym init_mmu_el2, + enable_fp = sym enable_fp, + phys_virt_offset = const axconfig::plat::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry_secondary, + ) + } } From 5795fc2ea7896c6314f770c32a5e0d1263a32371 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Thu, 9 Jan 2025 15:52:07 +0800 Subject: [PATCH 21/48] disable plic temporarily --- .../src/platform/riscv64_qemu_virt/irq.rs | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index feaeabaeae..ea25090564 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -1,10 +1,10 @@ -//! TODO: PLIC +//! Minimal PLIC support temporarily disabled. use kspin::SpinNoIrq; use lazyinit::LazyInit; use riscv::register::sie; -use riscv_plic::{HartContext, InterruptSource, Plic}; +// use riscv_plic::{HartContext, InterruptSource, Plic}; use sbi_rt::{send_ipi, HartMask}; use memory_addr::{pa, PhysAddr}; @@ -13,31 +13,31 @@ use crate::cpu::this_cpu_id; use crate::irq::IrqHandler; use crate::mem::phys_to_virt; -struct Context { - hart_id: usize, -} +// struct Context { +// hart_id: usize, +// } -impl HartContext for Context { - fn index(self) -> usize { - self.hart_id * 2 + 1 - } -} +// impl HartContext for Context { +// fn index(self) -> usize { +// self.hart_id * 2 + 1 +// } +// } -impl Context { - const fn new(hart_id: usize) -> Self { - Self { hart_id } - } -} +// impl Context { +// const fn new(hart_id: usize) -> Self { +// Self { hart_id } +// } +// } -struct Interrupt { - irq_num: usize, -} +// struct Interrupt { +// irq_num: usize, +// } -impl InterruptSource for Interrupt { - fn id(self) -> core::num::NonZeroU32 { - core::num::NonZeroU32::new(self.irq_num as u32).unwrap() - } -} +// impl InterruptSource for Interrupt { +// fn id(self) -> core::num::NonZeroU32 { +// core::num::NonZeroU32::new(self.irq_num as u32).unwrap() +// } +// } /// `Interrupt` bit in `scause` pub(super) const INTC_IRQ_BASE: usize = 1 << (usize::BITS - 1); @@ -79,21 +79,21 @@ macro_rules! with_cause { }; } -const PLIC_BASE: PhysAddr = pa!(axconfig::PLIC_PADDR); +// const PLIC_BASE: PhysAddr = pa!(axconfig::PLIC_PADDR); -static PLIC: SpinNoIrq = SpinNoIrq::new(Plic::new(phys_to_virt(PLIC_BASE).as_mut_ptr())); +// static PLIC: SpinNoIrq = SpinNoIrq::new(Plic::new(phys_to_virt(PLIC_BASE).as_mut_ptr())); /// Enables or disables the given IRQ. pub fn set_enable(irq_num: usize, enabled: bool) { // TODO: set enable in PLIC - let source = Interrupt { irq_num }; - let context = Context::new(this_cpu_id()); - - if enabled { - PLIC.lock().enable(source, context); - } else { - PLIC.lock().disable(source, context); - } + // let source = Interrupt { irq_num }; + // let context = Context::new(this_cpu_id()); + + // if enabled { + // PLIC.lock().enable(source, context); + // } else { + // PLIC.lock().disable(source, context); + // } } /// Registers an IRQ handler for the given IRQ. @@ -157,18 +157,19 @@ pub fn dispatch_irq(scause: usize) { IPI_HANDLER(); }, @EXT => { - if let Some(irq_num) = PLIC.lock().claim(Context::new(this_cpu_id())) { - let irq_num = irq_num.get() as usize; - trace!("IRQ: external {}", irq_num); - crate::irq::dispatch_irq_common(irq_num); - PLIC.lock().complete(Context::new(this_cpu_id()), Interrupt { irq_num }); - } + // if let Some(irq_num) = PLIC.lock().claim(Context::new(this_cpu_id())) { + // let irq_num = irq_num.get() as usize; + // trace!("IRQ: external {}", irq_num); + // crate::irq::dispatch_irq_common(irq_num); + // PLIC.lock().complete(Context::new(this_cpu_id()), Interrupt { irq_num }); + // } + crate::irq::dispatch_irq_common(0); }, ); } pub(super) fn init_percpu() { - PLIC.lock().init_by_context(Context::new(this_cpu_id())); + // PLIC.lock().init_by_context(Context::new(this_cpu_id())); // enable soft interrupts, timer interrupts, and external interrupts unsafe { From 42c2aabf483f422707fc4f5951e37b059e1f5bae Mon Sep 17 00:00:00 2001 From: aarkegz Date: Thu, 9 Jan 2025 16:01:44 +0800 Subject: [PATCH 22/48] update some `Cargo.lock` entries --- Cargo.lock | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 384a758a70..d9a4f7a809 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,9 +49,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -79,21 +79,21 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -770,9 +770,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "9560b07a799281c7e0958b9296854d6fafd4c5f31444a7e5bb1ad6dde5ccf1bd" dependencies = [ "clap_builder", "clap_derive", @@ -780,9 +780,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "874e0dd3eb68bf99058751ac9712f622e61e6f393a94f7128fa26e3f02f5c7cd" dependencies = [ "anstream", "anstyle", @@ -792,9 +792,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck", "proc-macro2", @@ -1013,11 +1013,11 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "home" -version = "0.5.11" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] From 534de36838358553fa416808ba5b9c8306beab4d Mon Sep 17 00:00:00 2001 From: Debin Date: Mon, 31 Mar 2025 22:44:04 +0800 Subject: [PATCH 23/48] debin/timer_api --- Cargo.lock | 2 +- modules/axhal/Cargo.toml | 14 ++++- modules/axhal/src/irq.rs | 4 +- .../axhal/src/platform/aarch64_common/gic.rs | 60 ++++++++++++++----- 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4451e1ed15..5fc9eca8d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,7 +188,7 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arm_gicv2?branch=sgi#956665adcdc6b09a28078db09ad3f09ed41af80b" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2.git?branch=sgi#956665adcdc6b09a28078db09ad3f09ed41af80b" dependencies = [ "tock-registers 0.8.1", ] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index ae3f31183d..623865f02b 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -19,7 +19,15 @@ tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] uspace = ["paging"] default = [] -hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "arm_gicv2/el2", "dep:crate_interface"] +hv = [ + "paging", + "cortex-a", + "percpu/arm-el2", + "page_table_entry/arm-el2", + "arm_gicv2/el2", + "arm_gicv2/hv", + "dep:crate_interface", +] ipi = ["irq"] [dependencies] @@ -53,13 +61,13 @@ x86_rtc = { version = "0.1", optional = true } [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] riscv = "0.12" sbi-rt = { version = "0.0.3", features = ["legacy"] } -riscv_plic = { version = "0.1.0" } +riscv_plic = { version = "0.1.0" } riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "10.0" tock-registers = "0.9" -arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", branch = "sgi" } +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", branch = "debin/2vm_timer" } crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 3b8f2d4bce..7b0b629086 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -5,10 +5,10 @@ use handler_table::HandlerTable; use crate::platform::irq::{MAX_IRQ_COUNT, dispatch_irq}; use crate::trap::{IRQ, register_trap_handler}; -pub use crate::platform::irq::{register_handler, set_enable}; +pub use crate::platform::irq::{MyVgic, inject_interrupt, register_handler, set_enable}; #[cfg(feature = "ipi")] -pub use crate::platform::irq::{send_sgi_all, send_sgi_one, IPI_IRQ_NUM}; +pub use crate::platform::irq::{IPI_IRQ_NUM, send_sgi_all, send_sgi_one}; #[cfg(target_arch = "aarch64")] pub use crate::platform::irq::fetch_irq; diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 44cc865b1e..663562d883 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -1,9 +1,10 @@ use crate::{irq::IrqHandler, mem::phys_to_virt}; -use arm_gicv2::{GicCpuInterface, GicDistributor, InterruptType, translate_irq}; -use axconfig::devices::{GICC_PADDR, GICD_PADDR, UART_IRQ}; +use arm_gicv2::{ + GicCpuInterface, GicDistributor, GicHypervisorInterface, InterruptType, translate_irq, +}; +use axconfig::devices::{GICC_PADDR, GICD_PADDR, GICH_PADDR, GICV_PADDR, UART_IRQ}; use kspin::SpinNoIrq; use memory_addr::PhysAddr; - /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; @@ -17,18 +18,21 @@ pub const TIMER_IRQ_NUM: usize = translate_irq(10, InterruptType::PPI).unwrap(); /// The UART IRQ number. pub const UART_IRQ_NUM: usize = translate_irq(UART_IRQ, InterruptType::SPI).unwrap(); - /// The IPI IRQ number. pub const IPI_IRQ_NUM: usize = translate_irq(1, InterruptType::SGI).unwrap(); - const GICD_BASE: PhysAddr = pa!(GICD_PADDR); const GICC_BASE: PhysAddr = pa!(GICC_PADDR); +const GICV_BASE: PhysAddr = pa!(GICV_PADDR); +const GICH_BASE: PhysAddr = pa!(GICH_PADDR); static GICD: SpinNoIrq = SpinNoIrq::new(GicDistributor::new(phys_to_virt(GICD_BASE).as_mut_ptr())); // per-CPU, no lock static GICC: GicCpuInterface = GicCpuInterface::new(phys_to_virt(GICC_BASE).as_mut_ptr()); +static GICV: GicCpuInterface = GicCpuInterface::new(phys_to_virt(GICV_BASE).as_mut_ptr()); +static GICH: GicHypervisorInterface = + GicHypervisorInterface::new(phys_to_virt(GICH_BASE).as_mut_ptr()); /// Enables or disables the given IRQ. pub fn set_enable(irq_num: usize, enabled: bool) { @@ -36,15 +40,6 @@ pub fn set_enable(irq_num: usize, enabled: bool) { GICD.lock().set_enable(irq_num as _, enabled); } -/// Registers an IRQ handler for the given IRQ. -/// -/// It also enables the IRQ if the registration succeeds. It returns `false` if -/// the registration failed. -pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { - warn!("register handler irq {}", irq_num); - crate::irq::register_handler_common(irq_num, handler) -} - /// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { GICD.lock().send_sgi(dest_cpu_id, irq_num); @@ -55,6 +50,15 @@ pub fn send_sgi_all(irq_num: usize) { GICD.lock().send_sgi_all_except_self(irq_num); } +/// Registers an IRQ handler for the given IRQ. +/// +/// It also enables the IRQ if the registration succeeds. It returns `false` if +/// the registration failed. +pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { + trace!("register handler irq {}", irq_num); + crate::irq::register_handler_common(irq_num, handler) +} + /// Fetches the IRQ number. pub fn fetch_irq() -> usize { GICC.iar() as usize @@ -87,6 +91,34 @@ pub(crate) fn init_primary() { info!("Initialize GICv2..."); GICD.lock().init(); GICC.init(); + GICV.init(); +} + +pub fn inject_interrupt(vector: usize) { + let hcr = GICH.get_hcr(); + GICH.set_hcr(hcr | 1 << 0); + let mut lr = 0; + lr |= vector << 0; + lr |= 1 << 19; + lr |= 1 << 28; + GICH.set_lr(0, lr as u32); +} + +pub struct MyVgic {} + +impl MyVgic { + pub fn get_gich() -> &'static GicHypervisorInterface { + &GICH + } + pub fn get_gicd() -> &'static SpinNoIrq { + &GICD + } + pub fn get_gicc() -> &'static GicCpuInterface { + &GICC + } + pub fn get_gicv() -> &'static GicCpuInterface { + &GICV + } } /// Initializes GICC on secondary CPUs. From d9c31951c735ceeb6af02be5bba570e339d7f035 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 14 Apr 2025 14:24:55 +0800 Subject: [PATCH 24/48] fixes after merge debin/timer_api --- Cargo.lock | 49 ++++++++++++------- Cargo.toml | 2 - modules/axhal/src/irq.rs | 5 +- .../axhal/src/platform/aarch64_common/gic.rs | 25 ++++++++-- .../src/platform/aarch64_qemu_virt/mod.rs | 1 + 5 files changed, 58 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fc9eca8d5..f80e5943b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,7 +92,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -188,7 +188,7 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arm_gicv2.git?branch=sgi#956665adcdc6b09a28078db09ad3f09ed41af80b" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2?branch=debin%2F2vm_timer#0ad077649e030b9836a5dc55a5a01b1fd0246e11" dependencies = [ "tock-registers 0.8.1", ] @@ -478,7 +478,7 @@ dependencies = [ "ns16550a", "page_table_entry", "page_table_multiarch", - "percpu", + "percpu 0.2.0", "raw-cpuid 11.4.0", "riscv", "riscv_goldfish", @@ -510,7 +510,7 @@ dependencies = [ "kspin", "lazyinit", "log", - "percpu", + "percpu 0.1.7", ] [[package]] @@ -597,7 +597,7 @@ dependencies = [ "crate_interface", "ctor_bare", "kernel_guard", - "percpu", + "percpu 0.2.0", ] [[package]] @@ -636,7 +636,7 @@ dependencies = [ "lazyinit", "log", "memory_addr", - "percpu", + "percpu 0.2.0", "rand", "scheduler", "timer_list", @@ -985,16 +985,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "errno" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - [[package]] name = "fatfs" version = "0.4.0" @@ -1374,6 +1364,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "percpu" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e24fa01075d3d4235b84d3dd5a0ec0fd3040602065ea155acd0849e9067007" +dependencies = [ + "cfg-if", + "percpu_macros 0.1.7", + "spin", + "x86", +] + [[package]] name = "percpu" version = "0.2.0" @@ -1382,11 +1384,22 @@ checksum = "01e56c0c558952222967b592899f98765b48590e7bd7403bfd7075f73afc6ed6" dependencies = [ "cfg-if", "kernel_guard", - "percpu_macros", + "percpu_macros 0.2.0", "spin", "x86", ] +[[package]] +name = "percpu_macros" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421082ebe971ea22e82137de45ab6b9500bb178b364ebba02858bf7f80ec6471" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "percpu_macros" version = "0.2.0" @@ -1576,7 +1589,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daae4c8e29d4d8c36fbda4c318ffa1755645766fefab8fcff61a8ecd577bb822" dependencies = [ - "tock-registers", + "tock-registers 0.8.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bd713f40cf..8d44d4f470 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,5 @@ axipi = { path = "modules/axipi" } allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } -allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } - [profile.release] lto = true diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 7b0b629086..e611e24fa0 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -5,7 +5,10 @@ use handler_table::HandlerTable; use crate::platform::irq::{MAX_IRQ_COUNT, dispatch_irq}; use crate::trap::{IRQ, register_trap_handler}; -pub use crate::platform::irq::{MyVgic, inject_interrupt, register_handler, set_enable}; +pub use crate::platform::irq::{register_handler, set_enable}; + +#[cfg(all(target_arch = "aarch64", feature = "hv"))] +pub use crate::platform::irq::{MyVgic, inject_interrupt}; #[cfg(feature = "ipi")] pub use crate::platform::irq::{IPI_IRQ_NUM, send_sgi_all, send_sgi_one}; diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 663562d883..47ae6de3a4 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -1,10 +1,16 @@ use crate::{irq::IrqHandler, mem::phys_to_virt}; use arm_gicv2::{ - GicCpuInterface, GicDistributor, GicHypervisorInterface, InterruptType, translate_irq, + GicCpuInterface, GicDistributor, InterruptType, translate_irq, }; -use axconfig::devices::{GICC_PADDR, GICD_PADDR, GICH_PADDR, GICV_PADDR, UART_IRQ}; +use axconfig::devices::{GICC_PADDR, GICD_PADDR, UART_IRQ}; use kspin::SpinNoIrq; use memory_addr::PhysAddr; + +#[cfg(feature = "hv")] +use arm_gicv2::GicHypervisorInterface; +#[cfg(feature = "hv")] +use axconfig::devices::{GICV_PADDR, GICH_PADDR}; + /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; @@ -20,9 +26,13 @@ pub const TIMER_IRQ_NUM: usize = translate_irq(10, InterruptType::PPI).unwrap(); pub const UART_IRQ_NUM: usize = translate_irq(UART_IRQ, InterruptType::SPI).unwrap(); /// The IPI IRQ number. pub const IPI_IRQ_NUM: usize = translate_irq(1, InterruptType::SGI).unwrap(); + const GICD_BASE: PhysAddr = pa!(GICD_PADDR); const GICC_BASE: PhysAddr = pa!(GICC_PADDR); + +#[cfg(feature = "hv")] const GICV_BASE: PhysAddr = pa!(GICV_PADDR); +#[cfg(feature = "hv")] const GICH_BASE: PhysAddr = pa!(GICH_PADDR); static GICD: SpinNoIrq = @@ -30,7 +40,9 @@ static GICD: SpinNoIrq = // per-CPU, no lock static GICC: GicCpuInterface = GicCpuInterface::new(phys_to_virt(GICC_BASE).as_mut_ptr()); +#[cfg(feature = "hv")] static GICV: GicCpuInterface = GicCpuInterface::new(phys_to_virt(GICV_BASE).as_mut_ptr()); +#[cfg(feature = "hv")] static GICH: GicHypervisorInterface = GicHypervisorInterface::new(phys_to_virt(GICH_BASE).as_mut_ptr()); @@ -91,9 +103,14 @@ pub(crate) fn init_primary() { info!("Initialize GICv2..."); GICD.lock().init(); GICC.init(); - GICV.init(); + + #[cfg(feature = "hv")] + { + GICV.init(); + } } +#[cfg(feature = "hv")] pub fn inject_interrupt(vector: usize) { let hcr = GICH.get_hcr(); GICH.set_hcr(hcr | 1 << 0); @@ -104,8 +121,10 @@ pub fn inject_interrupt(vector: usize) { GICH.set_lr(0, lr as u32); } +#[cfg(feature = "hv")] pub struct MyVgic {} +#[cfg(feature = "hv")] impl MyVgic { pub fn get_gich() -> &'static GicHypervisorInterface { &GICH diff --git a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs index d1dee09d70..4bb16d8450 100644 --- a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs @@ -21,6 +21,7 @@ pub mod misc { } unsafe extern "C" { + fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] fn rust_main_secondary(cpu_id: usize); From 1341bedc09fc28c093cab5a25518bfae6387f79a Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 14 Apr 2025 15:21:12 +0800 Subject: [PATCH 25/48] update percpu --- Cargo.lock | 33 +++++---------------------------- modules/axipi/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f80e5943b7..21e543c55b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,7 +478,7 @@ dependencies = [ "ns16550a", "page_table_entry", "page_table_multiarch", - "percpu 0.2.0", + "percpu", "raw-cpuid 11.4.0", "riscv", "riscv_goldfish", @@ -510,7 +510,7 @@ dependencies = [ "kspin", "lazyinit", "log", - "percpu 0.1.7", + "percpu", ] [[package]] @@ -597,7 +597,7 @@ dependencies = [ "crate_interface", "ctor_bare", "kernel_guard", - "percpu 0.2.0", + "percpu", ] [[package]] @@ -636,7 +636,7 @@ dependencies = [ "lazyinit", "log", "memory_addr", - "percpu 0.2.0", + "percpu", "rand", "scheduler", "timer_list", @@ -1364,18 +1364,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "percpu" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e24fa01075d3d4235b84d3dd5a0ec0fd3040602065ea155acd0849e9067007" -dependencies = [ - "cfg-if", - "percpu_macros 0.1.7", - "spin", - "x86", -] - [[package]] name = "percpu" version = "0.2.0" @@ -1384,22 +1372,11 @@ checksum = "01e56c0c558952222967b592899f98765b48590e7bd7403bfd7075f73afc6ed6" dependencies = [ "cfg-if", "kernel_guard", - "percpu_macros 0.2.0", + "percpu_macros", "spin", "x86", ] -[[package]] -name = "percpu_macros" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "421082ebe971ea22e82137de45ab6b9500bb178b364ebba02858bf7f80ec6471" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "percpu_macros" version = "0.2.0" diff --git a/modules/axipi/Cargo.toml b/modules/axipi/Cargo.toml index 3ba21b1149..e1ee1318f4 100644 --- a/modules/axipi/Cargo.toml +++ b/modules/axipi/Cargo.toml @@ -16,7 +16,7 @@ default = [] log = "=0.4.21" lazyinit = { version = "0.2" } -percpu = { version = "0.1.6" } +percpu = { version = "0.2" } kspin = { version = "0.1" } axhal = { workspace = true, features = ["ipi"] } From 35b7bc57a6cca5ac063a1d7b2748433687660b7c Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 14 Apr 2025 16:33:29 +0800 Subject: [PATCH 26/48] formatted --- modules/axhal/src/platform/aarch64_common/gic.rs | 6 ++---- modules/axhal/src/platform/riscv64_qemu_virt/irq.rs | 4 ++-- modules/axtask/src/task.rs | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 47ae6de3a4..8b53faed30 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -1,7 +1,5 @@ use crate::{irq::IrqHandler, mem::phys_to_virt}; -use arm_gicv2::{ - GicCpuInterface, GicDistributor, InterruptType, translate_irq, -}; +use arm_gicv2::{GicCpuInterface, GicDistributor, InterruptType, translate_irq}; use axconfig::devices::{GICC_PADDR, GICD_PADDR, UART_IRQ}; use kspin::SpinNoIrq; use memory_addr::PhysAddr; @@ -9,7 +7,7 @@ use memory_addr::PhysAddr; #[cfg(feature = "hv")] use arm_gicv2::GicHypervisorInterface; #[cfg(feature = "hv")] -use axconfig::devices::{GICV_PADDR, GICH_PADDR}; +use axconfig::devices::{GICH_PADDR, GICV_PADDR}; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index ea25090564..6b4c5122c8 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -5,9 +5,9 @@ use lazyinit::LazyInit; use riscv::register::sie; // use riscv_plic::{HartContext, InterruptSource, Plic}; -use sbi_rt::{send_ipi, HartMask}; +use sbi_rt::{HartMask, send_ipi}; -use memory_addr::{pa, PhysAddr}; +use memory_addr::{PhysAddr, pa}; use crate::cpu::this_cpu_id; use crate::irq::IrqHandler; diff --git a/modules/axtask/src/task.rs b/modules/axtask/src/task.rs index 4eea7038b5..b1f3088a21 100644 --- a/modules/axtask/src/task.rs +++ b/modules/axtask/src/task.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, string::String, sync::Arc}; use core::ops::Deref; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, AtomicU8, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU64, Ordering}; use core::{alloc::Layout, cell::UnsafeCell, fmt, ptr::NonNull}; #[cfg(feature = "preempt")] From 659f7acc3a7cc9da218f809bbfab1f0561b95db0 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Thu, 17 Apr 2025 21:40:37 +0800 Subject: [PATCH 27/48] remove a temp file --- qemu.dts | 211 ------------------------------------------------------- 1 file changed, 211 deletions(-) delete mode 100644 qemu.dts diff --git a/qemu.dts b/qemu.dts deleted file mode 100644 index f1efae6d64..0000000000 --- a/qemu.dts +++ /dev/null @@ -1,211 +0,0 @@ -/dts-v1/; - -/ { - #address-cells = <0x02>; - #size-cells = <0x02>; - compatible = "riscv-virtio"; - model = "riscv-virtio,qemu"; - - poweroff { - value = <0x5555>; - offset = <0x00>; - regmap = <0x04>; - compatible = "syscon-poweroff"; - }; - - reboot { - value = <0x7777>; - offset = <0x00>; - regmap = <0x04>; - compatible = "syscon-reboot"; - }; - - platform-bus@4000000 { - interrupt-parent = <0x03>; - ranges = <0x00 0x00 0x4000000 0x2000000>; - #address-cells = <0x01>; - #size-cells = <0x01>; - compatible = "qemu,platform\0simple-bus"; - }; - - memory@80000000 { - device_type = "memory"; - reg = <0x00 0x80000000 0x00 0x80000000>; - }; - - cpus { - #address-cells = <0x01>; - #size-cells = <0x00>; - timebase-frequency = <0x989680>; - - cpu@0 { - phandle = <0x01>; - device_type = "cpu"; - reg = <0x00>; - status = "okay"; - compatible = "riscv"; - riscv,cbop-block-size = <0x40>; - riscv,cboz-block-size = <0x40>; - riscv,cbom-block-size = <0x40>; - riscv,isa-extensions = "i\0m\0a\0f\0d\0c\0h\0zic64b\0zicbom\0zicbop\0zicboz\0ziccamoa\0ziccif\0zicclsm\0ziccrse\0zicntr\0zicsr\0zifencei\0zihintntl\0zihintpause\0zihpm\0za64rs\0zawrs\0zfa\0zca\0zcd\0zba\0zbb\0zbc\0zbs\0ssccptr\0sscounterenw\0sstc\0sstvala\0sstvecd\0svadu"; - riscv,isa-base = "rv64i"; - riscv,isa = "rv64imafdch_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zicsr_zifencei_zihintntl_zihintpause_zihpm_za64rs_zawrs_zfa_zca_zcd_zba_zbb_zbc_zbs_ssccptr_sscounterenw_sstc_sstvala_sstvecd_svadu"; - mmu-type = "riscv,sv57"; - - interrupt-controller { - #interrupt-cells = <0x01>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - phandle = <0x02>; - }; - }; - - cpu-map { - - cluster0 { - - core0 { - cpu = <0x01>; - }; - }; - }; - }; - - pmu { - riscv,event-to-mhpmcounters = <0x01 0x01 0x7fff9 0x02 0x02 0x7fffc 0x10019 0x10019 0x7fff8 0x1001b 0x1001b 0x7fff8 0x10021 0x10021 0x7fff8>; - compatible = "riscv,pmu"; - }; - - fw-cfg@10100000 { - dma-coherent; - reg = <0x00 0x10100000 0x00 0x18>; - compatible = "qemu,fw-cfg-mmio"; - }; - - flash@20000000 { - bank-width = <0x04>; - reg = <0x00 0x20000000 0x00 0x2000000 0x00 0x22000000 0x00 0x2000000>; - compatible = "cfi-flash"; - }; - - chosen { - stdout-path = "/soc/serial@10000000"; - rng-seed = <0xf57e47f3 0x8a852b2a 0xa547abb4 0x830202d1 0xab9aef61 0xa2c692fe 0xb55bb2c5 0x63aea82d>; - }; - - soc { - #address-cells = <0x02>; - #size-cells = <0x02>; - compatible = "simple-bus"; - ranges; - - rtc@101000 { - interrupts = <0x0b>; - interrupt-parent = <0x03>; - reg = <0x00 0x101000 0x00 0x1000>; - compatible = "google,goldfish-rtc"; - }; - - serial@10000000 { - interrupts = <0x0a>; - interrupt-parent = <0x03>; - clock-frequency = "\08@"; - reg = <0x00 0x10000000 0x00 0x100>; - compatible = "ns16550a"; - }; - - test@100000 { - phandle = <0x04>; - reg = <0x00 0x100000 0x00 0x1000>; - compatible = "sifive,test1\0sifive,test0\0syscon"; - }; - - virtio_mmio@10008000 { - interrupts = <0x08>; - interrupt-parent = <0x03>; - reg = <0x00 0x10008000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10007000 { - interrupts = <0x07>; - interrupt-parent = <0x03>; - reg = <0x00 0x10007000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10006000 { - interrupts = <0x06>; - interrupt-parent = <0x03>; - reg = <0x00 0x10006000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10005000 { - interrupts = <0x05>; - interrupt-parent = <0x03>; - reg = <0x00 0x10005000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10004000 { - interrupts = <0x04>; - interrupt-parent = <0x03>; - reg = <0x00 0x10004000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10003000 { - interrupts = <0x03>; - interrupt-parent = <0x03>; - reg = <0x00 0x10003000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10002000 { - interrupts = <0x02>; - interrupt-parent = <0x03>; - reg = <0x00 0x10002000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - virtio_mmio@10001000 { - interrupts = <0x01>; - interrupt-parent = <0x03>; - reg = <0x00 0x10001000 0x00 0x1000>; - compatible = "virtio,mmio"; - }; - - plic@c000000 { - phandle = <0x03>; - riscv,ndev = <0x5f>; - reg = <0x00 0xc000000 0x00 0x600000>; - interrupts-extended = <0x02 0x0b 0x02 0x09>; - interrupt-controller; - compatible = "sifive,plic-1.0.0\0riscv,plic0"; - #address-cells = <0x00>; - #interrupt-cells = <0x01>; - }; - - clint@2000000 { - interrupts-extended = <0x02 0x03 0x02 0x07>; - reg = <0x00 0x2000000 0x00 0x10000>; - compatible = "sifive,clint0\0riscv,clint0"; - }; - - pci@30000000 { - interrupt-map-mask = <0x1800 0x00 0x00 0x07>; - interrupt-map = <0x00 0x00 0x00 0x01 0x03 0x20 0x00 0x00 0x00 0x02 0x03 0x21 0x00 0x00 0x00 0x03 0x03 0x22 0x00 0x00 0x00 0x04 0x03 0x23 0x800 0x00 0x00 0x01 0x03 0x21 0x800 0x00 0x00 0x02 0x03 0x22 0x800 0x00 0x00 0x03 0x03 0x23 0x800 0x00 0x00 0x04 0x03 0x20 0x1000 0x00 0x00 0x01 0x03 0x22 0x1000 0x00 0x00 0x02 0x03 0x23 0x1000 0x00 0x00 0x03 0x03 0x20 0x1000 0x00 0x00 0x04 0x03 0x21 0x1800 0x00 0x00 0x01 0x03 0x23 0x1800 0x00 0x00 0x02 0x03 0x20 0x1800 0x00 0x00 0x03 0x03 0x21 0x1800 0x00 0x00 0x04 0x03 0x22>; - ranges = <0x1000000 0x00 0x00 0x00 0x3000000 0x00 0x10000 0x2000000 0x00 0x40000000 0x00 0x40000000 0x00 0x40000000 0x3000000 0x04 0x00 0x04 0x00 0x04 0x00>; - reg = <0x00 0x30000000 0x00 0x10000000>; - dma-coherent; - bus-range = <0x00 0xff>; - linux,pci-domain = <0x00>; - device_type = "pci"; - compatible = "pci-host-ecam-generic"; - #size-cells = <0x02>; - #interrupt-cells = <0x01>; - #address-cells = <0x03>; - }; - }; -}; From d8846a4860c75ddfaca03be3f4be48365a29b5e2 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 26 May 2025 11:47:46 +0800 Subject: [PATCH 28/48] fixes for mistakes in #49 --- Cargo.lock | 43 +++++++++++++++++ api/arceos_posix_api/build.rs | 7 ++- configs/platforms/aarch64-rk3588j.toml | 48 +++++++++++-------- modules/axhal/Cargo.toml | 6 --- .../src/platform/aarch64_common/gicv3.rs | 8 +++- ulib/axlibc/build.rs | 5 ++ 6 files changed, 88 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 21e543c55b..17682913e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,6 +185,20 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "arm-gic-driver" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be20d5253a221975e41674d35169a6f67887b012490b9e1ab33d7369caa7332" +dependencies = [ + "aarch64-cpu 10.0.0", + "bitflags 2.8.0", + "enum_dispatch", + "log", + "rdif-intc", + "tock-registers 0.9.0", +] + [[package]] name = "arm_gicv2" version = "0.1.0" @@ -454,6 +468,7 @@ name = "axhal" version = "0.1.0" dependencies = [ "aarch64-cpu 10.0.0", + "arm-gic-driver", "arm_gicv2", "arm_pl011", "arm_pl031", @@ -969,6 +984,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1495,6 +1522,22 @@ dependencies = [ "bitflags 2.8.0", ] +[[package]] +name = "rdif-base" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023bd968dcec1d1d01092fdf0e6473cd4fee4b3a4140e524d5952cf332882a36" + +[[package]] +name = "rdif-intc" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4dce859ee52f984c78d41f647eb3266e58824e385583ae64a5467597c84fd5a" +dependencies = [ + "cfg-if", + "rdif-base", +] + [[package]] name = "regex" version = "1.11.1" diff --git a/api/arceos_posix_api/build.rs b/api/arceos_posix_api/build.rs index bb8368511e..09adb71ac9 100644 --- a/api/arceos_posix_api/build.rs +++ b/api/arceos_posix_api/build.rs @@ -76,11 +76,12 @@ typedef struct {{ impl bindgen::callbacks::ParseCallbacks for MyCallbacks { fn include_file(&self, fname: &str) { if !fname.contains("ax_pthread_mutex.h") { - println!("cargo:rerun-if-changed={}", fname); + println!("cargo:rerun-if-changed={fname}"); } } } + let target = std::env::var("TARGET").unwrap(); let mut builder = bindgen::Builder::default() .header(in_file) .clang_arg("-I./../../ulib/axlibc/include") @@ -88,6 +89,10 @@ typedef struct {{ .derive_default(true) .size_t_is_usize(false) .use_core(); + if let Some(llvm_target) = target.strip_suffix("-softfloat") { + // remove "-softfloat" suffix for some targets + builder = builder.clang_arg(format!("--target={llvm_target}")); + } for ty in allow_types { builder = builder.allowlist_type(ty); } diff --git a/configs/platforms/aarch64-rk3588j.toml b/configs/platforms/aarch64-rk3588j.toml index 61d0793234..9a769f3279 100644 --- a/configs/platforms/aarch64-rk3588j.toml +++ b/configs/platforms/aarch64-rk3588j.toml @@ -1,25 +1,32 @@ # Architecture identifier. -arch = "aarch64" +arch = "aarch64" # str # Platform identifier. -platform = "aarch64-rk3588j" +platform = "aarch64-rk3588j" # str + +# +# Platform configs +# +[plat] # Platform family. family = "aarch64-rk3588j" # Base address of the whole physical memory. -phys-memory-base = "0x20_0000" +phys-memory-base = 0x20_0000 # uint # Size of the whole physical memory. -phys-memory-size = "0x800_0000" # 128M +phys-memory-size = 0x800_0000 # uint # Base physical address of the kernel image. -kernel-base-paddr = "0x48_0000" +kernel-base-paddr = 0x48_0000 # uint # Base virtual address of the kernel image. -kernel-base-vaddr = "0x0000_0000_0048_0000" +kernel-base-vaddr = "0x0000_0000_0048_0000" # uint # Linear mapping offset, for quick conversions between physical and virtual # addresses. -phys-virt-offset = "0xffff_0000_0000_0000" +phys-virt-offset = "0xffff_0000_0000_0000" # uint # Kernel address space base. -kernel-aspace-base = "0xffff_0000_0000_0000" +kernel-aspace-base = "0xffff_0000_0000_0000" # uint # Kernel address space size. -kernel-aspace-size = "0x0000_ffff_ffff_f000" +kernel-aspace-size = "0x0000_ffff_ffff_f000" # uint + +[devices] # MMIO regions with format (`base_paddr`, `size`). mmio-regions = [ # ["0x0900_0000", "0x1000"], # PL011 UART @@ -34,25 +41,25 @@ mmio-regions = [ ["0xa40c00000", "0x400000"], ["0xf4000000","0x1000000"], ["0xf3000000","0x1000000"], -] +] # [(uint, uint)] # VirtIO MMIO regions with format (`base_paddr`, `size`). -virtio-mmio-regions = [] +virtio-mmio-regions = [] # [(uint, uint)] # Base physical address of the PCIe ECAM space. -pci-ecam-base = "0xf4000000" +pci-ecam-base = "0xf4000000" # uint # End PCI bus number (`bus-range` property in device tree). -pci-bus-end = "0xff" +pci-bus-end = "0xff" # uint # PCI device memory ranges (`ranges` property in device tree). -pci-ranges = [] +pci-ranges = [] # [(uint, uint)] # UART Address -uart-paddr = "0xfeb5_0000" -uart-irq = "0x14d" +uart-paddr = "0xfeb5_0000" # uint +uart-irq = "0x14d" # uint # GICC Address -gicd-paddr = "0xfe600000" +gicd-paddr = "0xfe600000" # uint # GICR Address -gicc-paddr = "0xfe680000" -gicr-paddr = "0xfe680000" +gicc-paddr = "0xfe680000" # uint +gicr-paddr = "0xfe680000" # uint # PSCI psci-method = "smc" @@ -65,4 +72,5 @@ psci-method = "smc" # compatible = "arm,pl031\0arm,primecell"; # }; # RTC (PL031) Address -# rtc-paddr = "0x901_0000" \ No newline at end of file +# rtc-paddr = "0x901_0000" +nothing = "nothing" # str diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index ec12bbd5fe..2c7b0225e8 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -75,12 +75,6 @@ arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } dw_apb_uart = "0.1" -[target.'cfg(all(target_arch = "aarch64", not(feature = "gicv3"), feature = "hv"))'.dependencies] -arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", features = ["el2"] } - -[target.'cfg(all(target_arch = "aarch64", not(feature = "gicv3"), not(feature = "hv")))'.dependencies] -arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2" } - [target.'cfg(target_arch = "loongarch64")'.dependencies] loongArch64 = "0.2.4" ns16550a = "0.4.0" diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index 4dc0a6ff79..ffded325a6 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -4,6 +4,8 @@ use axconfig::devices::{GICC_PADDR, GICD_PADDR, GICR_PADDR, UART_IRQ}; use core::ptr::NonNull; use kspin::SpinNoIrq; use memory_addr::PhysAddr; +use arm_gicv2::InterruptType; +use arm_gicv2::translate_irq; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; @@ -18,9 +20,11 @@ pub const TIMER_IRQ_NUM: usize = arm_gic_driver::IntId::ppi(10).to_u32() as usiz /// The UART IRQ number. pub const UART_IRQ_NUM: usize = arm_gic_driver::IntId::spi(UART_IRQ as u32).to_u32() as usize; +/// The IPI IRQ number. +pub const IPI_IRQ_NUM: usize = translate_irq(1, InterruptType::SGI).unwrap(); const GICD_BASE: PhysAddr = pa!(GICD_PADDR); -const GICC_BASE: PhysAddr = pa!(GICR_PADDR); +const GICR_BASE: PhysAddr = pa!(GICR_PADDR); static GICD: SpinNoIrq> = SpinNoIrq::new(None); static GICC: SpinNoIrq>> = SpinNoIrq::new(None); @@ -81,7 +85,7 @@ pub(crate) fn init_primary() { info!("Initialize GICv3..."); let gicd = arm_gic_driver::v3::Gic::new( NonNull::new(phys_to_virt(GICD_BASE).as_mut_ptr()).unwrap(), - NonNull::new(phys_to_virt(GICC_BASE).as_mut_ptr()).unwrap(), + NonNull::new(phys_to_virt(GICR_BASE).as_mut_ptr()).unwrap(), arm_gic_driver::v3::Security::OneNS, ); let interface = gicd.cpu_interface(); diff --git a/ulib/axlibc/build.rs b/ulib/axlibc/build.rs index 6c33c929c7..d8d7ce0dbe 100644 --- a/ulib/axlibc/build.rs +++ b/ulib/axlibc/build.rs @@ -2,6 +2,7 @@ fn main() { fn gen_c_to_rust_bindings(in_file: &str, out_file: &str) { println!("cargo:rerun-if-changed={in_file}"); + let target = std::env::var("TARGET").unwrap(); let allow_types = ["tm", "jmp_buf"]; let mut builder = bindgen::Builder::default() .header(in_file) @@ -9,6 +10,10 @@ fn main() { .derive_default(true) .size_t_is_usize(false) .use_core(); + if let Some(llvm_target) = target.strip_suffix("-softfloat") { + // remove "-softfloat" suffix for some targets + builder = builder.clang_arg(format!("--target={llvm_target}")); + } for ty in allow_types { builder = builder.allowlist_type(ty); } From 1ad470f8b271adeba3ed142878e66315435c5237 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 26 May 2025 15:31:40 +0800 Subject: [PATCH 29/48] `inject_interrupt` on GICv3 based on hVisor --- .../src/platform/aarch64_common/gicv3.rs | 149 ++++++++++++++++-- 1 file changed, 139 insertions(+), 10 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index ffded325a6..a4e452c307 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -1,11 +1,12 @@ use crate::{arch::disable_irqs, irq::IrqHandler, mem::phys_to_virt}; use arm_gic_driver::*; use axconfig::devices::{GICC_PADDR, GICD_PADDR, GICR_PADDR, UART_IRQ}; -use core::ptr::NonNull; +use core::{panic, ptr::NonNull}; use kspin::SpinNoIrq; use memory_addr::PhysAddr; -use arm_gicv2::InterruptType; -use arm_gicv2::translate_irq; +use arm_gicv2::{translate_irq, InterruptType}; +#[cfg(feature = "hv")] +use arm_gicv2::GicHypervisorInterface; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; @@ -27,7 +28,7 @@ const GICD_BASE: PhysAddr = pa!(GICD_PADDR); const GICR_BASE: PhysAddr = pa!(GICR_PADDR); static GICD: SpinNoIrq> = SpinNoIrq::new(None); -static GICC: SpinNoIrq>> = SpinNoIrq::new(None); +static GICR: SpinNoIrq>> = SpinNoIrq::new(None); /// Enables or disables the given IRQ. pub fn set_enable(irq_num: usize, enabled: bool) { @@ -53,7 +54,7 @@ pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { /// Fetches the IRQ number. pub fn fetch_irq() -> usize { - GICC.lock() + GICR.lock() .as_mut() .unwrap() .ack() @@ -69,17 +70,145 @@ pub fn fetch_irq() -> usize { pub fn dispatch_irq(irq_num: usize) { let intid: Option; if irq_num == 0 { - intid = GICC.lock().as_mut().unwrap().ack(); + intid = GICR.lock().as_mut().unwrap().ack(); info!("interrupt {:?}", intid.unwrap()); } else { intid = Some(IrqId::from(irq_num)); } if let Some(intid) = intid { crate::irq::dispatch_irq_common(intid.into()); - GICC.lock().as_mut().unwrap().eoi(intid); + GICR.lock().as_mut().unwrap().eoi(intid); + } +} + +/// Reads and returns the value of the given aarch64 system register. +macro_rules! read_sysreg { + ($name:ident) => { + { + let mut value: u64; + unsafe{::core::arch::asm!( + concat!("mrs {value:x}, ", ::core::stringify!($name)), + value = out(reg) value, + options(nomem, nostack), + );} + value + } + } +} + +/// Writes the given value to the given aarch64 system register. +macro_rules! write_sysreg { + ($name:ident, $value:expr) => { + { + let v: u64 = $value; + unsafe{::core::arch::asm!( + concat!("msr ", ::core::stringify!($name), ", {value:x}"), + value = in(reg) v, + options(nomem, nostack), + )} + } + } +} + +#[cfg(feature = "hv")] +pub fn inject_interrupt(vector: usize) { + // mask + const LR_VIRTIRQ_MASK: usize = (1 << 32) - 1; + + let elsr: u64 = read_sysreg!(ich_elrsr_el2); + let vtr = read_sysreg!(ich_vtr_el2) as usize; + let lr_num: usize = (vtr & 0xf) + 1; + let mut free_lr = -1 as isize; + for i in 0..lr_num { + // find a free list register + if (1 << i) & elsr > 0 { + if free_lr == -1 { + free_lr = i as isize; + } + continue; + } + let lr_val = read_lr(i) as usize; + // if a virtual interrupt is enabled and equals to the physical interrupt irq_id + if (lr_val & LR_VIRTIRQ_MASK) == vector { + trace!("virtual irq {} enables again", vector); + } + } + trace!("To Inject IRQ {}, find lr {}", vector, free_lr); + + if free_lr == -1 { + panic!("No free list register to inject IRQ {}", vector); + } else { + let mut val = vector as u64; // vector + val |= 1 << 60; // group 1 + val |= 1 << 62; // state pending + // hardware interrupt not supported + write_lr(free_lr as usize, val); } } +fn read_lr(id: usize) -> u64 { + let id = id as u64; + match id { + //TODO get lr size from gic reg + 0 => read_sysreg!(ich_lr0_el2), + 1 => read_sysreg!(ich_lr1_el2), + 2 => read_sysreg!(ich_lr2_el2), + 3 => read_sysreg!(ich_lr3_el2), + 4 => read_sysreg!(ich_lr4_el2), + 5 => read_sysreg!(ich_lr5_el2), + 6 => read_sysreg!(ich_lr6_el2), + 7 => read_sysreg!(ich_lr7_el2), + 8 => read_sysreg!(ich_lr8_el2), + 9 => read_sysreg!(ich_lr9_el2), + 10 => read_sysreg!(ich_lr10_el2), + 11 => read_sysreg!(ich_lr11_el2), + 12 => read_sysreg!(ich_lr12_el2), + 13 => read_sysreg!(ich_lr13_el2), + 14 => read_sysreg!(ich_lr14_el2), + 15 => read_sysreg!(ich_lr15_el2), + _ => { + panic!("invalid lr id {}", id); + } + } +} + +fn write_lr(id: usize, val: u64) { + let id = id as u64; + match id { + 0 => write_sysreg!(ich_lr0_el2, val), + 1 => write_sysreg!(ich_lr1_el2, val), + 2 => write_sysreg!(ich_lr2_el2, val), + 3 => write_sysreg!(ich_lr3_el2, val), + 4 => write_sysreg!(ich_lr4_el2, val), + 5 => write_sysreg!(ich_lr5_el2, val), + 6 => write_sysreg!(ich_lr6_el2, val), + 7 => write_sysreg!(ich_lr7_el2, val), + 8 => write_sysreg!(ich_lr8_el2, val), + 9 => write_sysreg!(ich_lr9_el2, val), + 10 => write_sysreg!(ich_lr10_el2, val), + 11 => write_sysreg!(ich_lr11_el2, val), + 12 => write_sysreg!(ich_lr12_el2, val), + 13 => write_sysreg!(ich_lr13_el2, val), + 14 => write_sysreg!(ich_lr14_el2, val), + 15 => write_sysreg!(ich_lr15_el2, val), + _ => { + panic!("invalid lr id {}", id); + } + } +} + +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { + todo!() +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_sgi_all(irq_num: usize) { + todo!() +} + +pub struct MyVgic{} + /// Initializes GICD, GICC on the primary CPU. pub(crate) fn init_primary() { info!("Initialize GICv3..."); @@ -91,7 +220,7 @@ pub(crate) fn init_primary() { let interface = gicd.cpu_interface(); GICD.lock().replace(gicd); - GICC.lock().replace(interface); + GICR.lock().replace(interface); disable_irqs(); } @@ -100,6 +229,6 @@ pub(crate) fn init_primary() { #[cfg(feature = "smp")] pub(crate) fn init_secondary() { let interface = GICD.lock().as_mut().unwrap().cpu_interface(); - GICC.lock().replace(interface); - GICC.lock().as_mut().unwrap().setup(); + GICR.lock().replace(interface); + GICR.lock().as_mut().unwrap().setup(); } From 563378076598ce83921102e0cd6eb19b7178c5ba Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 26 May 2025 20:29:41 +0800 Subject: [PATCH 30/48] add sgi ipi interface for gicv3 driver --- .../src/platform/aarch64_common/gicv3.rs | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index a4e452c307..a5e724459e 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -133,7 +133,7 @@ pub fn inject_interrupt(vector: usize) { trace!("virtual irq {} enables again", vector); } } - trace!("To Inject IRQ {}, find lr {}", vector, free_lr); + trace!("use free lr {} to inject irq {}", free_lr, vector); if free_lr == -1 { panic!("No free list register to inject IRQ {}", vector); @@ -197,16 +197,40 @@ fn write_lr(id: usize, val: u64) { } } +fn send_sgi_inner(aff3: u8, aff2: u8, aff1: u8, target: u8, vector: usize, to_all: bool) { + let value = + ((vector & 0xF) << 24) | // vector + (1 << target) | // target bitmap + ((aff1 as usize) << 16) | // affinity level 1 + ((aff2 as usize) << 32) | // affinity level 2 + ((aff3 as usize) << 48) ; // affinity level 3 + + write_sysreg!(icc_sgi1r_el1, value as _); +} + /// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. -pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { - todo!() +pub fn send_sgi_one(dest: usize, vector: usize) { + #[cfg(platform_family = "aarch64-rk3588j")] + { + // learnt from hVisor, that rockchip socs follow the 0.0.x.0 affinity scheme + // while other socs follow 0.0.0.x + // + // the best and standard way is reading + send_sgi_inner(0, 0, dest as _, 0, vector, false); + } + #[cfg(not(platform_family = "aarch64-rk3588j"))] + { + // the default affinity scheme is 0.0.0.x + send_sgi_inner(0, 0, 0, dest as _, vector, false); + } } /// Sends a broadcast IPI to all CPUs. -pub fn send_sgi_all(irq_num: usize) { - todo!() +pub fn send_sgi_all(vector: usize) { + send_sgi_inner(0, 0, 0, 0, vector, true); } +// dummy implementation pub struct MyVgic{} /// Initializes GICD, GICC on the primary CPU. From 4104125074d5ae88a1a58ec466cd4485b359747b Mon Sep 17 00:00:00 2001 From: aarkegz Date: Mon, 26 May 2025 21:11:35 +0800 Subject: [PATCH 31/48] fix missing irm field in `send_sgi_inner` --- modules/axhal/src/platform/aarch64_common/gicv3.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index a5e724459e..baa096360a 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -37,9 +37,9 @@ pub fn set_enable(irq_num: usize, enabled: bool) { let mut gicd = GICD.lock(); let d = gicd.as_mut().unwrap(); if enabled { - d.irq_enable(irq_num.into()); + d.irq_enable(irq_num.into()).unwrap(); } else { - d.irq_disable(irq_num.into()); + d.irq_disable(irq_num.into()).unwrap(); } } @@ -203,7 +203,8 @@ fn send_sgi_inner(aff3: u8, aff2: u8, aff1: u8, target: u8, vector: usize, to_al (1 << target) | // target bitmap ((aff1 as usize) << 16) | // affinity level 1 ((aff2 as usize) << 32) | // affinity level 2 - ((aff3 as usize) << 48) ; // affinity level 3 + ((aff3 as usize) << 48) | // affinity level 3 + ((to_all as usize) << 40); // interrupt routing mode write_sysreg!(icc_sgi1r_el1, value as _); } From 9f8fa972f1eab1ee5d64309318b17f255acb6ec0 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Sun, 1 Jun 2025 21:57:13 +0800 Subject: [PATCH 32/48] [wip] only init necessery gicr settings --- Cargo.lock | 43 +++++++++ modules/axhal/src/arch/aarch64/mod.rs | 2 +- .../platform/aarch64_common/generic_timer.rs | 26 +++--- .../src/platform/aarch64_common/gicv3.rs | 87 +++++++++++++------ modules/axruntime/src/lib.rs | 4 +- 5 files changed, 124 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eed6a5a269..0cf3da24d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,20 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "arm-gic-driver" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be20d5253a221975e41674d35169a6f67887b012490b9e1ab33d7369caa7332" +dependencies = [ + "aarch64-cpu 10.0.0", + "bitflags 2.8.0", + "enum_dispatch", + "log", + "rdif-intc", + "tock-registers 0.9.0", +] + [[package]] name = "arm_gicv2" version = "0.1.0" @@ -452,6 +466,7 @@ name = "axhal" version = "0.1.0" dependencies = [ "aarch64-cpu 10.0.0", + "arm-gic-driver", "arm_gicv2", "arm_pl011", "arm_pl031", @@ -952,6 +967,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1478,6 +1505,22 @@ dependencies = [ "bitflags 2.8.0", ] +[[package]] +name = "rdif-base" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023bd968dcec1d1d01092fdf0e6473cd4fee4b3a4140e524d5952cf332882a36" + +[[package]] +name = "rdif-intc" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4dce859ee52f984c78d41f647eb3266e58824e385583ae64a5467597c84fd5a" +dependencies = [ + "cfg-if", + "rdif-base", +] + [[package]] name = "regex" version = "1.11.1" diff --git a/modules/axhal/src/arch/aarch64/mod.rs b/modules/axhal/src/arch/aarch64/mod.rs index 7b2ac336e9..8a663f51c6 100644 --- a/modules/axhal/src/arch/aarch64/mod.rs +++ b/modules/axhal/src/arch/aarch64/mod.rs @@ -20,7 +20,7 @@ pub use self::context::{FpState, TaskContext, TrapFrame}; /// Allows the current CPU to respond to interrupts. #[inline] pub fn enable_irqs() { - unsafe { asm!("msr daifclr, #2") }; + // unsafe { asm!("msr daifclr, #2") }; } /// Makes the current CPU to ignore interrupts. diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index 127db32c39..d6903a87ed 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -94,21 +94,27 @@ pub(crate) fn init_early() { } pub(crate) fn init_percpu() { + use aarch64_cpu::registers::{CNTHCTL_EL2, CNTVOFF_EL2}; + use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + // Disable EL1 timer traps and the timer offset. + CNTHCTL_EL2.modify(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + CNTVOFF_EL2.set(0); + #[cfg(all(feature = "irq", not(feature = "hv")))] { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); CNTP_TVAL_EL0.set(0); } - #[cfg(all(feature = "irq", feature = "hv"))] - { - unsafe { - // ENABLE, bit [0], Enables the timer. - // * 0b0: Timer disabled. - // * 0b1: Timer enabled. - core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); - core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); - } - } + // #[cfg(all(feature = "irq", feature = "hv"))] + // { + // unsafe { + // // ENABLE, bit [0], Enables the timer. + // // * 0b0: Timer disabled. + // // * 0b1: Timer enabled. + // core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); + // core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); + // } + // } #[cfg(feature = "irq")] crate::platform::irq::set_enable(crate::platform::irq::TIMER_IRQ_NUM, true); } diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index 4dc0a6ff79..ff63e9ae5e 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -1,9 +1,12 @@ -use crate::{arch::disable_irqs, irq::IrqHandler, mem::phys_to_virt}; +use crate::{arch::disable_irqs, cpu::this_cpu_id, irq::IrqHandler, mem::phys_to_virt}; use arm_gic_driver::*; use axconfig::devices::{GICC_PADDR, GICD_PADDR, GICR_PADDR, UART_IRQ}; use core::ptr::NonNull; use kspin::SpinNoIrq; -use memory_addr::PhysAddr; +use memory_addr::{MemoryAddr, PhysAddr}; + +use aarch64_cpu::registers::{ICC_SRE_EL2, SCTLR_EL3::I}; +use tock_registers::interfaces::{Readable, Writeable}; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; @@ -20,7 +23,8 @@ pub const TIMER_IRQ_NUM: usize = arm_gic_driver::IntId::ppi(10).to_u32() as usiz pub const UART_IRQ_NUM: usize = arm_gic_driver::IntId::spi(UART_IRQ as u32).to_u32() as usize; const GICD_BASE: PhysAddr = pa!(GICD_PADDR); -const GICC_BASE: PhysAddr = pa!(GICR_PADDR); +const GICC_BASE: PhysAddr = pa!(GICC_PADDR); +const GICR_BASE: PhysAddr = pa!(GICR_PADDR); static GICD: SpinNoIrq> = SpinNoIrq::new(None); static GICC: SpinNoIrq>> = SpinNoIrq::new(None); @@ -29,13 +33,13 @@ static GICC: SpinNoIrq>> = SpinNoIrq::new(None); pub fn set_enable(irq_num: usize, enabled: bool) { trace!("GICD set enable: {} {}", irq_num, enabled); - let mut gicd = GICD.lock(); - let d = gicd.as_mut().unwrap(); - if enabled { - d.irq_enable(irq_num.into()); - } else { - d.irq_disable(irq_num.into()); - } + // let mut gicd = GICD.lock(); + // let d = gicd.as_mut().unwrap(); + // if enabled { + // d.irq_enable(irq_num.into()); + // } else { + // d.irq_disable(irq_num.into()); + // } } /// Registers an IRQ handler for the given IRQ. @@ -76,26 +80,59 @@ pub fn dispatch_irq(irq_num: usize) { } } +const GICR_WAKER_PSLEEP_BIT: usize = 0x2; +const GICR_WAKER_CASLEEP_BIT: usize = 0x4; +const GICR_WAKER_OFFSET: usize = 0x14; + +fn wake_up_gicr(cpu_id: usize) { + let cur_gicr_base = phys_to_virt(GICR_BASE).add(cpu_id * 0x20000); + + debug!("CPU {} GICR base: {:#x}", cpu_id, cur_gicr_base); + + let gicr_waker = cur_gicr_base.add(GICR_WAKER_OFFSET).as_mut_ptr_of::(); + + unsafe { + // Set the GICR WAKER register to wake up the CPU. + *gicr_waker &= !(GICR_WAKER_PSLEEP_BIT as u32); + + // Wait until the GICR WAKER register is not in sleep state. + while (*gicr_waker & GICR_WAKER_CASLEEP_BIT as u32) != 0 { + // Spin until the CPU is awake. + } + } +} + /// Initializes GICD, GICC on the primary CPU. pub(crate) fn init_primary() { - info!("Initialize GICv3..."); - let gicd = arm_gic_driver::v3::Gic::new( - NonNull::new(phys_to_virt(GICD_BASE).as_mut_ptr()).unwrap(), - NonNull::new(phys_to_virt(GICC_BASE).as_mut_ptr()).unwrap(), - arm_gic_driver::v3::Security::OneNS, - ); - let interface = gicd.cpu_interface(); - - GICD.lock().replace(gicd); - GICC.lock().replace(interface); - - disable_irqs(); + warn!("Initialize GICv3..."); + + wake_up_gicr(this_cpu_id()); + + // SAFETY: Set the SRE[0] bit to 1 to enable Group 1 interrupts. + ICC_SRE_EL2.set(0b1); + + // let waker = self[current_cpu().id].WAKER.get(); + // self[current_cpu().id].WAKER.set(waker & !GICR_WAKER_PSLEEP_BIT as u32); + // while (self[current_cpu().id].WAKER.get() & GICR_WAKER_CASLEEP_BIT as u32) != 0 {} + + // let gicd = arm_gic_driver::v3::Gic::new( + // NonNull::new(phys_to_virt(GICD_BASE).as_mut_ptr()).unwrap(), + // NonNull::new(phys_to_virt(GICC_BASE).as_mut_ptr()).unwrap(), + // arm_gic_driver::v3::Security::OneNS, + // ); + // let interface = gicd.cpu_interface(); + + // GICD.lock().replace(gicd); + // GICC.lock().replace(interface); + + // disable_irqs(); } /// Initializes GICC on secondary CPUs. #[cfg(feature = "smp")] pub(crate) fn init_secondary() { - let interface = GICD.lock().as_mut().unwrap().cpu_interface(); - GICC.lock().replace(interface); - GICC.lock().as_mut().unwrap().setup(); + // let interface = GICD.lock().as_mut().unwrap().cpu_interface(); + // GICC.lock().replace(interface); + // GICC.lock().as_mut().unwrap().setup(); + wake_up_gicr(this_cpu_id()); } diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 1261f95116..6ecc57d760 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -173,8 +173,8 @@ pub extern "C" fn rust_main(cpu_id: usize, dtb: usize) -> ! { #[cfg(feature = "irq")] { - info!("Initialize interrupt handlers..."); - init_interrupt(); + // info!("Initialize interrupt handlers..."); + // init_interrupt(); } #[cfg(all(feature = "tls", not(feature = "multitask")))] From 863cd50e01fefd1759b211efc75428b1bb876f89 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Sat, 7 Jun 2025 14:36:57 +0800 Subject: [PATCH 33/48] [wip] boot on qemu --- .../axhal/src/platform/aarch64_common/boot_el2.rs | 4 ++++ .../src/platform/aarch64_common/generic_timer.rs | 13 ++++++++----- modules/axhal/src/platform/aarch64_common/gicv3.rs | 4 ++-- modules/axtask/src/api.rs | 6 +++--- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index 931a00e991..afcbc59f1a 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -51,6 +51,10 @@ unsafe fn init_mmu_el2() { + HCR_EL2::TSC::EnableTrapEl1SmcToEl2, ); + // Enable the GICv3 interface so that guest VMs can operate + // gicv3 with CSRs. + ICH_HCR_EL2.modify(ICH_HCR_EL2::En.val(1)); + // Device-nGnRE memory let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; // Normal memory diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index d6903a87ed..ee43e5e789 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -94,11 +94,14 @@ pub(crate) fn init_early() { } pub(crate) fn init_percpu() { - use aarch64_cpu::registers::{CNTHCTL_EL2, CNTVOFF_EL2}; - use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; - // Disable EL1 timer traps and the timer offset. - CNTHCTL_EL2.modify(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - CNTVOFF_EL2.set(0); + #[cfg(all(not(feature = "irq"), feature = "hv"))] + { + use aarch64_cpu::registers::{CNTHCTL_EL2, CNTVOFF_EL2}; + use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + // Disable EL1 timer traps and the timer offset. + CNTHCTL_EL2.modify(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + CNTVOFF_EL2.set(0); + } #[cfg(all(feature = "irq", not(feature = "hv")))] { diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index ff63e9ae5e..540a91387b 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -106,7 +106,7 @@ fn wake_up_gicr(cpu_id: usize) { pub(crate) fn init_primary() { warn!("Initialize GICv3..."); - wake_up_gicr(this_cpu_id()); + // wake_up_gicr(this_cpu_id()); // SAFETY: Set the SRE[0] bit to 1 to enable Group 1 interrupts. ICC_SRE_EL2.set(0b1); @@ -134,5 +134,5 @@ pub(crate) fn init_secondary() { // let interface = GICD.lock().as_mut().unwrap().cpu_interface(); // GICC.lock().replace(interface); // GICC.lock().as_mut().unwrap().setup(); - wake_up_gicr(this_cpu_id()); + // wake_up_gicr(this_cpu_id()); } diff --git a/modules/axtask/src/api.rs b/modules/axtask/src/api.rs index 7cf96a043c..d0c8f21bd1 100644 --- a/modules/axtask/src/api.rs +++ b/modules/axtask/src/api.rs @@ -210,8 +210,8 @@ pub fn exit(exit_code: i32) -> ! { pub fn run_idle() -> ! { loop { yield_now(); - debug!("idle task: waiting for IRQs..."); - #[cfg(feature = "irq")] - axhal::arch::wait_for_irqs(); + // debug!("idle task: waiting for IRQs..."); + // #[cfg(feature = "irq")] + // axhal::arch::wait_for_irqs(); } } From e34f5165bda8ce20a6cd0140b47c74b4cfcf53e9 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Sat, 7 Jun 2025 20:38:46 +0800 Subject: [PATCH 34/48] fixes after merge --- .gitignore | 1 + Cargo.lock | 23 +++++++---------------- Cargo.toml | 2 ++ examples/gicv3_tester/Cargo.toml | 10 ++++++++++ examples/gicv3_tester/src/main.rs | 22 ++++++++++++++++++++++ 5 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 examples/gicv3_tester/Cargo.toml create mode 100644 examples/gicv3_tester/src/main.rs diff --git a/.gitignore b/.gitignore index 2a2c5a3dd2..9e0184934b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ qemu.log rusty-tags.vi /.project* /.axconfig.* +zignore-* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 8e0cd516ec..1f53932ddd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1061,6 +1061,13 @@ dependencies = [ "wasi", ] +[[package]] +name = "gicv3_tester" +version = "0.1.0" +dependencies = [ + "axstd", +] + [[package]] name = "glob" version = "0.3.2" @@ -1563,22 +1570,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "rdif-base" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023bd968dcec1d1d01092fdf0e6473cd4fee4b3a4140e524d5952cf332882a36" - -[[package]] -name = "rdif-intc" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4dce859ee52f984c78d41f647eb3266e58824e385583ae64a5467597c84fd5a" -dependencies = [ - "cfg-if", - "rdif-base", -] - [[package]] name = "regex" version = "1.11.1" diff --git a/Cargo.toml b/Cargo.toml index 8d44d4f470..b49d1cc0c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,8 @@ members = [ "examples/httpserver", "examples/httpserver", "examples/shell", + + "examples/gicv3_tester", ] [workspace.package] diff --git a/examples/gicv3_tester/Cargo.toml b/examples/gicv3_tester/Cargo.toml new file mode 100644 index 0000000000..8da9e82aa0 --- /dev/null +++ b/examples/gicv3_tester/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "gicv3_tester" +version = "0.1.0" +edition.workspace = true +authors = ["Yuekai Jia "] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axstd = { workspace = true, features = ["irq", "multitask", "paging"] } diff --git a/examples/gicv3_tester/src/main.rs b/examples/gicv3_tester/src/main.rs new file mode 100644 index 0000000000..29616123fa --- /dev/null +++ b/examples/gicv3_tester/src/main.rs @@ -0,0 +1,22 @@ +#![no_std] +#![no_main] + +use core::time::Duration; + +use axstd::println; +use axstd::os::arceos::api; +use axstd::os::arceos::modules; + +use api::time::ax_monotonic_time; + +#[unsafe(no_mangle)] +fn main() { + println!("Hello, world!"); + + println!("Current monotonic time: {:?}", ax_monotonic_time()); + + for _ in 0..100 { + modules::axtask::sleep(Duration::from_millis(100)); + println!("Current monotonic time: {:?}", ax_monotonic_time()); + } +} From f44a68f9a9a74eb8280fbcb569f4ff2696b91565 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Sat, 7 Jun 2025 20:49:17 +0800 Subject: [PATCH 35/48] update gicv3_tester --- examples/gicv3_tester/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/gicv3_tester/src/main.rs b/examples/gicv3_tester/src/main.rs index 29616123fa..77d6532409 100644 --- a/examples/gicv3_tester/src/main.rs +++ b/examples/gicv3_tester/src/main.rs @@ -15,8 +15,8 @@ fn main() { println!("Current monotonic time: {:?}", ax_monotonic_time()); - for _ in 0..100 { - modules::axtask::sleep(Duration::from_millis(100)); + loop { + modules::axtask::sleep(Duration::from_millis(500)); println!("Current monotonic time: {:?}", ax_monotonic_time()); } } From 00480405fc3e89f40d6d4263054687ba5918c873 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Wed, 11 Jun 2025 15:43:16 +0800 Subject: [PATCH 36/48] disable el2 time ticks to allow axvisor run correctly --- Cargo.lock | 16 -------------- .../src/platform/aarch64_common/boot_el2.rs | 4 ++++ .../platform/aarch64_common/generic_timer.rs | 22 ++++++++++--------- .../src/platform/aarch64_common/gicv3.rs | 1 + 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b63c406552..1f53932ddd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1570,22 +1570,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "rdif-base" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023bd968dcec1d1d01092fdf0e6473cd4fee4b3a4140e524d5952cf332882a36" - -[[package]] -name = "rdif-intc" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4dce859ee52f984c78d41f647eb3266e58824e385583ae64a5467597c84fd5a" -dependencies = [ - "cfg-if", - "rdif-base", -] - [[package]] name = "regex" version = "1.11.1" diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index afcbc59f1a..76e2a43c86 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -43,6 +43,10 @@ unsafe fn init_mmu_el2() { // * Virtual IRQ interrupts are enabled; // * Physical FIQ interrupts are taken to EL2; // * Virtual FIQ interrupts are enabled. + // + // To passthrough IRQs to EL1, disable `HCR_EL2::IMO::EnableVirtualIRQ` and + // `HCR_EL2::FMO::EnableVirtualFIQ`, which can be done in `arm_vcpu`. + // `HCR_EL2` here is used only outside of guest VMs. HCR_EL2.modify( HCR_EL2::VM::Enable + HCR_EL2::RW::EL1IsAarch64 diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index ee43e5e789..14119bb2b3 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -108,16 +108,18 @@ pub(crate) fn init_percpu() { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); CNTP_TVAL_EL0.set(0); } - // #[cfg(all(feature = "irq", feature = "hv"))] - // { - // unsafe { - // // ENABLE, bit [0], Enables the timer. - // // * 0b0: Timer disabled. - // // * 0b1: Timer enabled. - // core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); - // core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); - // } - // } + #[cfg(all(feature = "irq", feature = "hv"))] + { + // `arm_vcpu` will set `CNTVOFF_EL2` and `CNTHCTL_EL2` later. + + unsafe { + // ENABLE, bit [0], Enables the timer. + // * 0b0: Timer disabled. + // * 0b1: Timer enabled. + // core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); + // core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); + } + } #[cfg(feature = "irq")] crate::platform::irq::set_enable(crate::platform::irq::TIMER_IRQ_NUM, true); } diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index 033ce1b8e0..62b30058bd 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -87,6 +87,7 @@ pub fn fetch_irq() -> usize { /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. pub fn dispatch_irq(irq_num: usize) { + trace!("dispatch_irq: {}", irq_num); let intid: Option; if irq_num == 0 { intid = GICR.lock().as_mut().unwrap().ack(); From 4b597c252a59ec6c526beb2bfebf1892c8cb555b Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 24 Jun 2025 09:54:46 +0800 Subject: [PATCH 37/48] re-enabling `init_interrupt` when not in aarch64-hv mode --- modules/axruntime/src/lib.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 3b96ba1b80..0a5ed10ff2 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -173,8 +173,16 @@ pub extern "C" fn rust_main(cpu_id: usize, dtb: usize) -> ! { #[cfg(feature = "irq")] { - // info!("Initialize interrupt handlers..."); - // init_interrupt(); + #[cfg(any(not(target_arch = "aarch64"), not(feature = "hv")))] + { + info!("Initialize interrupt handlers..."); + init_interrupt(); + } + + #[cfg(all(target_arch = "aarch64", feature = "hv"))] + { + info!("`init_interrupt` skipped for aarch64 hypervisor."); + } } #[cfg(all(feature = "tls", not(feature = "multitask")))] From 5d3aff6189a44524850e178e483655503495b794 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 15:06:01 +0800 Subject: [PATCH 38/48] update `memory_addr` --- Cargo.lock | 25 +++++++++++++------------ modules/axalloc/Cargo.toml | 2 +- modules/axdma/Cargo.toml | 2 +- modules/axhal/Cargo.toml | 6 +++--- modules/axmm/Cargo.toml | 4 ++-- modules/axmm/src/aspace.rs | 2 +- modules/axtask/Cargo.toml | 2 +- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f53932ddd..51d013d93f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,15 +1327,15 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memory_addr" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f769efcf10b9dfb4c913bebb409cda77b1a3f072b249bf5465e250bcb30eb49" +checksum = "3d4054cba279515fa87761b101d857333ce06391dbe8f18a11347204a7111416" [[package]] name = "memory_set" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335675b7ab07460f532d3b2b557313be73037fdf81b65464d8c0d8bd90d2fbf9" +checksum = "e7d47cbc25a4d00427f9070fd768eaf907f19c903fb72b547b19db2d56b9408e" dependencies = [ "memory_addr", ] @@ -1379,9 +1379,9 @@ checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "page_table_entry" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c097d641745a066856a26eed6e486d4430bb3e32c94f1203ea09c63239b360a0" +checksum = "ba811ef8ca8fb33d776e128624cb4fe25c9804cab96f83b822d4322431e6dd5a" dependencies = [ "aarch64-cpu 10.0.0", "bitflags 2.9.1", @@ -1391,10 +1391,11 @@ dependencies = [ [[package]] name = "page_table_multiarch" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4647889585d29762d747be0916d6d28db72967a697d142be86f187a6b496832a" +checksum = "98cb76e21ce462270afd83b331599d5b83f876c2a98c0a70382b20d73e1da6be" dependencies = [ + "bitmaps", "log", "memory_addr", "page_table_entry", @@ -1601,9 +1602,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "riscv" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea8ff73d3720bdd0a97925f0bf79ad2744b6da8ff36be3840c48ac81191d7a7" +checksum = "0f1671c79a01a149fe000af2429ce9ccc8e58cdecda72672355d50e5536b363c" dependencies = [ "critical-section", "embedded-hal", @@ -1614,9 +1615,9 @@ dependencies = [ [[package]] name = "riscv-macros" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f265be5d634272320a7de94cea15c22a3bfdd4eb42eb43edc528415f066a1f25" +checksum = "e8c4aa1ea1af6dcc83a61be12e8189f9b293c3ba5a487778a4cd89fb060fdbbc" dependencies = [ "proc-macro2", "quote", diff --git a/modules/axalloc/Cargo.toml b/modules/axalloc/Cargo.toml index c0314aa1ef..139012efdc 100644 --- a/modules/axalloc/Cargo.toml +++ b/modules/axalloc/Cargo.toml @@ -21,6 +21,6 @@ page-alloc-4g = ["allocator/page-alloc-4g"] # Support up to 4G memory capacity log = "=0.4.21" cfg-if = "1.0" kspin = "0.1" -memory_addr = "0.3" +memory_addr = "0.4" axerrno = "0.1" allocator = { git = "https://github.com/arceos-org/allocator.git", tag ="v0.1.1", features = ["bitmap"] } diff --git a/modules/axdma/Cargo.toml b/modules/axdma/Cargo.toml index 12095f703a..1dd462595c 100644 --- a/modules/axdma/Cargo.toml +++ b/modules/axdma/Cargo.toml @@ -13,7 +13,7 @@ documentation = "https://arceos-org.github.io/arceos/axdma/index.html" [dependencies] log = "=0.4.21" kspin = "0.1" -memory_addr = "0.3" +memory_addr = "0.4" axerrno = "0.1" allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } axalloc = { workspace = true } diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index ef135865d4..1788402665 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -42,10 +42,10 @@ kspin = "0.1" int_ratio = "0.1.1" lazyinit = "0.2" percpu = "0.2" -memory_addr = "0.3" +memory_addr = "0.4" handler_table = "0.1" page_table_entry = "0.5" -page_table_multiarch = "0.5" +page_table_multiarch = { version = "0.5", features = ["copy-from"] } axlog = { workspace = true } axconfig = { workspace = true } axalloc = { workspace = true, optional = true } @@ -60,7 +60,7 @@ raw-cpuid = "11.3" x86_rtc = { version = "0.1", optional = true } [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] -riscv = "0.12" +riscv = "0.14" sbi-rt = { version = "0.0.3", features = ["legacy"] } riscv_plic = { version = "0.1.0" } riscv_goldfish = { version = "0.1", optional = true } diff --git a/modules/axmm/Cargo.toml b/modules/axmm/Cargo.toml index faac117267..b53e27ecb4 100644 --- a/modules/axmm/Cargo.toml +++ b/modules/axmm/Cargo.toml @@ -17,6 +17,6 @@ axconfig = { workspace = true } log = "=0.4.21" axerrno = "0.1" lazyinit = "0.2" -memory_addr = "0.3" +memory_addr = "0.4" kspin = "0.1" -memory_set = "0.3" +memory_set = "0.4" diff --git a/modules/axmm/src/aspace.rs b/modules/axmm/src/aspace.rs index 39e377e065..0c52a2ffc7 100644 --- a/modules/axmm/src/aspace.rs +++ b/modules/axmm/src/aspace.rs @@ -85,7 +85,7 @@ impl AddrSpace { size: usize, limit: VirtAddrRange, ) -> Option { - self.areas.find_free_area(hint, size, limit) + self.areas.find_free_area(hint, size, limit, PAGE_SIZE_4K) } /// Add a new linear mapping. diff --git a/modules/axtask/Cargo.toml b/modules/axtask/Cargo.toml index 830d678f66..3a06503b19 100644 --- a/modules/axtask/Cargo.toml +++ b/modules/axtask/Cargo.toml @@ -43,7 +43,7 @@ axconfig = { workspace = true, optional = true } percpu = { version = "0.2", optional = true } kspin = { version = "0.1", optional = true } lazyinit = { version = "0.2", optional = true } -memory_addr = { version = "0.3", optional = true } +memory_addr = { version = "0.4", optional = true } timer_list = { version = "0.1", optional = true } kernel_guard = { version = "0.1", optional = true } crate_interface = { version = "0.1", optional = true } From 3cb30cefe1d1b3ec27038db317caa5f616048583 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 17:36:16 +0800 Subject: [PATCH 39/48] some small fixes --- modules/axhal/src/arch/aarch64/mod.rs | 2 +- .../src/platform/aarch64_common/boot_el2.rs | 3 +-- .../src/platform/aarch64_common/gicv3.rs | 19 ------------------- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/modules/axhal/src/arch/aarch64/mod.rs b/modules/axhal/src/arch/aarch64/mod.rs index 8a663f51c6..7b2ac336e9 100644 --- a/modules/axhal/src/arch/aarch64/mod.rs +++ b/modules/axhal/src/arch/aarch64/mod.rs @@ -20,7 +20,7 @@ pub use self::context::{FpState, TaskContext, TrapFrame}; /// Allows the current CPU to respond to interrupts. #[inline] pub fn enable_irqs() { - // unsafe { asm!("msr daifclr, #2") }; + unsafe { asm!("msr daifclr, #2") }; } /// Makes the current CPU to ignore interrupts. diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index 76e2a43c86..ebe600e675 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -55,8 +55,7 @@ unsafe fn init_mmu_el2() { + HCR_EL2::TSC::EnableTrapEl1SmcToEl2, ); - // Enable the GICv3 interface so that guest VMs can operate - // gicv3 with CSRs. + // By default we enable the virtual GIC CPU interface. `arm_vcpu` may override this. ICH_HCR_EL2.modify(ICH_HCR_EL2::En.val(1)); // Device-nGnRE memory diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index 62b30058bd..3e94d14917 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -275,25 +275,6 @@ pub(crate) fn init_primary() { GICD.lock().replace(gicd); GICR.lock().replace(interface); - - // SAFETY: Set the SRE[0] bit to 1 to enable Group 1 interrupts. - ICC_SRE_EL2.set(0b1); - - // let waker = self[current_cpu().id].WAKER.get(); - // self[current_cpu().id].WAKER.set(waker & !GICR_WAKER_PSLEEP_BIT as u32); - // while (self[current_cpu().id].WAKER.get() & GICR_WAKER_CASLEEP_BIT as u32) != 0 {} - - // let gicd = arm_gic_driver::v3::Gic::new( - // NonNull::new(phys_to_virt(GICD_BASE).as_mut_ptr()).unwrap(), - // NonNull::new(phys_to_virt(GICC_BASE).as_mut_ptr()).unwrap(), - // arm_gic_driver::v3::Security::OneNS, - // ); - // let interface = gicd.cpu_interface(); - - // GICD.lock().replace(gicd); - // GICC.lock().replace(interface); - - // disable_irqs(); } /// Initializes GICR on secondary CPUs. From a8dc94e820f6fbfb938044692a16777701977c23 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 17:41:00 +0800 Subject: [PATCH 40/48] rename `gicv3_tester` to `ticker` --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- examples/{gicv3_tester => ticker}/Cargo.toml | 4 ++-- examples/{gicv3_tester => ticker}/src/main.rs | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) rename examples/{gicv3_tester => ticker}/Cargo.toml (77%) rename examples/{gicv3_tester => ticker}/src/main.rs (93%) diff --git a/Cargo.lock b/Cargo.lock index 51d013d93f..53227619b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1061,13 +1061,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gicv3_tester" -version = "0.1.0" -dependencies = [ - "axstd", -] - [[package]] name = "glob" version = "0.3.2" @@ -1836,6 +1829,13 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "ticker" +version = "0.1.0" +dependencies = [ + "axstd", +] + [[package]] name = "timer_list" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b49d1cc0c9..28da4705af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ members = [ "examples/httpserver", "examples/shell", - "examples/gicv3_tester", + "examples/ticker", ] [workspace.package] diff --git a/examples/gicv3_tester/Cargo.toml b/examples/ticker/Cargo.toml similarity index 77% rename from examples/gicv3_tester/Cargo.toml rename to examples/ticker/Cargo.toml index 8da9e82aa0..0d0ed66830 100644 --- a/examples/gicv3_tester/Cargo.toml +++ b/examples/ticker/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "gicv3_tester" +name = "ticker" version = "0.1.0" edition.workspace = true -authors = ["Yuekai Jia "] +authors = ["aarkegz "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/examples/gicv3_tester/src/main.rs b/examples/ticker/src/main.rs similarity index 93% rename from examples/gicv3_tester/src/main.rs rename to examples/ticker/src/main.rs index 77d6532409..518b4640e4 100644 --- a/examples/gicv3_tester/src/main.rs +++ b/examples/ticker/src/main.rs @@ -11,7 +11,7 @@ use api::time::ax_monotonic_time; #[unsafe(no_mangle)] fn main() { - println!("Hello, world!"); + println!("Ticking!"); println!("Current monotonic time: {:?}", ax_monotonic_time()); From 6a5f9e21d5062cbed93399020b07a31701616cf8 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 17:41:21 +0800 Subject: [PATCH 41/48] allow `init_interrupt` when building `hv` on `aarch64`, as timer is disabled --- modules/axruntime/src/lib.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 0a5ed10ff2..bffb5c07eb 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -173,16 +173,8 @@ pub extern "C" fn rust_main(cpu_id: usize, dtb: usize) -> ! { #[cfg(feature = "irq")] { - #[cfg(any(not(target_arch = "aarch64"), not(feature = "hv")))] - { - info!("Initialize interrupt handlers..."); - init_interrupt(); - } - - #[cfg(all(target_arch = "aarch64", feature = "hv"))] - { - info!("`init_interrupt` skipped for aarch64 hypervisor."); - } + info!("Initialize interrupt handlers..."); + init_interrupt(); } #[cfg(all(feature = "tls", not(feature = "multitask")))] From 0e5dd38d58b0e32a750ae9eb07b1b08e43f7019d Mon Sep 17 00:00:00 2001 From: Su Mingxian Date: Tue, 15 Jul 2025 17:42:47 +0800 Subject: [PATCH 42/48] fix type Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- configs/platforms/aarch64-qemu-virt.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/platforms/aarch64-qemu-virt.toml b/configs/platforms/aarch64-qemu-virt.toml index add3f14f06..71260bb840 100644 --- a/configs/platforms/aarch64-qemu-virt.toml +++ b/configs/platforms/aarch64-qemu-virt.toml @@ -97,7 +97,7 @@ uart-irq = 1 # uint gicc-paddr = 0x0801_0000 # uint # GIC Distributor base address gicd-paddr = 0x0800_0000 # uint -# GIC Rdistributor base address +# GIC Redistributor base address gicr-paddr = 0x080a_0000 # PSCI From 61945d6eb951a65cd79d4dc7d1ae9b7d6718af97 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 19:46:42 +0800 Subject: [PATCH 43/48] reformatted --- examples/ticker/src/main.rs | 2 +- .../axhal/src/platform/aarch64_common/gicv3.rs | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/examples/ticker/src/main.rs b/examples/ticker/src/main.rs index 518b4640e4..f8b04313ad 100644 --- a/examples/ticker/src/main.rs +++ b/examples/ticker/src/main.rs @@ -3,9 +3,9 @@ use core::time::Duration; -use axstd::println; use axstd::os::arceos::api; use axstd::os::arceos::modules; +use axstd::println; use api::time::ax_monotonic_time; diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index 3e94d14917..bf4a75d7a5 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -1,14 +1,14 @@ extern crate alloc; -use alloc::boxed::Box; use crate::{arch::disable_irqs, cpu::this_cpu_id, irq::IrqHandler, mem::phys_to_virt}; +use alloc::boxed::Box; use arm_gic_driver::*; +#[cfg(feature = "hv")] +use arm_gicv2::GicHypervisorInterface; +use arm_gicv2::{InterruptType, translate_irq}; use axconfig::devices::{GICD_PADDR, GICR_PADDR, UART_IRQ}; use core::{panic, ptr::NonNull}; use kspin::SpinNoIrq; -use arm_gicv2::{translate_irq, InterruptType}; -#[cfg(feature = "hv")] -use arm_gicv2::GicHypervisorInterface; use memory_addr::{MemoryAddr, PhysAddr}; use aarch64_cpu::registers::{ICC_SRE_EL2, SCTLR_EL3::I}; @@ -218,13 +218,12 @@ fn write_lr(id: usize, val: u64) { } fn send_sgi_inner(aff3: u8, aff2: u8, aff1: u8, target: u8, vector: usize, to_all: bool) { - let value = - ((vector & 0xF) << 24) | // vector + let value = ((vector & 0xF) << 24) | // vector (1 << target) | // target bitmap ((aff1 as usize) << 16) | // affinity level 1 ((aff2 as usize) << 32) | // affinity level 2 ((aff3 as usize) << 48) | // affinity level 3 - ((to_all as usize) << 40); // interrupt routing mode + ((to_all as usize) << 40); // interrupt routing mode write_sysreg!(icc_sgi1r_el1, value as _); } @@ -252,7 +251,7 @@ pub fn send_sgi_all(vector: usize) { } // dummy implementation -pub struct MyVgic{} +pub struct MyVgic {} /// Initializes GICD, GICC on the primary CPU. pub(crate) fn init_primary() { From adbe081b03a2e686c7027d2aeae70c6286d92fc6 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 19:48:46 +0800 Subject: [PATCH 44/48] fix a compiler error caused by an update of `stvec::write` in `riscv` crate --- modules/axhal/src/arch/riscv/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/axhal/src/arch/riscv/mod.rs b/modules/axhal/src/arch/riscv/mod.rs index 07caf58643..f3df234408 100644 --- a/modules/axhal/src/arch/riscv/mod.rs +++ b/modules/axhal/src/arch/riscv/mod.rs @@ -5,8 +5,7 @@ mod context; mod trap; use memory_addr::{PhysAddr, VirtAddr}; -use riscv::asm; -use riscv::register::{satp, sstatus, stvec}; +use riscv::{asm, register::{satp, sstatus, stvec::{self, Stvec, TrapMode}}}; #[cfg(feature = "uspace")] pub use self::context::UspaceContext; @@ -85,7 +84,10 @@ pub fn flush_tlb(vaddr: Option) { /// Writes Supervisor Trap Vector Base Address Register (`stvec`). #[inline] pub fn set_trap_vector_base(stvec: usize) { - unsafe { stvec::write(stvec, stvec::TrapMode::Direct) } + let mut stvec = Stvec::from_bits(stvec); + stvec.set_trap_mode(TrapMode::Direct); + + unsafe { stvec::write(stvec) } } /// Reads the thread pointer of the current CPU. From 7424be8fbd70a067024d84600a618c1f0f65f2f2 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 19:55:26 +0800 Subject: [PATCH 45/48] rename IPI-related functions in `axhal`, add dummy implementation for loongarch64 --- modules/axhal/src/arch/riscv/mod.rs | 8 +++++++- modules/axhal/src/irq.rs | 2 +- modules/axhal/src/platform/aarch64_common/gic.rs | 4 ++-- modules/axhal/src/platform/aarch64_common/gicv3.rs | 4 ++-- modules/axhal/src/platform/dummy/mod.rs | 4 ++-- .../axhal/src/platform/loongarch64_qemu_virt/irq.rs | 12 ++++++++++++ modules/axhal/src/platform/riscv64_qemu_virt/irq.rs | 8 ++++---- modules/axhal/src/platform/x86_pc/apic.rs | 4 ++-- modules/axipi/src/lib.rs | 4 ++-- 9 files changed, 34 insertions(+), 16 deletions(-) diff --git a/modules/axhal/src/arch/riscv/mod.rs b/modules/axhal/src/arch/riscv/mod.rs index f3df234408..226d1b694f 100644 --- a/modules/axhal/src/arch/riscv/mod.rs +++ b/modules/axhal/src/arch/riscv/mod.rs @@ -5,7 +5,13 @@ mod context; mod trap; use memory_addr::{PhysAddr, VirtAddr}; -use riscv::{asm, register::{satp, sstatus, stvec::{self, Stvec, TrapMode}}}; +use riscv::{ + asm, + register::{ + satp, sstatus, + stvec::{self, Stvec, TrapMode}, + }, +}; #[cfg(feature = "uspace")] pub use self::context::UspaceContext; diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index e611e24fa0..b71c84ecf7 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -11,7 +11,7 @@ pub use crate::platform::irq::{register_handler, set_enable}; pub use crate::platform::irq::{MyVgic, inject_interrupt}; #[cfg(feature = "ipi")] -pub use crate::platform::irq::{IPI_IRQ_NUM, send_sgi_all, send_sgi_one}; +pub use crate::platform::irq::{IPI_IRQ_NUM, send_ipi_all_others, send_ipi_one}; #[cfg(target_arch = "aarch64")] pub use crate::platform::irq::fetch_irq; diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 8b53faed30..5d63ebe1a8 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -51,12 +51,12 @@ pub fn set_enable(irq_num: usize, enabled: bool) { } /// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. -pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { +pub fn send_ipi_one(dest_cpu_id: usize, irq_num: usize) { GICD.lock().send_sgi(dest_cpu_id, irq_num); } /// Sends a broadcast IPI to all CPUs. -pub fn send_sgi_all(irq_num: usize) { +pub fn send_ipi_all_others(irq_num: usize) { GICD.lock().send_sgi_all_except_self(irq_num); } diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index bf4a75d7a5..447ba1141f 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -229,7 +229,7 @@ fn send_sgi_inner(aff3: u8, aff2: u8, aff1: u8, target: u8, vector: usize, to_al } /// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. -pub fn send_sgi_one(dest: usize, vector: usize) { +pub fn send_ipi_one(dest: usize, vector: usize) { #[cfg(platform_family = "aarch64-rk3588j")] { // learnt from hVisor, that rockchip socs follow the 0.0.x.0 affinity scheme @@ -246,7 +246,7 @@ pub fn send_sgi_one(dest: usize, vector: usize) { } /// Sends a broadcast IPI to all CPUs. -pub fn send_sgi_all(vector: usize) { +pub fn send_ipi_all_others(vector: usize) { send_sgi_inner(0, 0, 0, 0, vector, true); } diff --git a/modules/axhal/src/platform/dummy/mod.rs b/modules/axhal/src/platform/dummy/mod.rs index 13ab8b0200..cedd20a359 100644 --- a/modules/axhal/src/platform/dummy/mod.rs +++ b/modules/axhal/src/platform/dummy/mod.rs @@ -93,10 +93,10 @@ pub mod irq { pub const IPI_IRQ_NUM: usize = 0; #[cfg(feature = "ipi")] - pub fn send_sgi_one(dest_cpu: usize, irq_num: usize) {} + pub fn send_ipi_one(dest_cpu: usize, irq_num: usize) {} #[cfg(feature = "ipi")] - pub fn send_sgi_all(irq_num: usize) {} + pub fn send_ipi_all_others(irq_num: usize) {} } /// Initializes the platform devices for the primary CPU. diff --git a/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs b/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs index 0d83f090a6..209a3effaa 100644 --- a/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs @@ -37,3 +37,15 @@ pub fn dispatch_irq(irq_num: usize) { } crate::irq::dispatch_irq_common(irq_num) } + +/// The IPI IRQ number. An placeholder now. +#[cfg(feature = "ipi")] +pub const IPI_IRQ_NUM: usize = 0; + +/// Send an IPI to the specified CPU. +#[cfg(feature = "ipi")] +pub fn send_ipi_one(dest_cpu: usize, irq_num: usize) {} + +/// Send a broadcast IPI to all CPUs. +#[cfg(feature = "ipi")] +pub fn send_ipi_all_others(irq_num: usize) {} diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index 6b4c5122c8..a9cb37671a 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -120,20 +120,20 @@ pub fn register_handler(scause: usize, handler: IrqHandler) -> bool { } /// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. -pub fn send_sgi_one(dest_cpu_id: usize, _irq_num: usize) { +pub fn send_ipi_one(dest_cpu_id: usize, _irq_num: usize) { let res = send_ipi(HartMask::from_mask_base(1, dest_cpu_id)); if res.is_err() { - warn!("send_sgi_one failed: {:?}", res); + warn!("send_ipi_one failed: {:?}", res); } } /// Sends a broadcast IPI to all CPUs. -pub fn send_sgi_all(_irq_num: usize) { +pub fn send_ipi_all_others(_irq_num: usize) { for i in 0..axconfig::SMP { if i != this_cpu_id() { let res = send_ipi(HartMask::from_mask_base(1, i)); if res.is_err() { - warn!("send_sgi_all failed: {:?}", res); + warn!("send_ipi_all_others failed: {:?}", res); break; } } diff --git a/modules/axhal/src/platform/x86_pc/apic.rs b/modules/axhal/src/platform/x86_pc/apic.rs index 1836ce3bec..7f0184836c 100644 --- a/modules/axhal/src/platform/x86_pc/apic.rs +++ b/modules/axhal/src/platform/x86_pc/apic.rs @@ -60,14 +60,14 @@ pub fn register_handler(vector: usize, handler: crate::irq::IrqHandler) -> bool } /// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. -pub fn send_sgi_one(dest_cpu_id: usize, irq_num: usize) { +pub fn send_ipi_one(dest_cpu_id: usize, irq_num: usize) { unsafe { local_apic().send_ipi(irq_num as _, dest_cpu_id as _); }; } /// Sends a broadcast IPI to all CPUs. -pub fn send_sgi_all(irq_num: usize) { +pub fn send_ipi_all_others(irq_num: usize) { use x2apic::lapic::IpiAllShorthand; unsafe { local_apic().send_ipi_all(irq_num as _, IpiAllShorthand::AllExcludingSelf); diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs index ebfdaa0723..5632897cbc 100644 --- a/modules/axipi/src/lib.rs +++ b/modules/axipi/src/lib.rs @@ -36,7 +36,7 @@ pub fn send_ipi_event_to_one>(dest_cpu: usize, callback: T) { unsafe { IPI_EVENT_QUEUE.remote_ref_raw(dest_cpu) } .lock() .push(this_cpu_id(), callback.into()); - axhal::irq::send_sgi_one(dest_cpu, IPI_IRQ_NUM); + axhal::irq::send_ipi_one(dest_cpu, IPI_IRQ_NUM); } /// Sends an IPI event to all processors except the current one. @@ -50,7 +50,7 @@ pub fn send_ipi_event_to_all>(callback: T) { .push(current_cpu_id, callback.clone().into_unicast()); } } - axhal::irq::send_sgi_all(IPI_IRQ_NUM); + axhal::irq::send_ipi_all_others(IPI_IRQ_NUM); } pub fn ipi_handler() { From 9f0a660230b0c7dc6a22944b5d5b835dc1b987d5 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 20:48:24 +0800 Subject: [PATCH 46/48] fix unit-test errors --- Cargo.lock | 14 +++++++------- examples/ticker/Cargo.toml | 2 +- examples/ticker/src/main.rs | 10 +++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53227619b6..891377d017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,6 +137,13 @@ dependencies = [ "crate_interface", ] +[[package]] +name = "arceos-ticker" +version = "0.1.0" +dependencies = [ + "axstd", +] + [[package]] name = "arceos_api" version = "0.1.0" @@ -1829,13 +1836,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "ticker" -version = "0.1.0" -dependencies = [ - "axstd", -] - [[package]] name = "timer_list" version = "0.1.0" diff --git a/examples/ticker/Cargo.toml b/examples/ticker/Cargo.toml index 0d0ed66830..dd59823f43 100644 --- a/examples/ticker/Cargo.toml +++ b/examples/ticker/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ticker" +name = "arceos-ticker" version = "0.1.0" edition.workspace = true authors = ["aarkegz "] diff --git a/examples/ticker/src/main.rs b/examples/ticker/src/main.rs index f8b04313ad..ce84c9a13a 100644 --- a/examples/ticker/src/main.rs +++ b/examples/ticker/src/main.rs @@ -1,5 +1,9 @@ -#![no_std] -#![no_main] +#![cfg_attr(feature = "axstd", no_std)] +#![cfg_attr(feature = "axstd", no_main)] + +#[cfg(feature = "axstd")] +use axstd::println; + use core::time::Duration; @@ -9,7 +13,7 @@ use axstd::println; use api::time::ax_monotonic_time; -#[unsafe(no_mangle)] +#[cfg_attr(feature = "axstd", unsafe(no_mangle))] fn main() { println!("Ticking!"); From a61a28dddf4b19b4c1e187bf9077881d9fded5a8 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 21:17:31 +0800 Subject: [PATCH 47/48] fix unit-test error --- examples/ticker/Cargo.toml | 2 +- examples/ticker/src/main.rs | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/ticker/Cargo.toml b/examples/ticker/Cargo.toml index dd59823f43..5acf582b68 100644 --- a/examples/ticker/Cargo.toml +++ b/examples/ticker/Cargo.toml @@ -7,4 +7,4 @@ authors = ["aarkegz "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axstd = { workspace = true, features = ["irq", "multitask", "paging"] } +axstd = { workspace = true, features = ["irq", "multitask", "paging"], optional = true } diff --git a/examples/ticker/src/main.rs b/examples/ticker/src/main.rs index ce84c9a13a..f99bf94043 100644 --- a/examples/ticker/src/main.rs +++ b/examples/ticker/src/main.rs @@ -1,26 +1,26 @@ #![cfg_attr(feature = "axstd", no_std)] #![cfg_attr(feature = "axstd", no_main)] -#[cfg(feature = "axstd")] -use axstd::println; - - use core::time::Duration; -use axstd::os::arceos::api; +#[cfg(feature = "axstd")] +use axstd::os::arceos::api::time::ax_monotonic_time; +#[cfg(feature = "axstd")] use axstd::os::arceos::modules; +#[cfg(feature = "axstd")] use axstd::println; -use api::time::ax_monotonic_time; - #[cfg_attr(feature = "axstd", unsafe(no_mangle))] fn main() { - println!("Ticking!"); + #[cfg(feature = "axstd")] + { + println!("Ticking!"); - println!("Current monotonic time: {:?}", ax_monotonic_time()); - - loop { - modules::axtask::sleep(Duration::from_millis(500)); println!("Current monotonic time: {:?}", ax_monotonic_time()); + + loop { + modules::axtask::sleep(Duration::from_millis(500)); + println!("Current monotonic time: {:?}", ax_monotonic_time()); + } } } From 9ae608ef30559c59daf048dee44d004da2616296 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 15 Jul 2025 21:22:07 +0800 Subject: [PATCH 48/48] fix missing docs --- modules/axipi/src/event.rs | 2 ++ modules/axipi/src/lib.rs | 1 + modules/axtask/src/task.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/modules/axipi/src/event.rs b/modules/axipi/src/event.rs index ff51ebad4d..a517999f73 100644 --- a/modules/axipi/src/event.rs +++ b/modules/axipi/src/event.rs @@ -45,6 +45,8 @@ impl From for MulticastCallback { /// An IPI event that is sent from a source CPU to the target CPU. pub struct IPIEvent { + /// The source CPU ID that sent the IPI event. pub src_cpu_id: usize, + /// The callback function that will be called when the IPI event is handled. pub callback: Callback, } diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs index 5632897cbc..fddf9b04f7 100644 --- a/modules/axipi/src/lib.rs +++ b/modules/axipi/src/lib.rs @@ -53,6 +53,7 @@ pub fn send_ipi_event_to_all>(callback: T) { axhal::irq::send_ipi_all_others(IPI_IRQ_NUM); } +/// The handler for IPI events. It retrieves the events from the queue and calls the corresponding callbacks. pub fn ipi_handler() { while let Some((src_cpu_id, callback)) = unsafe { IPI_EVENT_QUEUE.current_ref_mut_raw() } .lock() diff --git a/modules/axtask/src/task.rs b/modules/axtask/src/task.rs index b1f3088a21..c45304b372 100644 --- a/modules/axtask/src/task.rs +++ b/modules/axtask/src/task.rs @@ -406,6 +406,7 @@ impl TaskInner { self.ctx.get() } + /// Set the CPU ID where the task is running or will run. #[cfg(feature = "smp")] pub fn set_cpu_id(&self, cpu_id: u32) { self.cpu_id.store(cpu_id, Ordering::Release);