From 7d201302e6e99ff589378ff548b886467d95b1f6 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 23:46:02 +0300 Subject: [PATCH 1/2] x86/multikernel: inherit host timer calibration Spawn kernels cannot calibrate against host-owned PIT, PIC, or IO-APIC resources. Carry the host loops-per-jiffy, CPU and TSC frequencies, and local APIC timer calibration in the spawn boot context. Install fixed calibration callbacks before x86 timer initialization. Keep explicit command-line calibration authoritative. Signed-off-by: Nikolay Nikolaev --- arch/x86/include/asm/multikernel.h | 26 +++++++++++++++---- arch/x86/kernel/platform-quirks.c | 40 +++++++++++++++++++++++++++++- arch/x86/multikernel/spawn.c | 22 ++++++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 68ca8dd8cbf45e..f440441ac42c31 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -10,6 +10,8 @@ #ifndef __ASSEMBLY__ +#include +#include #include #include #include @@ -54,7 +56,7 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * from a page written by the host while running on CPUs parked by (possibly * differently built) spawn kernels. * - * The fields fall into two classes that must not be mixed up: + * The fields fall into three classes that must not be mixed up: * * - Anchor fields (self_phys, park_phys, park_cr3, ctrl_phys, * ctrl_size): the context's own identity, written once when the @@ -62,9 +64,13 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * CPU on halt or offline, so they must stay valid for the context's * whole lifetime. * - * - Dispatch fields (everything else): the wake mailbox, rewritten for - * every publication and staged into registers by the CPU that claims - * it. Reparking gets its own repark_* dispatch fields precisely so a + * - Primary boot data (bp and the calibration values appended after it): + * written before the boot CPU is released and consumed while that kernel + * initializes. Secondary and repark publications do not rewrite it. + * + * - Dispatch fields (the remaining fixed-size fields): the wake mailbox, + * rewritten for every publication and staged into registers by the CPU + * that claims it. Reparking gets its own repark_* dispatch fields so a * repark publication never overwrites the anchor: the two used to * share fields, and a repark left the anchor pointing at another * kernel's park area, which triple-faulted the next halt. @@ -91,10 +97,20 @@ struct mk_spawn_context { u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ u32 reserved; /* Padding for alignment */ - /* Variable-size struct last - size depends on kernel config */ + /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ + /* Optional boot data belongs after boot_params, in the zeroed tail. */ + unsigned long boot_lps; /* Host delay loops per second */ + unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ + unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ + unsigned long boot_apic_hz; /* Host local APIC timer frequency */ } __aligned(PAGE_SIZE); +static_assert(offsetof(struct mk_spawn_context, bp) == 144); +static_assert(offsetof(struct mk_spawn_context, boot_lps) == + 144 + sizeof(struct boot_params)); +static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); + /* Pool park loop code, copied by the host into per-instance park pages */ extern char mk_pool_park_start[]; extern char mk_pool_park_end[]; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 95d2cd2ccf74f5..feea109497efde 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -28,6 +30,35 @@ extern pmd_t *populate_extra_pmd(unsigned long vaddr); extern unsigned long orig_boot_params; #ifdef CONFIG_MULTIKERNEL +static unsigned long multikernel_cpu_khz; +static unsigned long multikernel_tsc_khz; + +static unsigned long multikernel_calibrate_cpu(void) +{ + return multikernel_cpu_khz; +} + +static unsigned long multikernel_calibrate_tsc(void) +{ + return multikernel_tsc_khz; +} + +static void __init multikernel_setup_calibration(void) +{ + phys_addr_t ctx_phys = orig_boot_params - + offsetof(struct mk_spawn_context, bp); + struct mk_spawn_context *ctx = __va(ctx_phys); + + if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + return; + + multikernel_tsc_khz = ctx->boot_tsc_khz; + multikernel_cpu_khz = ctx->boot_cpu_khz ?: ctx->boot_tsc_khz; + x86_platform.calibrate_cpu = multikernel_calibrate_cpu; + x86_platform.calibrate_tsc = multikernel_calibrate_tsc; + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); +} + /* * Custom wakeup for multikernel spawn kernels. * Uses shared spawn table instead of realmode trampoline. @@ -105,6 +136,10 @@ static void __init multikernel_parse_smp_config(void) */ apic_update_callback(wakeup_secondary_cpu_64, multikernel_wakeup_cpu); } +#else +static inline void multikernel_setup_calibration(void) +{ +} #endif /* CONFIG_MULTIKERNEL */ void __init x86_early_init_platform_quirks(void) @@ -135,6 +170,7 @@ void __init x86_early_init_platform_quirks(void) x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; break; case X86_SUBARCH_MULTIKERNEL: + multikernel_setup_calibration(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -175,7 +211,9 @@ void __init x86_early_init_platform_quirks(void) * the PIT - which belongs to the host - and then request * legacy IRQ0, which can never reach an instance CPU that * has neither a PIC nor an IO-APIC. Ticks come from the - * local APIC timer via setup_percpu_clockev() instead. + * local APIC timer initialized by setup_percpu_clockev(). + * Keeping global_clock_event unset bypasses LAPIC timer + * verification, whose fallback path requires legacy IRQ0. */ x86_init.timers.timer_init = x86_init_noop; x86_init.timers.wallclock_init = x86_init_noop; diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 932d18048c325f..095c8343164d73 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -472,6 +475,14 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, (unsigned long)instance->trampoline_va, virt_to_phys(instance->trampoline_va), park_phys); + instance->spawn_ctx->boot_lps = cpu_data(cpu).loops_per_jiffy; + if (!instance->spawn_ctx->boot_lps) + instance->spawn_ctx->boot_lps = loops_per_jiffy; + instance->spawn_ctx->boot_lps *= HZ; + instance->spawn_ctx->boot_cpu_khz = cpu_khz; + instance->spawn_ctx->boot_tsc_khz = tsc_khz; + instance->spawn_ctx->boot_apic_hz = + (unsigned long)lapic_timer_period * HZ; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -730,6 +741,17 @@ void mk_init_boot_context(phys_addr_t ctx_phys) } mk_boot_context = ctx; + /* + * A spawn kernel cannot calibrate against legacy timers because they + * belong to the host. Reuse the selected physical CPU's delay and local + * APIC timer calibration, while keeping explicit command-line values + * authoritative. + */ + if (!preset_lpj && ctx->boot_lps) + preset_lpj = DIV_ROUND_CLOSEST_ULL(ctx->boot_lps, HZ); + if (!lapic_timer_period && ctx->boot_apic_hz) + lapic_timer_period = + DIV_ROUND_CLOSEST_ULL(ctx->boot_apic_hz, HZ); /* * The host's control area (this context, the trampoline and park From 1a984b7270fd5db2e79b87dcf78dd98c9a36b913 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Wed, 12 Aug 2026 13:51:10 +0300 Subject: [PATCH 2/2] multikernel: make IPI publication ordered and recoverable Serialize shared-ring producers with a bounded owner-aware gate. Preserve FIFO publication and recover a gate only after its producer CPU is known to be parked. The gate and slot-state protocol change the private shared transport layout. Add exact pre-launch ABI checks and require an initialization acknowledgment before activation. Prepare each downlink with its target identity so callers can queue messages while the receiver boots, then drain those entries when its handlers publish readiness. Keep console-path failure diagnostics deferred to avoid recursive printk locking. Mark a launched instance active before releasing the global kexec lock, and wait for readiness without blocking crash kexec. Validate nested-parent identity dynamically and fail initialization cleanly when manifest restoration leaves no root instance. Fail invalid manifests and missing acknowledgments closed. A spawn started by a host without the pre-launch check validates the boot-context anchor before using shifted fields and parks locally on mismatch. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 8 +- arch/x86/include/asm/multikernel.h | 6 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 5 + arch/x86/kernel/kexec-vmlinux.c | 24 +- arch/x86/kernel/platform-quirks.c | 4 +- arch/x86/multikernel/head_64.S | 3 + arch/x86/multikernel/spawn.c | 132 ++++-- include/linux/multikernel.h | 85 +++- include/linux/multikernel_abi.h | 9 + kernel/kexec_core.c | 73 +++- kernel/multikernel/core.c | 138 +++++-- kernel/multikernel/instance_dt.c | 66 ++- kernel/multikernel/internal.h | 8 + kernel/multikernel/ipi.c | 561 +++++++++++++++++++++----- kernel/multikernel/manifest.c | 14 +- 16 files changed, 918 insertions(+), 219 deletions(-) create mode 100644 include/linux/multikernel_abi.h diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index f440441ac42c31..42dbd751a39f9e 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -104,11 +104,14 @@ struct mk_spawn_context { unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ unsigned long boot_apic_hz; /* Host local APIC timer frequency */ + u64 abi_magic; /* Validated context producer */ } __aligned(PAGE_SIZE); static_assert(offsetof(struct mk_spawn_context, bp) == 144); static_assert(offsetof(struct mk_spawn_context, boot_lps) == 144 + sizeof(struct boot_params)); +static_assert(offsetof(struct mk_spawn_context, abi_magic) == + 144 + sizeof(struct boot_params) + 4 * sizeof(unsigned long)); static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); /* Pool park loop code, copied by the host into per-instance park pages */ @@ -163,7 +166,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); /* Identity page table and trampoline setup */ diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 3a5b3fca1ab352..5fc9c5b5904afd 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a7d31a722f33b7..317764424ff61e 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -60,6 +60,12 @@ struct elf_kernel_info { unsigned long reloc_size; /* Size of relocation data */ }; +struct mk_elf_note_desc { + u64 entry; + u32 ipi_abi_version; + u32 reserved; +}; + /* * Find multikernel entry point from PT_NOTE section. * Looks for note with name "Linux" and type 0x4d4b ('MK'). @@ -93,12 +99,20 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (nhdr->n_type == 0x4d4b && nhdr->n_namesz == 6 && - nhdr->n_descsz == sizeof(u64) && + nhdr->n_descsz == sizeof(struct mk_elf_note_desc) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const struct mk_elf_note_desc *desc; + + desc = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (desc->ipi_abi_version != MK_IPI_ABI_VERSION) { + pr_err("multikernel IPI ABI %u is not supported\n", + desc->ipi_abi_version); + return 0; + } + pr_info("multikernel: entry=0x%llx, IPI ABI=%u\n", + desc->entry, desc->ipi_abi_version); + return desc->entry; } ptr += note_size; } diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index feea109497efde..9209b77d1c54e3 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..b726e74ff3e402 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -273,4 +274,6 @@ SYM_CODE_END(multikernel_secondary_startup) 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map + .long MK_IPI_ABI_VERSION + .long 0 4: .balign 4 diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 095c8343164d73..f1c01260faa383 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -71,6 +71,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * Spawn kernel's own trampoline for secondary CPU wakeup. @@ -84,6 +85,16 @@ static struct mk_spawn_context *mk_boot_context; */ static void *spawn_trampoline_va; static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; @@ -483,6 +494,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->spawn_ctx->boot_tsc_khz = tsc_khz; instance->spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -705,18 +717,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u64 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -733,14 +765,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -776,17 +822,25 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_boot_context; unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -794,34 +848,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - spawn_trampoline_phys = ctx->trampoline_phys; - spawn_trampoline_va = __va(spawn_trampoline_phys); - memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 7dcccc14f963cc..af9ae6fd3f141b 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -14,6 +14,10 @@ #include #include #include +#include +#include + +struct pci_bus; /** * Physical CPU identifiers @@ -75,8 +79,17 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_IPI_SLOT_EMPTY 0 +#define MK_IPI_SLOT_WRITING 1 +#define MK_IPI_SLOT_READY 2 +#define MK_IPI_SLOT_CONSUMING 3 +#define MK_IPI_SLOT_CANCELLED 4 +#define MK_IPI_ABI_MAGIC 0x4d4b495049303033ULL /* "MKIPI003" */ +#define MK_IPI_READY_TIMEOUT_MS 120000 + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + atomic_t state; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -85,9 +98,15 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ + atomic_t head; /* Producer allocation cursor */ + atomic_t tail; /* Consumer scan cursor */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + /* Appended shared ABI: do not move fields above this line. */ + atomic64_t producer_gate; /* Owner CPU and claimed slot */ + atomic_t producer_contention; /* Sends that observed a busy gate */ + atomic_t full_failures; /* Sends rejected by a full ring */ + atomic_t invalid_state; /* Invalid slot state observations */ + atomic_t cancelled_writes; /* Halted producer writes recovered */ }; /* Shared memory structures - per-instance design */ @@ -101,8 +120,47 @@ struct mk_shared_data { * CPUs the first one missed. */ u32 force_halt; + /* Appended ABI handshake; existing shared offsets stay unchanged. */ + u64 abi_magic; + u32 abi_version; + u32 abi_size; + s32 ready_instance_id; + atomic_t ready; }; +static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) +{ + unsigned int i; + + atomic_set(&ring->head, 0); + atomic_set(&ring->tail, 0); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + WRITE_ONCE(ring->entries[i].data_size, 0); + atomic_set(&ring->entries[i].state, MK_IPI_SLOT_EMPTY); + } + atomic_set(&ring->producer_contention, 0); + atomic_set(&ring->full_failures, 0); + atomic_set(&ring->invalid_state, 0); + atomic_set(&ring->cancelled_writes, 0); +} + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + mk_ipi_ring_reset_contents(ring); + atomic64_set(&ring->producer_gate, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->ring); + WRITE_ONCE(shared->force_halt, 0); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); +} + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -145,8 +203,15 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_shared_validate(const struct mk_shared_data *shared); +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id); +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms); +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared, + int instance_id); + +/* Recover a producer only after every CPU in @halted_cpus is parked. */ +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* * Multikernel Messaging System @@ -827,6 +892,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -841,6 +907,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -898,6 +965,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -905,7 +977,8 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 -#define MK_FDT_COMPATIBLE "multikernel-v1" +/* Bumped whenever the shared-memory layout or message semantics change. */ +#define MK_FDT_COMPATIBLE "multikernel-v3" /** * Property Names @@ -1031,6 +1104,8 @@ void mk_set_pool_cpu(int cpu, bool is_pool); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86). Registration diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..9f64fe3c398de1 --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* Private host/spawn transport compatibility constants. */ +#define MK_IPI_ABI_VERSION 3 +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 01382f40aeb724..1ee5ae1ca35e32 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1702,6 +1702,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1756,10 +1762,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,31 +1782,53 @@ int multikernel_kexec_by_id(int mk_id) } /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. + * Start the instance with an empty downlink after its CPUs have been + * confirmed parked. The host is the only producer for this ring, so no + * publisher can race the reset once the old receiver is quiesced. */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + if (instance->ipi_data) { + rc = mk_ipi_shared_reset_downlink(instance->ipi_data, mk_id); + if (rc) { + pr_err("Failed to reset instance %d IPI downlink: %d\n", + mk_id, rc); + goto unlock; + } + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) + goto unlock; /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * The instance is running once its CPUs leave the park loop. Publish that + * state before dropping the global kexec lock so another exec cannot race + * this boot while the readiness handshake is pending. */ - mk_ipi_ring_drop_pending(); + rc = mk_instance_set_kexec_active(mk_image->mk_id); + if (rc) { + int abort_ret = mk_instance_abort_spawn(instance); - rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc == 0) { - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) - pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); + if (abort_ret) + pr_crit("Instance %d activation abort failed: %d\n", + mk_id, abort_ret); + goto unlock; } + /* Do not block crash kexec while a spawn completes its boot handshake. */ + kexec_unlock(); + rc = mk_ipi_shared_wait_ready(instance->ipi_data, mk_id, + MK_IPI_READY_TIMEOUT_MS); + if (rc) { + int abort_ret; + + pr_err("Instance %d did not acknowledge IPI ABI %u: %d\n", + mk_id, MK_IPI_ABI_VERSION, rc); + abort_ret = mk_instance_abort_spawn(instance); + if (abort_ret) + pr_crit("Instance %d IPI ABI timeout abort failed: %d\n", + mk_id, abort_ret); + } + return rc; + unlock: kexec_unlock(); return rc; diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 0d03d008df4262..494da5d69bb022 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -286,6 +286,7 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; @@ -293,8 +294,22 @@ int mk_instance_confirm_parked(struct mk_instance *instance) /* Never started, so nothing of it is running */ if (!instance->spawn_ctx) return 0; + if (!instance->cpus_on_slot) { + pr_err("Instance %d (%s): missing parked-CPU tracking for a started instance\n", + instance->id, instance->name); + return -EINVAL; + } + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -302,6 +317,15 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + if (!failed) { + ret = mk_ipi_ring_recover_halted(snapshot); + if (ret) { + pr_err("Instance %d (%s): failed to recover halted IPI producer: %d\n", + instance->id, instance->name, ret); + failed++; + } + } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1533,38 +1557,16 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt - * - * Forces a spawn kernel's CPUs to stop by arming the force-halt marker - * in the instance's shared IPI area and sending NMIs directly to each - * CPU. The NMI handler tests the marker and parks the CPU in the pool. - * - * No message is queued and no doorbell is rung: a ring message is - * consumed by the instance's ordinary interrupt path, which on a - * responsive kernel races the NMIs for it and can leave them with - * nothing to act on. The marker is host-owned and survives until the - * instance is re-executed, so the NMIs act on it regardless of timing. - * - * Use when: The spawn kernel is stuck/crashed and not responding to graceful - * shutdown, or when graceful shutdown has failed. May be repeated: an - * already-halted instance absorbs the NMIs in the park loop, so a rerun - * only rescues CPUs an earlier halt missed. - * - * Returns: 0 on success, negative error code on failure - */ -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; /* * LOADED is allowed for the retry case: a previous halt already @@ -1573,20 +1575,19 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); + pr_err("Instance %d has no CPUs assigned\n", instance->id); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1598,22 +1599,70 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); + + ret = mk_instance_confirm_parked(instance); + if (ret) { + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + return ret; + } - /* - * The NMI handler parks each CPU on the instance's context. Wait - * for them to arrive before reporting the instance re-spawnable, - * exactly as the graceful path does after its shutdown ACK. - */ mk_instance_settle_halted(instance); - mk_instance_put(instance); return 0; } +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} + static int __init multikernel_init(void) { int ret; + if (!root_instance) { + pr_err("Multikernel root instance is unavailable\n"); + return -ENODEV; + } + /* Register NMI handler for forcible shutdown */ ret = mk_register_stop_nmi_handler(); if (ret < 0) { @@ -1651,6 +1700,17 @@ static int __init multikernel_init(void) return ret; } + ret = mk_ipi_shared_mark_ready(root_instance->ipi_data, + root_instance->id); + if (ret < 0) { + pr_err("Failed to publish multikernel IPI readiness: %d\n", ret); + mk_kernfs_cleanup(); + mk_hotplug_cleanup(); + mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_messaging_cleanup(); + return ret; + } + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 56d74a11523237..874a519d102636 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -34,6 +35,23 @@ struct mk_instance *root_instance = NULL; EXPORT_SYMBOL_GPL(root_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + ret = mk_register_stop_nmi_handler(); + if (ret) + pr_emerg("multikernel: stop-NMI registration failed while rejecting manifest: %d\n", + ret); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /* * Collect every CPU the instance might receive through hotplug later: * the unassigned pool plus every other kernel's CPUs (the host's and @@ -386,6 +404,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char pr_err("Failed to allocate IPI buffer for instance %d\n", instance_id); goto err_free_name; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -543,6 +562,12 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan (unsigned long long)ipi_phys, ipi_pages); return 0; } + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -591,6 +616,12 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest (unsigned long long)host_ipi_phys, host_ipi_pages); return NULL; } + if (host_ipi_size < sizeof(struct mk_shared_data)) { + pr_err("Host IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, host_ipi_size, + sizeof(struct mk_shared_data)); + return NULL; + } host_instance = alloc_mk_instance(0, "", false); if (!host_instance) @@ -645,6 +676,9 @@ int __init mk_instance_restore_from_manifest(void) const void *manifest = NULL; phys_addr_t fdt_phys; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + fdt_phys = mk_manifest_phys(); if (!fdt_phys) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -682,15 +716,15 @@ int __init mk_instance_restore_from_manifest(void) int mk_node = fdt_subnode_offset(manifest, 0, "multikernel"); if (mk_node < 0) { - pr_info("No multikernel node found in manifest\n"); - ret = 0; + pr_err("No multikernel node found in supplied manifest\n"); + ret = -EINVAL; goto cleanup_fdt; } const void *dtb_data = fdt_getprop(manifest, mk_node, "dtb-data", &dtb_len); if (!dtb_data || dtb_len <= 0) { - pr_info("No dtb-data property found in multikernel node\n"); - ret = 0; + pr_err("No dtb-data property found in multikernel node\n"); + ret = -EINVAL; goto cleanup_fdt; } @@ -787,8 +821,26 @@ int __init mk_instance_restore_from_manifest(void) host_instance = mk_restore_host_instance(manifest); if (!host_instance) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); - + mk_manifest_reject_and_park(-ENODEV); + + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_ipi_shared_validate(host_instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + if (!atomic_read_acquire(&host_instance->ipi_data->ready) || + READ_ONCE(host_instance->ipi_data->ready_instance_id) != + host_instance->id) + mk_manifest_reject_and_park(-EHOSTDOWN); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); + ret = mk_register_stop_nmi_handler(); + if (ret) + mk_manifest_reject_and_park(ret); pr_info("Successfully restored multikernel root instance %d ('%s') from manifest (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); @@ -822,6 +874,8 @@ int __init mk_instance_restore_from_manifest(void) kfree(dtb_virt); cleanup_fdt: early_memunmap((void *)manifest, PAGE_SIZE); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 2fe809f97c2281..ec086a4bebd4c9 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -13,6 +13,14 @@ extern struct idr mk_instance_idr; extern struct list_head mk_instance_list; extern struct mk_instance *root_instance; +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 749500ff1f6a2f..6afeab019750c9 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -12,49 +12,387 @@ #include #include #include +#include +#include #include "internal.h" /* Callback management */ static struct mk_ipi_handler *mk_handlers; static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RATELIMIT_STATE(mk_ipi_publish_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); static void mk_ipi_drain_ring(void); +#define MK_IPI_PRODUCER_RETRIES 10000 +#define MK_IPI_GATE_INDEX_BITS 6 +#define MK_IPI_GATE_INDEX_MASK (MK_IPI_RING_SIZE - 1) + /* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. + * A nonzero gate records both the physical producer CPU and the slot at head. + * This makes the serialization recoverable after that exact CPU is confirmed + * parked. A boolean shared lock would be unsafe because the force-stop NMI may + * prevent its owner from ever returning to release it. */ -static inline unsigned int mk_ring_idx(unsigned int i) +static u64 mk_ipi_gate_token(mk_phys_cpu_t owner, unsigned int idx) +{ + BUILD_BUG_ON(BIT(MK_IPI_GATE_INDEX_BITS) != MK_IPI_RING_SIZE); + if (owner >= (U64_MAX >> MK_IPI_GATE_INDEX_BITS)) + return 0; + + return ((owner + 1) << MK_IPI_GATE_INDEX_BITS) | idx; +} + +static mk_phys_cpu_t mk_ipi_gate_owner(u64 token) +{ + return (token >> MK_IPI_GATE_INDEX_BITS) - 1; +} + +static unsigned int mk_ipi_gate_index(u64 token) +{ + return token & MK_IPI_GATE_INDEX_MASK; +} + +/* + * Serialize producers with preemption disabled so the physical owner encoded + * in the gate remains stable. Keep local IRQs enabled while waiting for a + * producer in another kernel, then disable them only for the short publish + * critical section. No NMI path sends general messages; force halt uses a + * persistent host-owned marker. + * Advancing head before READY lets recovery distinguish both interruption + * windows without allowing another producer to pass the gate. + */ +static int mk_ipi_ring_publish(struct mk_shared_data *shared, int instance_id, + const void *data, size_t data_size, + unsigned long type) +{ + struct mk_ipi_ring *ring = &shared->ring; + struct mk_ipi_data *slot; + mk_phys_cpu_t owner; + unsigned long flags; + bool contended = false; + unsigned int retry; + unsigned int idx; + u64 token, old; + int state; + int head; + int ret; + + preempt_disable(); + owner = arch_cpu_physical_id(smp_processor_id()); + + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_enable; + } + + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) { + if ((atomic_read(&ring->head) & + MK_IPI_GATE_INDEX_MASK) != idx) { + atomic64_set_release(&ring->producer_gate, 0); + contended = true; + cpu_relax(); + continue; + } + break; + } + contended = true; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_count_contention; + } + cpu_relax(); + } + + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_count_contention; + } + + local_irq_save(flags); + if (contended) + atomic_inc(&ring->producer_contention); + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance_id) { + ret = -ESHUTDOWN; + goto out_release_gate; + } + + slot = &ring->entries[idx]; + state = atomic_cmpxchg(&slot->state, MK_IPI_SLOT_EMPTY, + MK_IPI_SLOT_WRITING); + if (state != MK_IPI_SLOT_EMPTY) { + if (state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CONSUMING || + state == MK_IPI_SLOT_CANCELLED) { + atomic_inc(&ring->full_failures); + ret = -ENOSPC; + } else { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + goto out_release_gate; + } + + WRITE_ONCE(slot->data_size, 0); + WRITE_ONCE(slot->sender_cpu, owner); + WRITE_ONCE(slot->type, type); + if (data_size) + memcpy(slot->buffer, data, data_size); + WRITE_ONCE(slot->data_size, data_size); + atomic_set(&ring->head, (idx + 1) & MK_IPI_GATE_INDEX_MASK); + atomic_set_release(&slot->state, MK_IPI_SLOT_READY); + ret = 0; + +out_release_gate: + atomic64_set_release(&ring->producer_gate, 0); + local_irq_restore(flags); +out_enable: + preempt_enable(); + return ret; + +out_count_contention: + atomic_inc(&ring->producer_contention); + goto out_enable; +} + +static bool mk_ipi_slot_is_pending(int state) { - return i & (MK_IPI_RING_SIZE - 1); + return state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CANCELLED; +} + +static void mk_ipi_slot_release(struct mk_ipi_data *slot) +{ + WRITE_ONCE(slot->data_size, 0); + atomic_set_release(&slot->state, MK_IPI_SLOT_EMPTY); +} + +int mk_ipi_shared_validate(const struct mk_shared_data *shared) +{ + if (!shared) + return -ENODEV; + if (READ_ONCE(shared->abi_magic) != MK_IPI_ABI_MAGIC || + READ_ONCE(shared->abi_version) != MK_IPI_ABI_VERSION || + READ_ONCE(shared->abi_size) != sizeof(*shared)) + return -EPROTO; + + return 0; +} + +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id) +{ + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + WRITE_ONCE(shared->ready_instance_id, instance_id); + atomic_set_release(&shared->ready, 1); + /* Drain messages queued after the host prepared the ring for this boot. */ + mk_ipi_drain_ring(); + return 0; +} + +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms) +{ + unsigned long deadline; + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(timeout_ms); + do { + if (atomic_read_acquire(&shared->ready)) + return READ_ONCE(shared->ready_instance_id) == instance_id ? + 0 : -EPROTO; + msleep(20); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared, + int instance_id) +{ + struct mk_ipi_ring *ring; + mk_phys_cpu_t owner; + unsigned long flags; + unsigned int retry; + unsigned int idx; + u64 token, old; + int head; + int ret = 0; + + if (!shared) + return -EINVAL; + + /* Exclude new publishers before waiting for an in-flight one. */ + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set_release(&shared->ready, 0); + /* Pair identity invalidation with the publisher's identity check. */ + smp_mb(); + ring = &shared->ring; + preempt_disable(); + owner = arch_cpu_physical_id(smp_processor_id()); + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_enable; + } + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) + break; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_enable; + } + cpu_relax(); + } + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_enable; + } + + local_irq_save(flags); + /* The old receiver is parked and every pre-existing publisher drained. */ + mk_ipi_ring_reset_contents(ring); + WRITE_ONCE(shared->force_halt, 0); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + /* + * Publish the target identity before launch so callers can queue while + * the receiver boots. Readiness remains clear until its handlers exist. + */ + WRITE_ONCE(shared->ready_instance_id, instance_id); + atomic_set(&shared->ready, 0); + atomic64_set_release(&ring->producer_gate, 0); + + local_irq_restore(flags); +out_enable: + preempt_enable(); + return ret; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_ring_recover_halted - Recover a ring producer after it is parked + * @halted_cpus: Exact set of CPUs confirmed parked by the caller * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * A force-stop NMI can park a producer while it owns the shared gate. Only its + * receiver may recover the write, and only after the owner CPU is proven not + * to be executing it. Other instances may still publish into the host ring, + * so this function never resets the ring or touches another owner's gate. */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus) { + struct mk_ipi_data *slot; struct mk_ipi_ring *ring; - unsigned int head, tail; - - if (!root_instance || !root_instance->ipi_data) - return; + mk_phys_cpu_t owner; + mk_phys_cpu_t target; + unsigned int idx; + unsigned int next; + u64 token; + int state; + int head; + int ret = 0; + + if (!halted_cpus || !root_instance || !root_instance->ipi_data) + return -EINVAL; ring = &root_instance->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + token = atomic64_read_acquire(&ring->producer_gate); + if (!token) + goto kick; + if (!(token >> MK_IPI_GATE_INDEX_BITS)) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + goto kick; + } + + owner = mk_ipi_gate_owner(token); + if (!mk_cpu_set_contains(halted_cpus, owner)) + goto kick; + + idx = mk_ipi_gate_index(token); + next = (idx + 1) & MK_IPI_GATE_INDEX_MASK; + slot = &ring->entries[idx]; + state = atomic_read_acquire(&slot->state); + head = atomic_read(&ring->head) & MK_IPI_GATE_INDEX_MASK; + switch (state) { + case MK_IPI_SLOT_EMPTY: + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_WRITING: + if (head == idx) { + atomic_set(&ring->head, next); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + atomic_set_release(&slot->state, MK_IPI_SLOT_CANCELLED); + atomic_inc(&ring->cancelled_writes); + break; + case MK_IPI_SLOT_READY: + case MK_IPI_SLOT_CANCELLED: + if (head == idx) { + /* Repair an interrupted publication before releasing its gate. */ + atomic_set(&ring->head, next); + atomic_inc(&ring->invalid_state); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_CONSUMING: + /* + * The consumer may claim a full-ring tail while this producer + * waits to test it, leaving head at idx with nothing published. + * Head at next means the previous publication was claimed before + * its now-stale gate could be released. Both cursors are valid. + */ + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + default: + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + /* Keep the gate closed when cursor repair cannot make the FIFO safe. */ + if (ret) + goto kick; + if (atomic64_cmpxchg_release(&ring->producer_gate, token, 0) != token) { + atomic_inc(&ring->invalid_state); + ret = -EAGAIN; + } + +kick: + /* The producer may have published and parked before ringing the bell. */ + target = mk_cpu_set_first(root_instance->cpus); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); - atomic_set(&ring->tail, head); + return ret; } /** @@ -166,7 +504,7 @@ int mk_arm_force_halt(struct mk_instance *instance) /** * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID + * @instance: Target multikernel instance * @data: Pointer to data to send * @data_size: Size of data * @type: User-defined type identifier @@ -176,121 +514,124 @@ int mk_arm_force_halt(struct mk_instance *instance) * * Returns 0 on success, negative error code on failure */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { - struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; mk_phys_cpu_t target; + int instance_id; + int ret; if (!instance) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + instance_id = instance->id; + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - } target = mk_cpu_set_first(instance->cpus); if (target == MK_PHYS_CPU_INVALID) { pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - mk_instance_put(instance); return -ENODEV; } if (!mk_instance_ipi_area(instance)) { pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); return -ENODEV; } - - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + return ret; + /* A prepared downlink may accept messages before the receiver is ready. */ + if (READ_ONCE(instance->ipi_data->ready_instance_id) != instance_id) + return -EPROTO; + + ret = mk_ipi_ring_publish(instance->ipi_data, instance_id, data, + data_size, type); + if (ret) { + /* + * A doorbell can be coalesced while the target is draining this + * ring. Kick it again before reporting backpressure so READY + * entries cannot remain stranded without another notification. + */ + mk_arch_send_ipi(target); + /* Console writers reach this path with their lock and IRQs held. */ + if (ret != -EDEADLK && __ratelimit(&mk_ipi_publish_rs)) { + if (ret == -ENOSPC) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance_id); + else if (ret == -EAGAIN) + printk_deferred(KERN_WARNING + "multikernel: IPI producer busy for instance %d\n", + instance_id); + else + printk_deferred(KERN_ERR + "multikernel: IPI publish failed for instance %d: %d\n", + instance_id, ret); } + return ret; + } + mk_arch_send_ipi(target); - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); - - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + return 0; +} - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance; + int ret; + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } static void mk_ipi_drain_ring(void) { struct mk_ipi_data *slot; struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; + struct mk_ipi_ring *ring; + unsigned int tail, idx; size_t data_size; + int state; int messages_processed = 0; if (!root_instance || !root_instance->ipi_data) return; - while (1) { - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - if (tail == head) + ring = &root_instance->ipi_data->ring; + while (messages_processed < MK_IPI_RING_SIZE) { + tail = atomic_read(&ring->tail); + idx = tail & (MK_IPI_RING_SIZE - 1); + slot = &ring->entries[idx]; + + state = atomic_read_acquire(&slot->state); + if (!mk_ipi_slot_is_pending(state)) { + if (state != MK_IPI_SLOT_EMPTY && + state != MK_IPI_SLOT_WRITING && + state != MK_IPI_SLOT_CONSUMING) { + atomic_inc(&ring->invalid_state); + pr_warn_once("Multikernel IPI slot %u has bad state %d\n", + idx, state); + } break; + } - slot = &root_instance->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (atomic_cmpxchg_acquire(&slot->state, state, + MK_IPI_SLOT_CONSUMING) != state) break; + if (state == MK_IPI_SLOT_CANCELLED) + goto advance_tail; + + data_size = READ_ONCE(slot->data_size); if (data_size > MK_MAX_DATA_SIZE) { pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - continue; + idx, data_size); + goto advance_tail; } /* Dispatch to registered handler */ @@ -308,17 +649,29 @@ static void mk_ipi_drain_ring(void) raw_spin_unlock(&mk_handlers_lock); advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); + mk_ipi_slot_release(slot); + atomic_set(&ring->tail, (idx + 1) & (MK_IPI_RING_SIZE - 1)); messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; } } +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + mk_phys_cpu_t target; + + if (!root_instance) + return; + target = mk_cpu_set_first(root_instance->cpus); + if (target == MK_PHYS_CPU_INVALID || + target != arch_cpu_physical_id(smp_processor_id())) + return; + + local_irq_save(flags); + mk_ipi_drain_ring(); + local_irq_restore(flags); +} + /** * multikernel_interrupt_handler - Handle the multikernel IPI * diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 89a8ee0d2c59b3..19cdc625e9e04d 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -23,12 +23,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -50,6 +56,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -68,14 +75,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /**