diff --git a/drivers/hv/mshv_vtl.h b/drivers/hv/mshv_vtl.h index 82fa731fc779..73ccac20f569 100644 --- a/drivers/hv/mshv_vtl.h +++ b/drivers/hv/mshv_vtl.h @@ -118,6 +118,20 @@ struct tdx_vp_context { static_assert(offsetof(struct tdx_vp_context, l2_enter_guest_state) + 272 == 512); static_assert(sizeof(struct tdx_vp_context) == 1024); +struct snp_vp_context { + __u8 vmsa_tweak_bitmap[64]; + __u64 stimer0_config; + __u64 stimer0_count; + __u64 stimer0_programmed_ref_time; + __u32 stimer0_flags; + __u8 pad[932]; +}; + +static_assert(sizeof(struct snp_vp_context) == 1024); + +#define MSHV_VTL_SNP_STIMER0_CONFIG_VALID BIT(0) +#define MSHV_VTL_SNP_STIMER0_KERNEL_UPDATE BIT(1) +#define MSHV_VTL_SNP_STIMER0_EXPIRED BIT(2) #endif struct mshv_vtl_run { @@ -134,6 +148,7 @@ struct mshv_vtl_run { #ifdef CONFIG_X86_64 struct tdx_vp_context tdx_context; + struct snp_vp_context snp_context; #endif /* * Reserving room for the cpu context to grow and to maintain compatibility diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c index a19b2e5c3a1e..4e559bce4ed6 100644 --- a/drivers/hv/mshv_vtl_main.c +++ b/drivers/hv/mshv_vtl_main.c @@ -196,6 +196,7 @@ struct mshv_vtl_per_cpu { #endif #if defined(CONFIG_X86_64) && defined(CONFIG_SEV_GUEST) struct page *snp_secure_avic_page; + struct hrtimer snp_stimer0_timer; #endif }; @@ -216,6 +217,7 @@ static union hv_register_vsm_capabilities mshv_vsm_capabilities; static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file); static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions); static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu); +static DEFINE_PER_CPU(struct task_struct *, mshv_vtl_thread); static const union hv_input_vtl input_vtl_zero; static const union hv_input_vtl input_vtl_normal = { @@ -561,28 +563,28 @@ static int mshv_update_proxy_irr_for_icr_write(u32 icr_lo, struct cpumask *local const u64 bank = vector / 32; const u32 mask = BIT(vector % 32); const u32 self = smp_processor_id(); - unsigned int cpu; - bool send_ipi; - send_ipi = false; for_each_cpu(cpu, local_mask) { + struct mshv_vtl_run *run = mshv_vtl_cpu_run(cpu); + /* * The kernel doesn't provide an atomic_or which operates on u32, * so cast to atomic_t, which should have the same layout */ static_assert(sizeof(atomic_t) == sizeof(u32)); - atomic_or(mask, (atomic_t *) - (&(mshv_vtl_cpu_run(cpu)->proxy_irr[bank]))); - /* Make update visible to other CPUs */ - smp_store_release(&mshv_vtl_cpu_run(cpu)->scan_proxy_irr, 1); - send_ipi |= cpu != self; + atomic_or(mask, (atomic_t *)&run->proxy_irr[bank]); + + /* + * The first producer with pending work wakes the target. Later + * producers coalesce behind the scan that is already pending. + */ + if (xchg(&run->scan_proxy_irr, 1) || cpu == self) + cpumask_clear_cpu(cpu, local_mask); } - if (send_ipi) { - cpumask_clear_cpu(self, local_mask); + if (!cpumask_empty(local_mask)) __apic_send_IPI_mask(local_mask, RESCHEDULE_VECTOR); - } return 0; } @@ -1045,6 +1047,30 @@ static void mshv_vtl_set_tsc_deadline(u64 vm_idx, u64 deadline) #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_SEV_GUEST) +static enum hrtimer_restart mshv_snp_stimer0_timer_fn(struct hrtimer *timer) +{ + struct mshv_vtl_per_cpu *per_cpu = + container_of(timer, struct mshv_vtl_per_cpu, snp_stimer0_timer); + struct mshv_vtl_run *run = READ_ONCE(per_cpu->run); + + if (run) { + struct task_struct *thread; + + /* Wake userspace to handle timer delivery. */ + atomic_or(MSHV_VTL_SNP_STIMER0_EXPIRED, + (atomic_t *)&run->snp_context.stimer0_flags); + WRITE_ONCE(run->cancel, 1); + + thread = this_cpu_read(mshv_vtl_thread); + if (thread) + wake_up_process(thread); + } + + return HRTIMER_NORESTART; +} +#endif + static int mshv_vtl_alloc_context(unsigned int cpu) { struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu); @@ -1091,6 +1117,10 @@ static int mshv_vtl_alloc_context(unsigned int cpu) struct page *snp_secure_avic_page; int ret; + hrtimer_setup(&per_cpu->snp_stimer0_timer, + mshv_snp_stimer0_timer_fn, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); + ret = mshv_snp_configure_vmsa_page(0, &per_cpu->vmsa_page); if (ret < 0) return ret; @@ -1732,7 +1762,6 @@ static void mshv_tdx_halt_timer_post(enum TDX_HALT_TIMER armed) {} #endif static bool in_idle_is_enabled; -DEFINE_PER_CPU(struct task_struct *, mshv_vtl_thread); static void mshv_vtl_switch_to_vtl0_irqoff(void) { @@ -2064,6 +2093,88 @@ static struct sev_es_save_area *snp_this_vmsa(void) return page_address(vmsa_page); } +static u64 mshv_snp_read_vmsa_reg(struct mshv_vtl_run *run, + struct sev_es_save_area *vmsa, size_t offset) +{ + u8 byte_index = offset / 64; + u8 bit_index = (offset % 64) / 8; + u64 value = READ_ONCE(*(u64 *)((u8 *)vmsa + offset)); + + if (READ_ONCE(run->snp_context.vmsa_tweak_bitmap[byte_index]) & BIT(bit_index)) { + static_assert(offsetof(struct sev_es_save_area, reserved_0x300) == 0x300); + value ^= READ_ONCE(*(u64 *)vmsa->reserved_0x300); + } + + return value; +} + +static void mshv_snp_write_vmsa_reg(struct mshv_vtl_run *run, + struct sev_es_save_area *vmsa, + size_t offset, u64 value) +{ + u8 byte_index = offset / 64; + u8 bit_index = (offset % 64) / 8; + + if (READ_ONCE(run->snp_context.vmsa_tweak_bitmap[byte_index]) & BIT(bit_index)) + value ^= READ_ONCE(*(u64 *)vmsa->reserved_0x300); + + WRITE_ONCE(*(u64 *)((u8 *)vmsa + offset), value); +} + +#define MSHV_STIMER_ENABLE BIT_ULL(0) +#define MSHV_STIMER_PERIODIC BIT_ULL(1) +#define MSHV_STIMER_AUTO_ENABLE BIT_ULL(3) + +static bool mshv_snp_try_handle_stimer0_count(struct mshv_vtl_run *run, + struct sev_es_save_area *vmsa, + u64 count) +{ + struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu); + atomic_t *flags = (atomic_t *)&run->snp_context.stimer0_flags; + union hv_input_vtl target_vtl = READ_ONCE(run->target_vtl); + u32 old_flags = atomic_read(flags); + u64 config; + u64 now; + u64 delta; + u64 ns; + + if ((target_vtl.use_target_vtl && target_vtl.target_vtl > 0) || + !(old_flags & MSHV_VTL_SNP_STIMER0_CONFIG_VALID)) + return false; + + if (old_flags & MSHV_VTL_SNP_STIMER0_KERNEL_UPDATE) { + hrtimer_cancel(&per_cpu->snp_stimer0_timer); + old_flags = atomic_read(flags); + if (old_flags & MSHV_VTL_SNP_STIMER0_EXPIRED) + return false; + } + + config = READ_ONCE(run->snp_context.stimer0_config); + now = hv_read_reference_counter(); + + WRITE_ONCE(run->snp_context.stimer0_count, count); + WRITE_ONCE(run->snp_context.stimer0_programmed_ref_time, now); + atomic_set(flags, (old_flags | MSHV_VTL_SNP_STIMER0_KERNEL_UPDATE) & + ~MSHV_VTL_SNP_STIMER0_EXPIRED); + + if (count && ((config & MSHV_STIMER_ENABLE) || + (config & MSHV_STIMER_AUTO_ENABLE))) { + if (config & MSHV_STIMER_PERIODIC) + delta = count; + else + delta = (s64)(count - now) > 0 ? count - now : 0; + ns = delta > KTIME_MAX / 100 ? KTIME_MAX : delta * 100; + hrtimer_start(&per_cpu->snp_stimer0_timer, ns_to_ktime(ns), + HRTIMER_MODE_REL_PINNED); + } + + mshv_snp_write_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, rip), + mshv_snp_read_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, guest_nrip))); + return true; +} + /* * Sets a benign guest error code so that there won't be another * #VMEXIT for the just processed one and marks the VMSA as @@ -2079,6 +2190,25 @@ static void mshv_snp_clear_exit_code(struct sev_es_save_area *vmsa, bool int_sha vmsa->vintr_ctrl &= ~V_GUEST_BUSY_MASK; } +/* + * Determine if this interrupt was handled completely in the kernel. + * + * Returns true if the exit was handled entirely in kernel, and the VMPL should be re-entered. + * Returns false if the exit must be handled by user-space. + */ +static bool mshv_snp_try_handle_interrupt_entry(struct mshv_vtl_run *run) +{ + struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; + + if (!(run->offload_flags & MSHV_VTL_OFFLOAD_FLAG_INTR_INJECT) || + READ_ONCE(hvp->vtl_entry_reason) != MSHV_ENTRY_REASON_INTERRUPT) + return false; + + mshv_snp_clear_exit_code(snp_this_vmsa(), false); + return true; +} + + /* * Try to handle the incomplete IPI SEV-SNP exit. * @@ -2147,11 +2277,36 @@ static bool mshv_snp_try_handle_exit(struct mshv_vtl_run *run) return true; break; case SVM_EXIT_MSR: - if (vmsa->rcx == HV_X64_MSR_GUEST_IDLE && !(vmsa->guest_exit_info_1 & 1)) { + if (READ_ONCE(hv_vp_assist_page[smp_processor_id()]->vtl_entry_reason) == + MSHV_ENTRY_REASON_INTERCEPT && + (u32)mshv_snp_read_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, rcx)) == + HV_X64_MSR_STIMER0_COUNT && + (vmsa->guest_exit_info_1 & 1)) { + u64 count = (u32)mshv_snp_read_vmsa_reg( + run, vmsa, + offsetof(struct sev_es_save_area, rax)) | + ((u64)(u32)mshv_snp_read_vmsa_reg( + run, vmsa, + offsetof(struct sev_es_save_area, rdx)) + << 32); + + if (mshv_snp_try_handle_stimer0_count(run, vmsa, count)) + goto handled; + } else if ((u32)mshv_snp_read_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, rcx)) == + HV_X64_MSR_GUEST_IDLE && + !(vmsa->guest_exit_info_1 & 1)) { + u64 next_rip = mshv_snp_read_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, guest_nrip)); + /* The guest indicates it's idle by reading this synthetic MSR. */ - vmsa->rax = 0; - vmsa->rdx = 0; - vmsa->rip = vmsa->guest_nrip; + mshv_snp_write_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, rax), 0); + mshv_snp_write_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, rdx), 0); + mshv_snp_write_vmsa_reg( + run, vmsa, offsetof(struct sev_es_save_area, rip), next_rip); run->offload_flags &= ~MSHV_VTL_OFFLOAD_FLAG_HALT_HLT; run->offload_flags |= MSHV_VTL_OFFLOAD_FLAG_HALT_IDLE; @@ -2174,6 +2329,15 @@ static bool mshv_snp_try_handle_exit(struct mshv_vtl_run *run) return true; } +static void mshv_snp_release_stimer0(struct mshv_vtl_run *run) +{ + if (!(atomic_read((atomic_t *)&run->snp_context.stimer0_flags) & + MSHV_VTL_SNP_STIMER0_KERNEL_UPDATE)) + return; + + hrtimer_cancel(&this_cpu_ptr(&mshv_vtl_per_cpu)->snp_stimer0_timer); +} + static bool mshv_snp_try_handle_intercept(struct mshv_vtl_run *run) { struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; @@ -2223,6 +2387,7 @@ static bool mshv_snp_try_handle_intercept(struct mshv_vtl_run *run) return false; case HVMSG_X64_HALT: run->flags |= MSHV_VTL_RUN_FLAG_HALTED; + run->offload_flags &= ~MSHV_VTL_OFFLOAD_FLAG_HALT_IDLE; run->offload_flags |= MSHV_VTL_OFFLOAD_FLAG_HALT_HLT; break; default: @@ -2334,8 +2499,11 @@ static int mshv_vtl_ioctl_return_to_lower_vtl(void) #endif } else if (hv_isolation_type_snp()) { #ifdef CONFIG_SEV_GUEST - if (mshv_snp_try_handle_intercept(mshv_vtl_this_run()) && - mshv_snp_try_handle_exit(mshv_vtl_this_run())) + struct mshv_vtl_run *run = mshv_vtl_this_run(); + + if (mshv_snp_try_handle_intercept(run) && + (mshv_snp_try_handle_interrupt_entry(run) || + mshv_snp_try_handle_exit(run))) continue; /* Exit handled entirely in kernel */ #endif } @@ -2367,6 +2535,11 @@ static int mshv_vtl_ioctl_return_to_lower_vtl(void) } done: +#ifdef CONFIG_SEV_GUEST + if (hv_isolation_type_snp()) + mshv_snp_release_stimer0(mshv_vtl_this_run()); +#endif + preempt_enable(); return 0;