From 94f2c49ca73d2f7dabf101b6debeebedcc4dbce2 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 25 Jun 2026 16:33:43 +0800 Subject: [PATCH 01/26] HAOC/IEE: Upgrade: prepare for haoc-4.0 Update the IEE base code with bitmap infrastructure, staged initialization, and arm64 gate setup. - Add x86 and arm64 HAOC bitmap helpers, and wire the bitmap-type update handler into the IEE gate table. - Add typed page-marking helpers for IEE-owned pages alongside the existing order-based set_iee_page() interface. - Split x86 IEE setup into early, init, and post-init phases. - Add x86 gate dispatch that uses direct handler calls before HAOC has enabled the protected gate path. - Add the arm64 direct-call gate path, post-init gate rewrite, bitmap setup, and early IEE data bitmap tagging. - Add the arm64 IEE debug and split-linear-mapping Kconfig hooks. Signed-off-by: Liu Zhehui --- arch/arm64/include/asm/haoc/haoc-bitmap.h | 90 +++++++++ arch/arm64/include/asm/haoc/haoc-def.h | 2 + arch/arm64/include/asm/haoc/haoc.h | 6 + arch/arm64/include/asm/haoc/iee-cred.h | 6 + arch/arm64/include/asm/haoc/iee-func.h | 4 + arch/arm64/kernel/haoc/Kconfig | 20 ++ arch/arm64/kernel/haoc/Makefile | 4 +- arch/arm64/kernel/haoc/credp/credp.c | 35 ++++ arch/arm64/kernel/haoc/haoc-bitmap.c | 226 ++++++++++++++++++++++ arch/arm64/kernel/haoc/haoc.c | 2 + arch/arm64/kernel/haoc/iee/iee-func.c | 8 + arch/arm64/kernel/haoc/iee/iee-gate.S | 20 ++ arch/arm64/kernel/haoc/iee/iee-init.c | 26 ++- arch/arm64/kernel/haoc/iee/iee-mmu.c | 45 +++-- arch/arm64/kernel/haoc/iee/iee-si-gate.S | 14 ++ arch/x86/include/asm/haoc/haoc-bitmap.h | 59 ++++++ arch/x86/include/asm/haoc/haoc-def.h | 2 + arch/x86/include/asm/haoc/haoc.h | 6 + arch/x86/include/asm/haoc/iee-cred.h | 6 + arch/x86/include/asm/haoc/iee-func.h | 4 + arch/x86/include/asm/haoc/iee.h | 5 +- arch/x86/kernel/cpu/common.c | 4 - arch/x86/kernel/haoc/Makefile | 4 +- arch/x86/kernel/haoc/bitmap.c | 191 ++++++++++++++++++ arch/x86/kernel/haoc/credp/credp.c | 32 +++ arch/x86/kernel/haoc/haoc.c | 2 + arch/x86/kernel/haoc/iee/iee-func.c | 21 +- arch/x86/kernel/haoc/iee/iee-gate.S | 95 ++++----- arch/x86/kernel/haoc/iee/iee-init.c | 158 +++++++++++---- arch/x86/mm/init_64.c | 20 +- init/main.c | 7 + security/keys/process_keys.c | 83 ++++---- 32 files changed, 1022 insertions(+), 185 deletions(-) create mode 100644 arch/arm64/include/asm/haoc/haoc-bitmap.h create mode 100644 arch/arm64/kernel/haoc/haoc-bitmap.c create mode 100644 arch/x86/include/asm/haoc/haoc-bitmap.h create mode 100644 arch/x86/kernel/haoc/bitmap.c diff --git a/arch/arm64/include/asm/haoc/haoc-bitmap.h b/arch/arm64/include/asm/haoc/haoc-bitmap.h new file mode 100644 index 0000000000000..3e2ad505877fc --- /dev/null +++ b/arch/arm64/include/asm/haoc/haoc-bitmap.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HAOC_BITMAP_H +#define _LINUX_HAOC_BITMAP_H + +#include +#include + +/* HAOC_NORMAL means */ +enum HAOC_BITMAP_TYPE { + IEE_NORMAL = 0, /* Non-IEE pages */ + IEE_DATA, + IEE_TOKEN, + IEE_PGTABLE, + IEE_USER_PGTABLE, /* for PTP_S */ + IEE_KEY, + IEE_SELINUX, + IEE_VARP, +}; + +#include + +#define HAOC_BITMAP_START VMEMMAP_END +#define haoc_bitmap_base ((uint8_t *)HAOC_BITMAP_START - \ + (memstart_addr >> PAGE_SHIFT)) + +#define __pfn_to_haoc_bitmap(pfn) (haoc_bitmap_base + (pfn)) + +#define __va_to_haoc_bitmap(va) ({ \ + uint8_t *__ret; \ + if (__is_lm_address((u64)va)) \ + __ret = __pfn_to_haoc_bitmap(PHYS_PFN(__pa(va))); \ + else \ + __ret = __pfn_to_haoc_bitmap(PHYS_PFN(__pa_symbol(va))); \ + __ret; \ +}) + +extern bool haoc_enabled; +extern bool haoc_bitmap_ready; + +extern int haoc_bitmap_sparse_init(void); +extern void haoc_bitmap_setup(void); +extern void setup_iee_early_data_bitmap(void); + +static inline enum HAOC_BITMAP_TYPE iee_get_bitmap_type(unsigned long va) +{ + return *__va_to_haoc_bitmap(va); +} + +void _iee_set_bitmap_type(unsigned long __unused, + u64 va, enum HAOC_BITMAP_TYPE type, int num_pages); + +static inline void iee_set_bitmap_type(unsigned long va, + int num_pages, enum HAOC_BITMAP_TYPE type) +{ + iee_rw_gate(IEE_OP_SET_BITMAP_TYPE, va, type, num_pages); +} + +static inline void iee_verify_type(unsigned long va, enum HAOC_BITMAP_TYPE type, + const char *name) +{ + if (haoc_enabled) { + uint8_t bit_type = iee_get_bitmap_type(va); + + if (unlikely(bit_type != type)) + panic("IEE detected type: %d, fake %s: va(0x%lx)", bit_type, name, va); + } +} + +static inline unsigned long iee_test_not_normal(void *start, void *end) +{ + unsigned long addr; + + if (!haoc_enabled) + return 0; + addr = (unsigned long)start; + while (addr < (unsigned long)end) { + if (unlikely(iee_get_bitmap_type(addr) == IEE_NORMAL)) + return addr; + addr += PAGE_SIZE; + } + return 0; +} + +#define iee_verify_not_normal(start, end) ({ \ + unsigned long __addr = iee_test_not_normal(start, end); \ + if (__addr) \ + panic("HAOC: (%s)operate on IEE bitmap type normal 0x%lx.", __func__, __addr); \ +}) + +#endif diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index 61b201914d5ed..6f26d88fe27e3 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -21,12 +21,14 @@ enum { IEE_OP_INVALIDATE_TOKEN, IEE_OP_VALIDATE_TOKEN, #endif + IEE_OP_SET_BITMAP_TYPE, #ifdef CONFIG_CREDP IEE_OP_COPY_CRED, IEE_OP_COPY_KERNEL_CRED, IEE_OP_INIT_COPIED_CRED, IEE_OP_COMMIT_CRED, IEE_OP_ABORT_CRED, + IEE_OP_FILL_SESSION_KEYRING_CRED, IEE_OP_SET_CRED_UID, IEE_OP_SET_CRED_GID, IEE_OP_SET_CRED_SUID, diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index a8d5fcabbf95d..c06f1d6b094ba 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -10,6 +10,7 @@ #ifndef _LINUX_HAOC_H #define _LINUX_HAOC_H +#include #include #include @@ -26,6 +27,8 @@ void _iee_init_token(unsigned long __unused, struct task_struct *tsk); void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); #endif +void _iee_set_bitmap_type(unsigned long __unused, u64 va, + enum HAOC_BITMAP_TYPE type, int num_pages); #ifdef CONFIG_CREDP #include @@ -33,6 +36,9 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); void _iee_copy_cred(unsigned long __unused, struct cred *new); void _iee_copy_kernel_cred(unsigned long iee_offset, const struct cred *old, struct cred *new); void _iee_abort_cred(unsigned long iee_offset, const struct cred *cred); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, + struct cred *new, + const struct cred *old); void _iee_init_copied_cred(unsigned long iee_offset, struct task_struct *new_task, struct cred *new); void _iee_commit_creds(unsigned long iee_offset, const struct cred *new); diff --git a/arch/arm64/include/asm/haoc/iee-cred.h b/arch/arm64/include/asm/haoc/iee-cred.h index e5297dabafc37..ca40e034ead5e 100644 --- a/arch/arm64/include/asm/haoc/iee-cred.h +++ b/arch/arm64/include/asm/haoc/iee-cred.h @@ -36,6 +36,12 @@ static void __maybe_unused iee_abort_creds(struct cred *cred) iee_rw_gate(IEE_OP_ABORT_CRED, cred); } +static void __maybe_unused iee_fill_cred_for_session_keyring(struct cred *new, + const struct cred *old) +{ + iee_rw_gate(IEE_OP_FILL_SESSION_KEYRING_CRED, new, old); +} + static void __maybe_unused iee_commit_creds(const struct cred *new) { iee_rw_gate(IEE_OP_COMMIT_CRED, new); diff --git a/arch/arm64/include/asm/haoc/iee-func.h b/arch/arm64/include/asm/haoc/iee-func.h index 2c57ea4aff81e..11bb9750afa8f 100644 --- a/arch/arm64/include/asm/haoc/iee-func.h +++ b/arch/arm64/include/asm/haoc/iee-func.h @@ -10,9 +10,13 @@ #ifndef _LINUX_IEE_ASM_FUNC_H #define _LINUX_IEE_ASM_FUNC_H +#include + extern void set_iee_address(unsigned long addr, unsigned int order, bool valid); extern void iee_set_logical_mem(unsigned long addr, unsigned int order, bool prot); extern void put_pages_into_iee(unsigned long addr, int order); +extern void set_iee_page_type(unsigned long addr, int order, + enum HAOC_BITMAP_TYPE type); extern void set_iee_page(unsigned long addr, int order); extern void unset_iee_page(unsigned long addr, int order); diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index 7aa900608650f..10b43695101e2 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -14,6 +14,26 @@ config IEE verified in IEE APIs. Needs hardware support FEAT_HPDS. +config IEE_ALLOW_SPLIT_LM + bool "Allow IEE split linear mapping" + depends on IEE + def_bool n + help + Allow the IEE initialization path to split arm64 linear-map block + mappings when a protected range cannot be described safely at the + current table level. This is mainly useful for bring-up on layouts + where protected code or data crosses an existing block boundary. + +config IEE_DEBUG + bool "Skip IEE stack switch for debug" + depends on IEE + def_bool n + help + Build the arm64 IEE gate with a debug path that skips the IEE stack + switch and clears HPD before reporting bad gate state. This keeps + early gate failures easier to inspect while debugging IEE bring-up + code, and should remain disabled for normal kernels. + config IEE_PTRP bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile index 62b467471edc0..0bcecf0ea9876 100644 --- a/arch/arm64/kernel/haoc/Makefile +++ b/arch/arm64/kernel/haoc/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += haoc.o +obj-y += haoc.o haoc-bitmap.o obj-y += iee/ -obj-$(CONFIG_CREDP) += credp/ \ No newline at end of file +obj-$(CONFIG_CREDP) += credp/ diff --git a/arch/arm64/kernel/haoc/credp/credp.c b/arch/arm64/kernel/haoc/credp/credp.c index 35b486cceb565..aeec727f9a58c 100644 --- a/arch/arm64/kernel/haoc/credp/credp.c +++ b/arch/arm64/kernel/haoc/credp/credp.c @@ -3,6 +3,8 @@ #include #include #include +#include +#include extern struct cred init_cred; void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, @@ -260,6 +262,39 @@ void _iee_abort_cred(unsigned long __unused, const struct cred *cred) token->new_cred = NULL; } +void __iee_code _iee_fill_cred_for_session_keyring(unsigned long __unused, + struct cred *new, + const struct cred *old) +{ + struct cred *_new = __ptr_to_iee(new); + struct task_token *token = (struct task_token *)__addr_to_iee(current); + + token->new_cred = new; + _new->uid = old->uid; + _new->euid = old->euid; + _new->suid = old->suid; + _new->fsuid = old->fsuid; + _new->gid = old->gid; + _new->egid = old->egid; + _new->sgid = old->sgid; + _new->fsgid = old->fsgid; + _new->user = get_uid(old->user); + _new->ucounts = old->ucounts; + _new->user_ns = get_user_ns(old->user_ns); + _new->group_info = get_group_info(old->group_info); + + _new->securebits = old->securebits; + _new->cap_inheritable = old->cap_inheritable; + _new->cap_permitted = old->cap_permitted; + _new->cap_effective = old->cap_effective; + _new->cap_ambient = old->cap_ambient; + _new->cap_bset = old->cap_bset; + + _new->jit_keyring = old->jit_keyring; + _new->thread_keyring = key_get(old->thread_keyring); + _new->process_keyring = key_get(old->process_keyring); +} + void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); diff --git a/arch/arm64/kernel/haoc/haoc-bitmap.c b/arch/arm64/kernel/haoc/haoc-bitmap.c new file mode 100644 index 0000000000000..694f6256c0fc5 --- /dev/null +++ b/arch/arm64/kernel/haoc/haoc-bitmap.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +bool haoc_bitmap_ready; + +static void * __init haoc_bitmap_alloc_block_zero(unsigned long size) +{ + #ifdef CONFIG_PTP + void *p = __va(early_iee_pgtable_alloc(0)); + + if (!p) + return NULL; + #else + void *p = memblock_alloc(size, size); + + if (!p) + return NULL; + memset(p, 0, size); + #endif + return p; +} + +/* Set APTable on pgd entry of bitmap addresses to prevent kernel writes. */ +static inline void bitmap_p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) +{ + p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF | PGD_APTABLE_RO; + + p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; + __p4d_populate(p4dp, __pa(pudp), p4dval); +} + +static pte_t * __init haoc_bitmap_pte_populate(pmd_t *pmd, unsigned long addr) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + + if (pte_none(ptep_get(pte))) { + pte_t entry; + void *p = __va(early_iee_data_alloc(0)); + + if (!p) + return NULL; + + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); + #ifdef CONFIG_PTP + write_sysreg((read_sysreg(TCR_EL1) | TCR_HPD1) & ~TCR_A1, tcr_el1); + isb(); + WRITE_ONCE(*__ptr_to_iee(pte), entry); + write_sysreg((read_sysreg(TCR_EL1) & ~TCR_HPD1) | TCR_A1, tcr_el1); + isb(); + #else + set_pte_at(&init_mm, addr, pte, entry); + #endif + } + return pte; +} + +static pmd_t * __init haoc_bitmap_pmd_populate(pud_t *pud, unsigned long addr) +{ + pmd_t *pmd = pmd_offset(pud, addr); + void *p; + + if (pmd_none(*pmd)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); + } + return pmd; +} + +static pud_t * __init haoc_bitmap_pud_populate(p4d_t *p4d, unsigned long addr) +{ + pud_t *pud = pud_offset(p4d, addr); + void *p; + + if (pud_none(*pud)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pmd_init(p); + pud_populate(&init_mm, pud, p); + } + return pud; +} + +static p4d_t * __init haoc_bitmap_p4d_populate(pgd_t *pgd, unsigned long addr) +{ + p4d_t *p4d = p4d_offset(pgd, addr); + void *p; + + if (p4d_none(*p4d)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pud_init(p); + bitmap_p4d_populate(&init_mm, p4d, p); + } + return p4d; +} + +static pgd_t * __init haoc_bitmap_pgd_populate(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + void *p; + + if (pgd_none(*pgd)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); + } + return pgd; +} + +/* Create mappings if that address is not mapped. */ +static pte_t * __init haoc_bitmap_populate_address(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = haoc_bitmap_pgd_populate(addr); + if (!pgd) + return NULL; + p4d = haoc_bitmap_p4d_populate(pgd, addr); + if (!p4d) + return NULL; + pud = haoc_bitmap_pud_populate(p4d, addr); + if (!pud) + return NULL; + pmd = haoc_bitmap_pmd_populate(pud, addr); + if (!pmd) + return NULL; + pte = haoc_bitmap_pte_populate(pmd, addr); + if (!pte) + return NULL; + + return pte; +} + +/* Map haoc bitmap array after vmemmap region. */ +int __init haoc_bitmap_sparse_init(void) +{ + unsigned long start_pfn, end_pfn; + int i, nid; + /* Iterate through available memory blocks. */ + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + u64 bitmap_start = ALIGN_DOWN(((u64)__pfn_to_haoc_bitmap(start_pfn)), + PAGE_SIZE); + u64 bitmap_end = ALIGN(((u64)__pfn_to_haoc_bitmap(end_pfn)), + PAGE_SIZE); + u64 addr = bitmap_start; + pte_t *pte; + + for (; addr < bitmap_end; addr += PAGE_SIZE) { + pte = haoc_bitmap_populate_address(addr); + if (!pte) + pr_err("HAOC: failed on bitmap init."); + } + } + haoc_bitmap_ready = true; + return 0; +} + +void __init haoc_bitmap_setup(void) +{ + unsigned long __maybe_unused start; + unsigned long __maybe_unused end; + int __maybe_unused num_pages; + + /* Setup bitmap types of global data. */ + #ifdef CONFIG_PTP + start = (unsigned long)idmap_pg_dir; + end = (unsigned long)__iee_ptp_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_PGTABLE); + start = (unsigned long)init_pg_dir; + end = (unsigned long)init_pg_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_PGTABLE); + #endif + #ifdef CONFIG_IEE_SELINUX_P + start = (unsigned long)__iee_selinux_data_start; + end = (unsigned long)__iee_selinux_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_SELINUX); + #endif + #ifdef CONFIG_VARP + /* Map .iee.varp as RO pages, the variable need to be protected can be added */ + start = (unsigned long)__iee_varp_data_start; + end = (unsigned long)__iee_varp_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_VARP); + #endif + + /* Mark bitmaps of early allocated iee data. */ + setup_iee_early_data_bitmap(); +} + +#pragma GCC push_options +#pragma GCC optimize("O0") +static void __iee_code _iee_bitmap_memset(void *ptr, int data, size_t n) +{ + char *_ptr; + + _ptr = (char *)ptr; + + while (n--) + *_ptr++ = data; +} +#pragma GCC pop_options + +void __iee_code _iee_set_bitmap_type(unsigned long __unused, + u64 va, enum HAOC_BITMAP_TYPE type, int num_pages) +{ + _iee_bitmap_memset(__va_to_haoc_bitmap(va), type, num_pages); +} diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index 33bf9b57c56a4..bbb071026ffc2 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -28,12 +28,14 @@ iee_func iee_funcs[] = { (iee_func)_iee_invalidate_token, (iee_func)_iee_validate_token, #endif + (iee_func)_iee_set_bitmap_type, #ifdef CONFIG_CREDP (iee_func)_iee_copy_cred, (iee_func)_iee_copy_kernel_cred, (iee_func)_iee_init_copied_cred, (iee_func)_iee_commit_creds, (iee_func)_iee_abort_cred, + (iee_func)_iee_fill_cred_for_session_keyring, (iee_func)_iee_set_cred_uid, (iee_func)_iee_set_cred_gid, (iee_func)_iee_set_cred_suid, diff --git a/arch/arm64/kernel/haoc/iee/iee-func.c b/arch/arm64/kernel/haoc/iee/iee-func.c index cef63afc408be..a625bac775d96 100644 --- a/arch/arm64/kernel/haoc/iee/iee-func.c +++ b/arch/arm64/kernel/haoc/iee/iee-func.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -400,6 +401,13 @@ static void remove_pages_from_iee(unsigned long addr, int order) isb(); } +/* See put_pages_into_iee(). */ +void set_iee_page_type(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) +{ + put_pages_into_iee(addr, order); + iee_set_bitmap_type(addr, 1UL << order, type); +} + /* See put_pages_into_iee(). */ void set_iee_page(unsigned long addr, int order) { diff --git a/arch/arm64/kernel/haoc/iee/iee-gate.S b/arch/arm64/kernel/haoc/iee/iee-gate.S index 47246ab0ec320..050844b2e87ce 100644 --- a/arch/arm64/kernel/haoc/iee/iee-gate.S +++ b/arch/arm64/kernel/haoc/iee/iee-gate.S @@ -11,7 +11,18 @@ #include #include #include + +.data +.align 3 +.global iee_gate_inst +iee_gate_inst: + .quad iee_gate_start + +.text + SYM_FUNC_START(iee_rw_gate) +iee_gate_start: + b 1f /* Rewritten to NOP when HAOC is enabled. */ /* save daif, close irq */ mrs x13, daif msr daifset, #0x2 @@ -24,6 +35,15 @@ SYM_FUNC_START(iee_rw_gate) /* restore daif */ msr daif, x13 ret +1: + /* Call IEE functions directly when HAOC stays disabled. */ + adrp x12, iee_funcs + add x12, x12, x0, lsl #3 + ldr x12, [x12, #:lo12:iee_funcs] + stp x29, x30, [sp, #-16]! + blr x12 + ldp x29, x30, [sp], #16 + ret SYM_FUNC_END(iee_rw_gate) EXPORT_SYMBOL(iee_rw_gate) diff --git a/arch/arm64/kernel/haoc/iee/iee-init.c b/arch/arm64/kernel/haoc/iee/iee-init.c index 416ca52e3e9a5..f328e715f58bd 100644 --- a/arch/arm64/kernel/haoc/iee/iee-init.c +++ b/arch/arm64/kernel/haoc/iee/iee-init.c @@ -9,7 +9,10 @@ #include #include +#include #include +#include +#include #include #include #ifdef CONFIG_IEE_PTRP @@ -46,6 +49,22 @@ void iee_setup_asid(void) /* Load IEE ASID into ttbr0 to use it in IEE. */ write_sysreg(ttbr0, ttbr0_el1); isb(); + local_flush_tlb_all(); +} + +extern unsigned int *iee_gate_inst; + +static void iee_setup_rw_gate(void) +{ + unsigned int old_gate_inst = *iee_gate_inst; + unsigned long iee_gate_page = ALIGN_DOWN((unsigned long)&iee_rw_gate, PAGE_SIZE); + unsigned int *iee_gate_inst_lm = (unsigned int *)__va(__pa_symbol(iee_gate_inst)); + + *iee_gate_inst_lm = 0xd503201f; /* NOP */ + icache_inval_pou(iee_gate_page, iee_gate_page + PAGE_SIZE); + + pr_info("HAOC: rewrite iee gate inst 0x%p:0x%x to 0x%x", + (void *)iee_gate_inst, old_gate_inst, *iee_gate_inst); } void __init iee_init_post(void) @@ -53,14 +72,19 @@ void __init iee_init_post(void) if (!haoc_enabled) return; + iee_setup_rw_gate(); iee_setup_asid(); /* Flush tlb to enable IEE. */ flush_tlb_all(); + iee_init_done = true; + + haoc_bitmap_sparse_init(); + haoc_bitmap_setup(); + #ifdef CONFIG_IEE_PTRP iee_prepare_init_task_token(); #endif - iee_init_done = true; #ifdef CONFIG_IEE_SIP extern void iee_si_init(void); diff --git a/arch/arm64/kernel/haoc/iee/iee-mmu.c b/arch/arm64/kernel/haoc/iee/iee-mmu.c index c7b227907c3f0..91ad698d612e4 100644 --- a/arch/arm64/kernel/haoc/iee/iee-mmu.c +++ b/arch/arm64/kernel/haoc/iee/iee-mmu.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -201,31 +202,23 @@ phys_addr_t __init early_iee_data_alloc(int shift) return iee_early_alloc(&iee_data, 0); } +static void iee_clear_pgtable(phys_addr_t phys) +{ + void *table = pte_set_fixmap(phys); + + clear_page(table); + pte_clear_fixmap(); +} + static phys_addr_t __init early_pgtable_alloc(int shift) { phys_addr_t phys; - void *ptr; phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, MEMBLOCK_ALLOC_NOLEAKTRACE); if (!phys) panic("Failed to allocate page table page\n"); - /* - * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE - * slot will be free, so we can (ab)use the FIX_PTE slot to initialise - * any level of table. - */ - ptr = pte_set_fixmap(phys); - - memset(ptr, 0, PAGE_SIZE); - - /* - * Implicit barriers also ensure the zeroed page is visible to the page - * table walker - */ - pte_clear_fixmap(); - return phys; } @@ -271,6 +264,7 @@ static void iee_alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, pmdval |= PMD_TABLE_PXN; IEE_CHECK(!pgtable_alloc); pte_phys = pgtable_alloc(PAGE_SHIFT); + iee_clear_pgtable(pte_phys); __pmd_populate(pmdp, pte_phys, pmdval); pmd = READ_ONCE(*pmdp); } @@ -349,6 +343,7 @@ static void iee_alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, pudval |= PUD_TABLE_PXN; IEE_CHECK(!pgtable_alloc); pmd_phys = pgtable_alloc(PMD_SHIFT); + iee_clear_pgtable(pmd_phys); __pud_populate(pudp, pmd_phys, pudval); pud = READ_ONCE(*pudp); } @@ -388,6 +383,7 @@ static void iee_alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long en p4dval |= P4D_TABLE_PXN; IEE_CHECK(!pgtable_alloc); pud_phys = pgtable_alloc(PUD_SHIFT); + iee_clear_pgtable(pud_phys); __p4d_populate(p4dp, pud_phys, p4dval); p4d = READ_ONCE(*p4dp); } @@ -550,6 +546,23 @@ void __init iee_init_mappings(pgd_t *pgdp) iee_setup_bootcpu_stack(); } +static void setup_iee_data_cache_bitmap(struct iee_early_alloc *cache, + enum HAOC_BITMAP_TYPE type) +{ + int block_nr = cache->curr_block_nr + 1; + + for (int j = 0; j < block_nr; j++) { + iee_set_bitmap_type((unsigned long)__va(cache->blocks[j].start), + 1 << cache->blocks[j].order, type); + } +} + +void __init setup_iee_early_data_bitmap(void) +{ + setup_iee_data_cache_bitmap(&iee_data, IEE_DATA); + setup_iee_data_cache_bitmap(&iee_stack, IEE_DATA); +} + static void prot_iee_early_data_cache(struct iee_early_alloc *cache) { int block_nr = cache->curr_block_nr + 1; diff --git a/arch/arm64/kernel/haoc/iee/iee-si-gate.S b/arch/arm64/kernel/haoc/iee/iee-si-gate.S index 07fe1a0a7c8ec..52b654273949d 100644 --- a/arch/arm64/kernel/haoc/iee/iee-si-gate.S +++ b/arch/arm64/kernel/haoc/iee/iee-si-gate.S @@ -40,6 +40,9 @@ SYM_FUNC_END(iee_rwx_gate) SYM_FUNC_START(iee_rwx_gate_tramp) /* entry gate */ mrs x12, tcr_el1 +#ifdef CONFIG_IEE_DEBUG + tbnz x12, #42, 114f +#endif orr x12, x12, #TCR_HPD1 bic x12, x12, #TCR_A1 msr tcr_el1, x12 @@ -52,10 +55,12 @@ SYM_FUNC_START(iee_rwx_gate_tramp) cmp x9, x12 b.ne 3f +#ifndef CONFIG_IEE_DEBUG /* Switch to iee stack by per cpu ptr. */ ldr_this_cpu x9, iee_cpu_stack_ptr, x10 mov x14, sp mov sp, x9 +#endif 1: stp x13, x14, [sp, #-32]! stp x29, x30, [sp, #16] @@ -78,10 +83,19 @@ SYM_FUNC_START(iee_rwx_gate_tramp) cmp x9, x12 b.ne 3f +#ifndef CONFIG_IEE_DEBUG /* Switch to kernel stack */ mov sp, x14 +#endif 2: ret +#ifdef CONFIG_IEE_DEBUG + /* Close HPDS before panic. */ +114: + bic x12, x12, #TCR_HPD1 + msr tcr_el1, x12 + isb +#endif 3: mov x0, sp mov x1, #BAD_TCR_EL1 diff --git a/arch/x86/include/asm/haoc/haoc-bitmap.h b/arch/x86/include/asm/haoc/haoc-bitmap.h new file mode 100644 index 0000000000000..608defe0feda0 --- /dev/null +++ b/arch/x86/include/asm/haoc/haoc-bitmap.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HAOC_BITMAP_H +#define _LINUX_HAOC_BITMAP_H + +#include +#include +#include +extern unsigned long iee_bitmap_base; +#define __VIEEBITMAP_BASE_L4 0xffffeb0000000000UL +#define __VIEEBITMAP_BASE_L5 0xffd6000000000000UL +#define __pfn_to_iee_bitmap(pfn) ((uint8_t *)iee_bitmap_base + (pfn)) + +enum HAOC_BITMAP_TYPE { + IEE_NORMAL = 0, + IEE_PGTABLE, + IEE_USER_PGTABLE, // modifying IEE_USER_PGTABLE needn't use ptp_rw_gate + IEE_FIXED_PGTABLE, + IEE_POLICY, + IEE_KEY, + IEE_SIP_DATA, + IEE_BITMAP, + IEE_VARP, +}; + +extern void iee_init_bitmap(void); +#define IEE_PHYS_BIT_PTR(phys_addr) ((uint8_t *)((uint8_t *)iee_bitmap_base + PHYS_PFN(phys_addr))) + +extern unsigned long long iee_rw_gate(int flag, ...); + +static inline void iee_set_bitmap_type(phys_addr_t phys_addr, int num_pages, + enum HAOC_BITMAP_TYPE type) +{ + uint8_t *bitmap_ptr = IEE_PHYS_BIT_PTR(phys_addr); + + iee_rw_gate(IEE_OP_SET_BITMAP_TYPE, bitmap_ptr, type, num_pages); +} + +static inline enum HAOC_BITMAP_TYPE iee_get_bitmap_type(phys_addr_t phys_addr) +{ + return *IEE_PHYS_BIT_PTR(phys_addr); +} + +static inline void iee_verify_type(phys_addr_t phys_addr, + enum HAOC_BITMAP_TYPE type, + const char *name) +{ + uint8_t bit_type; + + if (unlikely(!haoc_init_done)) + return; + + bit_type = iee_get_bitmap_type(phys_addr); + + if (unlikely(bit_type != type)) + panic("IEE detected type: %d, fake %s: va(0x%lx)", + bit_type, name, (unsigned long)__va(phys_addr)); +} + +#endif diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index f8139664ca0be..a3512114ef190 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -20,12 +20,14 @@ enum { IEE_OP_INVALIDATE_TOKEN, IEE_OP_VALIDATE_TOKEN, #endif + IEE_OP_SET_BITMAP_TYPE, #ifdef CONFIG_CREDP IEE_OP_COPY_CRED, IEE_OP_COPY_KERNEL_CRED, IEE_OP_INIT_COPIED_CRED, IEE_OP_COMMIT_CRED, IEE_OP_ABORT_CRED, + IEE_OP_FILL_SESSION_KEYRING_CRED, IEE_OP_SET_CRED_UID, IEE_OP_SET_CRED_GID, IEE_OP_SET_CRED_SUID, diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index ef665161259de..44f96b892ec7c 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -10,6 +10,7 @@ #ifndef _LINUX_HAOC_H #define _LINUX_HAOC_H +#include #include #include @@ -24,6 +25,8 @@ void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); #endif +void _iee_set_bitmap_type(unsigned long __unused, uint8_t *bitmap_ptr, + enum HAOC_BITMAP_TYPE type, int num_pages); #ifdef CONFIG_CREDP #include @@ -31,6 +34,9 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); void _iee_copy_cred(unsigned long __unused, struct cred *new); void _iee_copy_kernel_cred(unsigned long iee_offset, const struct cred *old, struct cred *new); void _iee_abort_cred(unsigned long iee_offset, const struct cred *cred); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, + struct cred *new, + const struct cred *old); void _iee_init_copied_cred(unsigned long iee_offset, struct task_struct *new_task, struct cred *new); void _iee_commit_creds(unsigned long iee_offset, const struct cred *new); diff --git a/arch/x86/include/asm/haoc/iee-cred.h b/arch/x86/include/asm/haoc/iee-cred.h index 58b4971d711b3..e33cab7449c1c 100644 --- a/arch/x86/include/asm/haoc/iee-cred.h +++ b/arch/x86/include/asm/haoc/iee-cred.h @@ -29,6 +29,12 @@ static void __maybe_unused iee_abort_creds(struct cred *cred) iee_rw_gate(IEE_OP_ABORT_CRED, cred); } +static void __maybe_unused iee_fill_cred_for_session_keyring(struct cred *new, + const struct cred *old) +{ + iee_rw_gate(IEE_OP_FILL_SESSION_KEYRING_CRED, new, old); +} + static void __maybe_unused iee_commit_creds(const struct cred *new) { iee_rw_gate(IEE_OP_COMMIT_CRED, new); diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h index ffad94ecb8b63..5a52de12f2f53 100644 --- a/arch/x86/include/asm/haoc/iee-func.h +++ b/arch/x86/include/asm/haoc/iee-func.h @@ -12,7 +12,11 @@ #define HUGE_PMD_ORDER 9 +#include #include +extern void set_iee_pages(unsigned long addr, int num_pages, + enum HAOC_BITMAP_TYPE type); +extern void unset_iee_pages(unsigned long addr, int num_pages); extern void set_iee_page(unsigned long addr, unsigned int order); extern void unset_iee_page(unsigned long addr, unsigned int order); extern bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order); diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h index bb9bb11875006..432d2bc9d27e2 100644 --- a/arch/x86/include/asm/haoc/iee.h +++ b/arch/x86/include/asm/haoc/iee.h @@ -13,6 +13,8 @@ #include extern unsigned long IEE_OFFSET; +extern bool haoc_enabled; +extern bool haoc_init_done; #define __iee_pa(x) (__pa(x - IEE_OFFSET)) #define __phys_to_iee(x) ((void *)(__va(x) + IEE_OFFSET)) #define __page_to_phys(x) (page_to_pfn(x) << PAGE_SHIFT) @@ -48,7 +50,8 @@ struct iee_stack { DECLARE_PER_CPU(struct iee_stack, iee_stacks); extern void *alloc_low_pages(unsigned int num); +extern void iee_early_init(void); extern void iee_init(void); -extern bool haoc_enabled; +extern void iee_post_init(void); extern bool iee_init_done; #endif diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index d9e596eae225c..bd02c484b78c7 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2644,8 +2644,4 @@ void __init arch_cpu_finalize_init(void) * hypercalls work when the SWIOTLB bounce buffers are decrypted. */ mem_encrypt_init(); - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_sip_init(); - #endif } diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index 367af97ba37ea..a1b0b8e5c7162 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -1,3 +1,3 @@ -obj-y += haoc.o +obj-y += haoc.o bitmap.o obj-y += iee/ -obj-$(CONFIG_CREDP) += credp/ \ No newline at end of file +obj-$(CONFIG_CREDP) += credp/ diff --git a/arch/x86/kernel/haoc/bitmap.c b/arch/x86/kernel/haoc/bitmap.c new file mode 100644 index 0000000000000..57fc0ff87cf19 --- /dev/null +++ b/arch/x86/kernel/haoc/bitmap.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +unsigned long iee_bitmap_base __ro_after_init = __VIEEBITMAP_BASE_L4; +EXPORT_SYMBOL(iee_bitmap_base); + +static pte_t * __init iee_bitmap_pte_populate(pmd_t *pmd, unsigned long addr) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + + if (pte_none(*pte)) { + void *p = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 0); + + if (!p) + return NULL; + pte_t entry = pfn_pte(__pa(p) >> PAGE_SHIFT, + __pgprot(pgprot_val(PAGE_KERNEL) & (~__RW) & (~___D))); + + set_pte_at(&init_mm, addr, pte, entry); + } + return pte; +} + +static pmd_t * __init iee_bitmap_pmd_populate(pud_t *pud, unsigned long addr) +{ + pmd_t *pmd = pmd_offset(pud, addr); + + if (pmd_none(*pmd)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); + } + return pmd; +} + +static pud_t * __init iee_bitmap_pud_populate(p4d_t *p4d, unsigned long addr) +{ + pud_t *pud = pud_offset(p4d, addr); + + if (pud_none(*pud)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + pud_populate(&init_mm, pud, p); + } + return pud; +} + +static p4d_t * __init iee_bitmap_p4d_populate(pgd_t *pgd, unsigned long addr) +{ + p4d_t *p4d = p4d_offset(pgd, addr); + + if (p4d_none(*p4d)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + p4d_populate(&init_mm, p4d, p); + } + return p4d; +} + +static pgd_t * __init iee_bitmap_pgd_populate(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + + if (pgd_none(*pgd)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); + } + return pgd; +} + +static pte_t * __init iee_bitmap_populate(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = iee_bitmap_pgd_populate(addr); + if (!pgd) + return NULL; + p4d = iee_bitmap_p4d_populate(pgd, addr); + if (!p4d) + return NULL; + pud = iee_bitmap_pud_populate(p4d, addr); + if (!pud) + return NULL; + pmd = iee_bitmap_pmd_populate(pud, addr); + if (!pmd) + return NULL; + pte = iee_bitmap_pte_populate(pmd, addr); + return pte; +} + +static void __init iee_set_bitmap_type_init(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + phys_addr_t phys_addr; + + pgd = pgd_offset_k(addr); + p4d = p4d_offset(pgd, addr); + pud = pud_offset(p4d, addr); + pmd = pmd_offset(pud, addr); + pte = pte_offset_kernel(pmd, addr); + phys_addr = pte_pfn(*pte) << PAGE_SHIFT; + iee_set_bitmap_type(phys_addr, 1, IEE_BITMAP); +} + +void __init iee_init_bitmap(void) +{ + unsigned long addr, start, end; + pte_t *pte; + + start = PAGE_ALIGN_DOWN((unsigned long)(__pfn_to_iee_bitmap(0))); + end = PAGE_ALIGN((unsigned long)(__pfn_to_iee_bitmap(max_pfn))); + addr = start; + for (; addr < end; addr += PAGE_SIZE) { + pte = iee_bitmap_populate(addr); + if (!pte) + panic("No mem for iee bitmap init"); + } + addr = start; + for (; addr < end; addr += PAGE_SIZE) + iee_set_bitmap_type_init(addr); + pr_info("IEE bitmap: [0x%lx, 0x%lx]!", start, end); +} + +static inline void verify_bitmap_ptr(uint8_t *bitmap_ptr) +{ + unsigned long addr = (unsigned long)bitmap_ptr; + unsigned long start = PAGE_ALIGN_DOWN((unsigned long)(__pfn_to_iee_bitmap(0))); + unsigned long end = PAGE_ALIGN((unsigned long)(__pfn_to_iee_bitmap(max_pfn))); + + if (addr < start || addr >= end) + panic("IEE detected non-bitmap ptr: %lx", addr); +} + +static uint8_t *iee_bitmap_rw_alias(uint8_t *bitmap_ptr) +{ + unsigned long addr = (unsigned long)bitmap_ptr; + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); + pmd_t *pmd = pmd_offset(pud, addr); + pte_t *pte = pte_offset_kernel(pmd, addr); + phys_addr_t phys = (pte_pfn(*pte) << PAGE_SHIFT) | (addr & ~PAGE_MASK); + + return __va(phys); +} + +void _iee_set_bitmap_type(unsigned long __unused, + uint8_t *bitmap_ptr, enum HAOC_BITMAP_TYPE type, int num_pages) +{ + verify_bitmap_ptr(bitmap_ptr); + if (!haoc_enabled) + bitmap_ptr = iee_bitmap_rw_alias(bitmap_ptr); + memset(bitmap_ptr, type, num_pages); +} diff --git a/arch/x86/kernel/haoc/credp/credp.c b/arch/x86/kernel/haoc/credp/credp.c index 2ecc48a9fe152..7e07cfdec5ff6 100644 --- a/arch/x86/kernel/haoc/credp/credp.c +++ b/arch/x86/kernel/haoc/credp/credp.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -237,6 +238,37 @@ void _iee_abort_cred(unsigned long __unused, const struct cred *cred) token->new_cred = NULL; } +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old) +{ + struct task_token *token = (struct task_token *)__addr_to_iee(current); + + token->new_cred = new; + new->uid = old->uid; + new->euid = old->euid; + new->suid = old->suid; + new->fsuid = old->fsuid; + new->gid = old->gid; + new->egid = old->egid; + new->sgid = old->sgid; + new->fsgid = old->fsgid; + new->user = get_uid(old->user); + new->ucounts = old->ucounts; + new->user_ns = get_user_ns(old->user_ns); + new->group_info = get_group_info(old->group_info); + + new->securebits = old->securebits; + new->cap_inheritable = old->cap_inheritable; + new->cap_permitted = old->cap_permitted; + new->cap_effective = old->cap_effective; + new->cap_ambient = old->cap_ambient; + new->cap_bset = old->cap_bset; + + new->jit_keyring = old->jit_keyring; + new->thread_keyring = key_get(old->thread_keyring); + new->process_keyring = key_get(old->process_keyring); +} + void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) { cred->gid = gid; diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index b9576af3fc913..9e62a8bc4caab 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -20,12 +20,14 @@ iee_func iee_funcs[] = { (iee_func)_iee_invalidate_token, (iee_func)_iee_validate_token, #endif + (iee_func)_iee_set_bitmap_type, #ifdef CONFIG_CREDP (iee_func)_iee_copy_cred, (iee_func)_iee_copy_kernel_cred, (iee_func)_iee_init_copied_cred, (iee_func)_iee_commit_creds, (iee_func)_iee_abort_cred, + (iee_func)_iee_fill_cred_for_session_keyring, (iee_func)_iee_set_cred_uid, (iee_func)_iee_set_cred_gid, (iee_func)_iee_set_cred_suid, diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index 0054c1b3d6169..aa3a9de793803 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -15,6 +15,18 @@ #endif extern bool haoc_enabled; +void set_iee_pages(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) +{ + set_memory_ro(addr, num_pages); + iee_set_bitmap_type(__pa(addr), num_pages, type); +} + +void unset_iee_pages(unsigned long addr, int num_pages) +{ + iee_set_bitmap_type(__pa(addr), num_pages, IEE_NORMAL); + set_memory_rw(addr, num_pages); +} + void set_iee_page(unsigned long addr, unsigned int order) { set_memory_ro(addr, 1 << order); @@ -34,11 +46,14 @@ struct iee_free_slab_work { void iee_free_slab(struct kmem_cache *s, struct slab *slab, void (*do_free_slab)(struct work_struct *work)) { - if(haoc_enabled) - return; struct iee_free_slab_work *iee_free_slab_work = kmalloc(sizeof(struct iee_free_slab_work), GFP_ATOMIC); + if (!iee_free_slab_work) { + pr_warn("HAOC: failed to allocate deferred slab free work\n"); + return; + } + iee_free_slab_work->s = s; iee_free_slab_work->slab = slab; INIT_WORK(&iee_free_slab_work->work, do_free_slab); @@ -88,4 +103,4 @@ unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) order = IEE_DATA_ORDER; #endif return order; -} \ No newline at end of file +} diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index 5497c3788b026..491978ab46917 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -1,12 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * HAOC feature support - * - * Copyright (C) 2025 ZGCLAB - * Authors: Shu Hang - * Hu Bing - */ - #include #include #include @@ -16,18 +8,18 @@ #define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) /* - * scratch_reg would be changed, - * caller should dertimine if scratch_reg should be saved and restored. + * scratch_reg is clobbered, so callers must decide whether it should be + * saved and restored around these macros. */ .macro DISABLE_WP scratch_reg:req - /* Disable write protection*/ + /* Disable write protection. */ movq %cr0, %\scratch_reg andq $(~X86_CR0_WP), %\scratch_reg movq %\scratch_reg, %cr0 .endm .macro ENABLE_WP scratch_reg:req - /* Enable write protection */ + /* Enable write protection. */ movq %cr0, %\scratch_reg 1: orq $X86_CR0_WP, %\scratch_reg @@ -36,85 +28,80 @@ je 1b .endm -/* - * IEE memory access gate. - * Kernel calls the gate to modify IEE-protected memory. - */ - SYM_FUNC_START(iee_rw_gate) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_rw + +.L_haoc_disabled_rw: + leaq iee_funcs(%rip), %rax + call *(%rax, %rdi, 8) + jmp __x86_return_thunk + +.L_haoc_enabled_rw: /* save Interrupt flag */ pushfq - /* close irq*/ + /* close irq */ cli pushq %r12 DISABLE_WP r12 - /* switch to iee stack */ - movq %rsp, %r12 - movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp - /* call iee func */ leaq iee_funcs(%rip), %rax call *(%rax, %rdi, 8) - /* switch to kernel stack */ - movq %r12, %rsp - ENABLE_WP r12 popq %r12 - /* restore irq*/ + /* restore irq */ popfq - jmp __x86_return_thunk /* ret */ + jmp __x86_return_thunk SYM_FUNC_END(iee_rw_gate) EXPORT_SYMBOL(iee_rw_gate) #ifdef CONFIG_IEE_SIP SYM_FUNC_START(iee_rwx_gate) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_rwx + +.L_haoc_disabled_rwx: + call _iee_si_handler + jmp __x86_return_thunk + +.L_haoc_enabled_rwx: pushq %r12 - - /* save Interrupt flag*/ + + /* save Interrupt flag */ pushfq /* close irq */ cli - + /* set SMEP=0 to enable supervisor-mode exec user-mode insn */ - movq %cr4, %rax /* rax -> cr4 */ + movq %cr4, %rax andq $(~X86_CR4_SMEP), %rax movq %rax, %cr4 - + DISABLE_WP r12 - - movq %rsp, %r12 - /* If iee hasn't been initialized, skip stack switch. */ - cmpb $0, iee_init_done(%rip) - jz 2f - - /* switch to iee stack */ - movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp - -2: call _iee_si_handler - /* switch to kernel stack. If iee hasn't been initialized, skip switch*/ - movq %r12, %rsp - + + call _iee_si_handler + ENABLE_WP r12 - + /* set SMEP=1 to disable supervisor-mode exec user-mode insn */ - movq %cr4, %rax /* rax -> cr4 */ -1: orq $X86_CR4_SMEP_SMAP, %rax + movq %cr4, %rax +1: orq $X86_CR4_SMEP_SMAP, %rax movq %rax, %cr4 andq $X86_CR4_SMEP_SMAP, %rax cmpq $X86_CR4_SMEP_SMAP, %rax - jnz 1 - - /* restore irq*/ + jnz 1b + + /* restore irq */ popfq - + popq %r12 - jmp __x86_return_thunk /* ret */ + jmp __x86_return_thunk SYM_FUNC_END(iee_rwx_gate) -#endif \ No newline at end of file +#endif diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 7ac4704769f4d..f1ef8b56f42fd 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -7,19 +7,24 @@ * Hu Bing */ -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include #include #include +#include +#include #ifdef CONFIG_IEE_SIP #include #endif /* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ unsigned long IEE_OFFSET = 0x200000000000; + #ifdef CONFIG_IEE_SIP bool iee_init_done __iee_si_data; #else @@ -27,7 +32,24 @@ bool iee_init_done; #endif DEFINE_PER_CPU(struct iee_stack, iee_stacks); -static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, unsigned long end) +bool __ro_after_init haoc_enabled; +EXPORT_SYMBOL(haoc_enabled); + +bool __ro_after_init haoc_init_done; + +/* + * Deepin keeps HAOC disabled by default. The boot parameter enables the + * three-stage IEE initialization when set to a true value, for example haoc=on. + */ +static bool cmd_haoc_enabled; +static int __init parse_cmd_haoc_enabled(char *str) +{ + return kstrtobool(str, &cmd_haoc_enabled); +} +early_param("haoc", parse_cmd_haoc_enabled); + +static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, + unsigned long end) { void *p; pmd_t *pmd; @@ -37,10 +59,10 @@ static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, uns addr = ALIGN_DOWN(addr, PMD_SIZE); phys = __iee_pa(addr); - pgprot_shadow_pmd = __pgprot(pgprot_val(PAGE_KERNEL_LARGE) & (~__RW) & (~___D)); + pgprot_shadow_pmd = __pgprot(pgprot_val(PAGE_KERNEL_LARGE) & ~__RW & ~___D); if (pud_none(*pud)) { - p = alloc_low_pages(1); + p = ptdesc_to_virt(pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0)); pud_populate(&init_mm, pud, p); } @@ -52,14 +74,15 @@ static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, uns } while (pmd++, addr = pmd_next, addr != end); } -static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, unsigned long end) +static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, + unsigned long end) { void *p; pud_t *pud; unsigned long pud_next; if (p4d_none(*p4d)) { - p = alloc_low_pages(1); + p = ptdesc_to_virt(pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0)); p4d_populate(&init_mm, p4d, p); } @@ -72,14 +95,15 @@ static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, uns } while (pud++, addr = pud_next, addr != end); } -static void __init _iee_mapping_populate_pgd(pgd_t *pgd, unsigned long addr, unsigned long end) +static void __init _iee_mapping_populate_pgd(pgd_t *pgd, unsigned long addr, + unsigned long end) { void *p; p4d_t *p4d; unsigned long p4d_next; if (pgd_none(*pgd)) { - p = alloc_low_pages(1); + p = ptdesc_to_virt(pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0)); pgd_populate(&init_mm, pgd, p); } @@ -97,7 +121,6 @@ static void __init _iee_init_mapping(phys_addr_t start_paddr, phys_addr_t end_pa unsigned long addr = (unsigned long)__phys_to_iee(start_paddr); unsigned long end = (unsigned long)__phys_to_iee(end_paddr); unsigned long pgd_next; - pgd_t *pgd = pgd_offset_k(addr); spin_lock(&pgd_lock); @@ -110,12 +133,17 @@ static void __init _iee_init_mapping(phys_addr_t start_paddr, phys_addr_t end_pa spin_unlock(&pgd_lock); } -static void __init _iee_mapping_init(void) +static void __init iee_init_offset(void) +{ + if (pgtable_l5_enabled()) + IEE_OFFSET = 0x40000000000000; +} + +static void __init iee_init_mapping(void) { struct memblock_region *r; unsigned long start_pfn, end_pfn; phys_addr_t start_paddr, end_paddr; - unsigned long nr_pages = 0; for_each_mem_region(r) { start_pfn = memblock_region_memory_base_pfn(r); @@ -124,8 +152,6 @@ static void __init _iee_mapping_init(void) start_paddr = PFN_PHYS(start_pfn); end_paddr = PFN_PHYS(end_pfn); - nr_pages += end_pfn - start_pfn; - pr_info("IEE: mapping iee mapping [mem %#010lx-%#010lx]\n", (unsigned long)start_paddr, (unsigned long)end_paddr); @@ -134,52 +160,98 @@ static void __init _iee_mapping_init(void) pr_info("IEE: IEE shadow mapping init done"); } -static void __init _iee_stack_init(void) +static void __init iee_init_stack(void) { int cpu; struct iee_stack *iee_stack; - void *stack_base; + unsigned long stack_base; struct page *page; for_each_possible_cpu(cpu) { - stack_base = (void *)page_address(alloc_pages(GFP_KERNEL, IEE_STACK_ORDER)); iee_stack = per_cpu_ptr(&iee_stacks, cpu); page = alloc_pages(GFP_KERNEL, IEE_STACK_ORDER); - iee_stack->stack = (void *)page_address(page) + PAGE_SIZE * (1 << IEE_STACK_ORDER); - pr_info("IEE: cpu %d, iee_stack 0x%lx", cpu, (unsigned long)iee_stack->stack); - set_memory_ro((unsigned long)stack_base, (1 << IEE_STACK_ORDER)); + if (!page) + panic("IEE: no memory for iee stack"); + + stack_base = (unsigned long)page_address(page); + iee_stack->stack = (void *)stack_base + PAGE_SIZE * (1 << IEE_STACK_ORDER); + pr_info("IEE: cpu %d, iee_stack 0x%lx", cpu, + (unsigned long)iee_stack->stack); } } -static void __init _iee_offset_init(void) +static void __init iee_stack_set_ro(void) { - if (pgtable_l5_enabled()) - IEE_OFFSET = 0x40000000000000; + int cpu; + struct iee_stack *iee_stack; + unsigned long stack_base; + + for_each_possible_cpu(cpu) { + iee_stack = per_cpu_ptr(&iee_stacks, cpu); + stack_base = (unsigned long)iee_stack->stack - PAGE_SIZE * (1 << IEE_STACK_ORDER); + set_iee_pages(stack_base, 1 << IEE_STACK_ORDER, IEE_SIP_DATA); + } } -void __init iee_init(void) +static bool __init check_haoc_hardware_support(void) { - _iee_offset_init(); - _iee_mapping_init(); - _iee_stack_init(); + return cpu_feature_enabled(X86_FEATURE_SMAP) && + cpu_feature_enabled(X86_FEATURE_SMEP); } -bool __ro_after_init haoc_enabled; -EXPORT_SYMBOL(haoc_enabled); #ifdef CONFIG_IEE_SIP extern unsigned long cr4_pinned_mask; + +static void __init iee_setup_cr4_pinning(void) +{ + cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | + X86_CR4_FSGSBASE | X86_CR4_CET; +} #endif -static int __init parse_haoc_enabled(char *str) + +void __init iee_early_init(void) { - int ret = kstrtobool(str, &haoc_enabled); - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - { - cr4_pinned_mask = - X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET; - } - #endif - return ret; + if (!cmd_haoc_enabled) + return; + + if (!check_haoc_hardware_support()) + pr_info("HAOC disabled because SMAP or SMEP is not available"); +} + +void __init iee_init(void) +{ + if (!cmd_haoc_enabled) + return; + + if (!check_haoc_hardware_support()) + return; + + iee_init_offset(); + iee_init_mapping(); + iee_init_bitmap(); + iee_init_stack(); + + /* + * HAOC temporarily clears CR0.WP in the write gate. Disable CET while + * HAOC is active to avoid the CR4.CET and CR0.WP architectural conflict. + */ + cet_disable(); + + haoc_enabled = true; +#ifdef CONFIG_IEE_SIP + iee_setup_cr4_pinning(); +#endif + pr_info("HAOC initialized"); +} + +void __init iee_post_init(void) +{ + if (!cmd_haoc_enabled || !haoc_enabled) + return; + + iee_stack_set_ro(); +#ifdef CONFIG_IEE_SIP + iee_sip_init(); +#endif + haoc_init_done = true; } -early_param("haoc", parse_haoc_enabled); diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 8bfa00bc2c79e..32f1769a85dd5 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1364,6 +1364,10 @@ static void __init preallocate_vmalloc_pages(void) void __init mem_init(void) { +#ifdef CONFIG_IEE + iee_early_init(); +#endif + pci_iommu_alloc(); /* clear_bss() already clear the empty_zero_page */ @@ -1385,19 +1389,11 @@ void __init mem_init(void) if (get_gate_vma(&init_mm)) kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); - #ifdef CONFIG_IEE - /* - * Split the linear mapping region of the kernel address space into two equally-sized parts. - * The lower region retains the original linear mapping. - * The upper region becomes the IEE linear mapping area. - * Note that the IEE mapping region is mapped with read-only permissions. - */ - if (haoc_enabled) { - pr_info("HAOC is enabled by kernel command line."); - iee_init(); - } - #endif preallocate_vmalloc_pages(); + +#ifdef CONFIG_IEE + iee_init(); +#endif } #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT diff --git a/init/main.c b/init/main.c index 1187c46b11dca..cfb550a81208d 100644 --- a/init/main.c +++ b/init/main.c @@ -106,6 +106,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #define CREATE_TRACE_POINTS #include @@ -1044,6 +1047,10 @@ void start_kernel(void) arch_post_acpi_subsys_init(); kcsan_init(); +#if defined(CONFIG_IEE) && defined(CONFIG_X86_64) + iee_post_init(); +#endif + /* Do the rest non-__init'ed, we're now alive */ arch_call_rest_init(); diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index aa56394380fd7..8517970bb56ce 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -919,6 +919,38 @@ long join_session_keyring(const char *name) return ret; } +/* + * Copy the credential fields that a session keyring change needs, + * reusing cred_alloc_blank()'s blank allocation. + */ +static void fill_cred_for_session_keyring(struct cred *new, + const struct cred *old) +{ + new-> uid = old-> uid; + new-> euid = old-> euid; + new-> suid = old-> suid; + new->fsuid = old->fsuid; + new-> gid = old-> gid; + new-> egid = old-> egid; + new-> sgid = old-> sgid; + new->fsgid = old->fsgid; + new->user = get_uid(old->user); + new->ucounts = old->ucounts; + new->user_ns = get_user_ns(old->user_ns); + new->group_info = get_group_info(old->group_info); + + new->securebits = old->securebits; + new->cap_inheritable = old->cap_inheritable; + new->cap_permitted = old->cap_permitted; + new->cap_effective = old->cap_effective; + new->cap_ambient = old->cap_ambient; + new->cap_bset = old->cap_bset; + + new->jit_keyring = old->jit_keyring; + new->thread_keyring = key_get(old->thread_keyring); + new->process_keyring = key_get(old->process_keyring); +} + /* * Replace a process's session keyring on behalf of one of its children when * the target process is about to resume userspace execution. @@ -949,54 +981,11 @@ void key_change_session_keyring(struct callback_head *twork) } #ifdef CONFIG_CREDP - iee_set_cred_uid(new, old->uid); - iee_set_cred_euid(new, old->euid); - iee_set_cred_suid(new, old->suid); - iee_set_cred_fsuid(new, old->fsuid); - iee_set_cred_gid(new, old->gid); - iee_set_cred_egid(new, old->egid); - iee_set_cred_sgid(new, old->sgid); - iee_set_cred_fsgid(new, old->fsgid); - iee_set_cred_user(new, get_uid(old->user)); - iee_set_cred_ucounts(new, old->ucounts); - iee_set_cred_user_ns(new, get_user_ns(old->user_ns)); - iee_set_cred_group_info(new, get_group_info(old->group_info)); - - iee_set_cred_securebits(new, old->securebits); - iee_set_cred_cap_inheritable(new, old->cap_inheritable); - iee_set_cred_cap_permitted(new, old->cap_permitted); - iee_set_cred_cap_effective(new, old->cap_effective); - iee_set_cred_cap_ambient(new, old->cap_ambient); - iee_set_cred_cap_bset(new, old->cap_bset); - - iee_set_cred_jit_keyring(new, old->jit_keyring); - iee_set_cred_thread_keyring(new, key_get(old->thread_keyring)); - iee_set_cred_process_keyring(new, key_get(old->process_keyring)); - #else - new-> uid = old-> uid; - new-> euid = old-> euid; - new-> suid = old-> suid; - new->fsuid = old->fsuid; - new-> gid = old-> gid; - new-> egid = old-> egid; - new-> sgid = old-> sgid; - new->fsgid = old->fsgid; - new->user = get_uid(old->user); - new->ucounts = old->ucounts; - new->user_ns = get_user_ns(old->user_ns); - new->group_info = get_group_info(old->group_info); - - new->securebits = old->securebits; - new->cap_inheritable = old->cap_inheritable; - new->cap_permitted = old->cap_permitted; - new->cap_effective = old->cap_effective; - new->cap_ambient = old->cap_ambient; - new->cap_bset = old->cap_bset; - - new->jit_keyring = old->jit_keyring; - new->thread_keyring = key_get(old->thread_keyring); - new->process_keyring = key_get(old->process_keyring); + if (haoc_enabled) + iee_fill_cred_for_session_keyring(new, old); + else #endif + fill_cred_for_session_keyring(new, old); security_transfer_creds(new, old); From f3d44ecd7c483b4859a34645347a7be9541504da Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 25 Jun 2026 17:36:28 +0800 Subject: [PATCH 02/26] HAOC/IEE_SIP: Upgrade: prepare for haoc-4.0 Update IEE_SIP with bitmap-backed SIP data, CR4 pin handling, and arm64 SI text and TTBR0 gate updates. - Mark x86 .iee.si_data as HAOC SIP data instead of generic read-only memory. - Restore CR4 pin checking inside the x86 SIP handler, and move SMEP/SMAP pin removal into IEE post-init. - Move kernel_set_to_readonly into SIP data and update it through iee_memcpy after rodata has been protected. - Let arm64 split SI text PMD block mappings, clear continuous mapping bits, and mark SI text PTEs non-global before applying PXNTable. - Route arm64 reserved TTBR0 writes through the SIP handler, and tag TTBR0 with the IEE ASID from the enabled HAOC path. - Add arm64 SIP debug gate handling so debug mode can skip the IEE stack switch and clear HPD before reporting a bad gate state. Signed-off-by: Liu Zhehui --- arch/arm64/include/asm/mmu_context.h | 4 + arch/arm64/kernel/haoc/iee/iee-si.c | 125 ++++++++++++++++++++++++--- arch/x86/kernel/haoc/iee/iee-init.c | 9 +- arch/x86/kernel/haoc/iee/iee-si.c | 19 +++- arch/x86/mm/init_64.c | 17 ++++ 5 files changed, 154 insertions(+), 20 deletions(-) diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index f905e7814a78d..7875ca1692042 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -54,11 +54,15 @@ static inline void cpu_set_reserved_ttbr0_nosync(void) { unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); +#ifdef CONFIG_IEE_SIP + iee_rwx_gate(IEE_SI_SET_TTBR0, ttbr); +#else #ifdef CONFIG_IEE if (iee_init_done) ttbr |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); #endif write_sysreg(ttbr, ttbr0_el1); +#endif } static inline void cpu_set_reserved_ttbr0(void) diff --git a/arch/arm64/kernel/haoc/iee/iee-si.c b/arch/arm64/kernel/haoc/iee/iee-si.c index 09ab873ad16c3..9d85d880eb37a 100644 --- a/arch/arm64/kernel/haoc/iee/iee-si.c +++ b/arch/arm64/kernel/haoc/iee/iee-si.c @@ -1,12 +1,18 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include +#include #include #include #include #include +#include #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_PTP +#include +#endif u64 __ro_after_init iee_si_reserved_pg_dir; @@ -57,7 +63,8 @@ unsigned long __iee_si_code iee_si_handler(int flag, ...) if (iee_init_done) iee_si_check_ttbr0(); /* Load the reserved IEE ASID into TTBR0.*/ - ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); + if (haoc_enabled) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); write_sysreg(ttbr0, ttbr0_el1); break; case IEE_SI_CONTEXT_SWITCH: @@ -82,6 +89,8 @@ unsigned long __iee_si_code iee_si_handler(int flag, ...) if (iee_init_done) panic("IEE: Using legacy context switch after IEE init."); + if (haoc_enabled) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); break; @@ -102,6 +111,73 @@ unsigned long __iee_si_code iee_si_handler(int flag, ...) return 0; } +static phys_addr_t __init iee_si_early_pgtable_alloc(void) +{ + phys_addr_t phys; + +#ifdef CONFIG_PTP + phys = early_iee_pgtable_alloc(0); +#else + phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, + MEMBLOCK_ALLOC_NOLEAKTRACE); +#endif + if (!phys) + panic("Failed to allocate page table page\n"); + + return phys; +} + +static inline pgprot_t iee_pmd_pgprot(pmd_t pmd) +{ + unsigned long pfn = pmd_pfn(pmd); + + return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd)); +} + +/* Split PMD block mappings so SI text can carry table/page permissions. */ +static void __init iee_si_split_pmd_early(pud_t *pudp, unsigned long addr) +{ + pmd_t *pmdp = pmd_offset(pudp, addr); + struct page *origin_page; + phys_addr_t pte_phys; + + if (!pmd_leaf(*pmdp)) + return; + + if (pmd_val(*pmdp) & PTE_CONT) { + pmd_t *cont_pmdp = pmd_offset(pudp, addr & CONT_PMD_MASK); + int i; + + for (i = 0; i < CONT_PMDS; i++, cont_pmdp++) + set_pmd(cont_pmdp, __pmd(pmd_val(*cont_pmdp) & ~PTE_CONT)); + } + + origin_page = pmd_page(*pmdp); + pte_phys = iee_si_early_pgtable_alloc(); + if (!pte_phys) + panic("Alloc pgtable error.\n"); + else { + pte_t *ptep = __va(pte_phys); + pgprot_t pgprot = iee_pmd_pgprot(*pmdp); + int i; + +#ifdef CONFIG_PTP + iee_memset(ptep, 0, PAGE_SIZE); +#else + memset(ptep, 0, PAGE_SIZE); +#endif + for (i = 0; i < PTRS_PER_PMD; i++, ptep++) { + pte_t entry; + + pgprot = __pgprot(pgprot_val(pgprot) | PTE_NG | PTE_TYPE_PAGE); + entry = mk_pte(origin_page + i, pgprot); + set_pte(ptep, entry); + } + } + + __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE | PMD_TABLE_UXN); +} + static inline void iee_si_setup_data(void) { iee_si_reserved_pg_dir = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); @@ -111,20 +187,47 @@ static inline void iee_si_setup_data(void) static int __init iee_si_init_code(void) { u64 addr = (u64)__iee_si_text_start; + u64 end = (u64)__iee_si_text_end; pgd_t *pgdir = swapper_pg_dir; - pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); - p4d_t *p4dp = p4d_offset(pgdp, addr); - pud_t *pudp = pud_offset(p4dp, addr); - pmd_t *pmdp = pmd_offset(pudp, addr); - pmd_t pmd = READ_ONCE(*pmdp); - if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) { - pr_err("IEE SI: addr 0x%llx is pmd block. SI init failed.", addr); - return 0; + while (addr < end) { + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + p4d_t *p4dp = p4d_offset(pgdp, addr); + pud_t *pudp = pud_offset(p4dp, addr); + pmd_t *pmdp = pmd_offset(pudp, addr); + pmd_t pmd = READ_ONCE(*pmdp); + pte_t *ptep; + pte_t pte; + + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) { + pr_info("IEE SI: code is on a pmd block. Splitting..."); + iee_si_split_pmd_early(pudp, addr); + pmd = READ_ONCE(*pmdp); + } + + pmd = __pmd(pmd_val(pmd) | PGD_PXNTABLE); + set_pmd(pmdp, pmd); + + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE) { + ptep = pte_offset_kernel(pmdp, addr); + pte = READ_ONCE(*ptep); + + if (pte_val(pte) & PTE_CONT) { + pte_t *cont_ptep = pte_offset_kernel(pmdp, + addr & CONT_PTE_MASK); + int i; + + for (i = 0; i < CONT_PTES; i++, cont_ptep++) + set_pte(cont_ptep, + __pte(pte_val(*cont_ptep) & ~PTE_CONT)); + pte = READ_ONCE(*ptep); + } + pte = __pte(pte_val(pte) | PTE_NG); + set_pte(ptep, pte); + } + addr += PAGE_SIZE; } - pmd = __pmd(pmd_val(pmd) | PGD_PXNTABLE); - set_pmd(pmdp, pmd); flush_tlb_all(); return 1; } diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index f1ef8b56f42fd..c2aa93bf76af9 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -200,12 +200,9 @@ static bool __init check_haoc_hardware_support(void) } #ifdef CONFIG_IEE_SIP -extern unsigned long cr4_pinned_mask; - -static void __init iee_setup_cr4_pinning(void) +static void __init iee_setup_sip_cr4_pinning(void) { - cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET; + cr4_pinned_mask &= ~(X86_CR4_SMEP | X86_CR4_SMAP); } #endif @@ -239,7 +236,7 @@ void __init iee_init(void) haoc_enabled = true; #ifdef CONFIG_IEE_SIP - iee_setup_cr4_pinning(); + iee_setup_sip_cr4_pinning(); #endif pr_info("HAOC initialized"); } diff --git a/arch/x86/kernel/haoc/iee/iee-si.c b/arch/x86/kernel/haoc/iee/iee-si.c index 214c7617cfed0..8feb9d8312246 100644 --- a/arch/x86/kernel/haoc/iee/iee-si.c +++ b/arch/x86/kernel/haoc/iee/iee-si.c @@ -4,6 +4,7 @@ #include #include #include +#include unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) { @@ -36,9 +37,21 @@ unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) break; } case IEE_WRITE_CR4: { + unsigned long bits_changed = 0; + val = va_arg(pArgs, u64); val &= ~(X86_CR4_SMEP); +set_register_cr4: asm volatile("mov %0,%%cr4" : "+r" (val) : : "memory"); + if (static_branch_likely(&cr_pinning)) { + if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) { + bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits; + val = (val & ~cr4_pinned_mask) | cr4_pinned_bits; + goto set_register_cr4; + } + WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n", + bits_changed); + } break; } case IEE_LOAD_IDT: { @@ -104,12 +117,12 @@ void __init iee_sip_init(void) _iee_set_kernel_upage((unsigned long)__va(__pa(addr))); } iee_init_done = true; - /* Map .iee.data as RO pages */ + /* Map .iee.si_data as IEE SIP data pages. */ start = (unsigned long)__iee_si_data_start; end = (unsigned long)__iee_si_data_end; num_pages = (end - start) / PAGE_SIZE; - set_memory_ro(start, num_pages); + set_iee_pages(start, num_pages, IEE_SIP_DATA); /* All initialization is done. Do some simple tests. */ pr_info("IEE: testing iee_exec_entry si_test..."); iee_sip_test(); -} \ No newline at end of file +} diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 32f1769a85dd5..a32e06859fee0 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -59,6 +59,10 @@ #ifdef CONFIG_IEE #include #endif +#ifdef CONFIG_IEE_SIP +#include +#include +#endif #include "mm_internal.h" #include "ident_map.c" @@ -1408,7 +1412,11 @@ int __init deferred_page_init_max_threads(const struct cpumask *node_cpumask) } #endif +#ifdef CONFIG_IEE_SIP +int kernel_set_to_readonly __iee_si_data; +#else int kernel_set_to_readonly; +#endif void mark_rodata_ro(void) { @@ -1418,12 +1426,17 @@ void mark_rodata_ro(void) unsigned long text_end = PFN_ALIGN(_etext); unsigned long rodata_end = PFN_ALIGN(__end_rodata); unsigned long all_end; +#ifdef CONFIG_IEE_SIP + int kernel_set_to_readonly_value = 1; +#endif printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", (end - start) >> 10); set_memory_ro(start, (end - start) >> PAGE_SHIFT); +#ifndef CONFIG_IEE_SIP kernel_set_to_readonly = 1; +#endif /* * The rodata/data/bss/brk section (but not the kernel text!) @@ -1456,6 +1469,10 @@ void mark_rodata_ro(void) (void *)rodata_end, (void *)_sdata); debug_checkwx(); +#ifdef CONFIG_IEE_SIP + iee_memcpy(&kernel_set_to_readonly, &kernel_set_to_readonly_value, + sizeof(int)); +#endif } /* From 0e93f9da831f7779ff4fd5b0c9b54c4239632493 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 25 Jun 2026 18:36:41 +0800 Subject: [PATCH 03/26] HAOC/IEE_PTRP: Upgrade: prepare for haoc-4.0 Update IEE_PTRP with the HAOC4.0 token block layout and token lookup by task slab page index. - Add the 64-byte token block layout and derive token allocation order from the task slab order. - Add iee_get_task_token() on x86 and arm64 so token users locate the token slot by slab page index. - Switch token gate handlers, the SIP TTBR0 check, and scheduler PGD verification to the new token lookup. - Reuse Deepin's existing page allocation path on arm64, but free token pages with the derived token order. Signed-off-by: Liu Zhehui --- arch/arm64/include/asm/haoc/iee-token.h | 24 +++++++++++-- arch/arm64/kernel/haoc/credp/credp.c | 12 +++---- arch/arm64/kernel/haoc/iee/iee-func.c | 30 +++++++++++++++++ arch/arm64/kernel/haoc/iee/iee-si.c | 2 +- arch/arm64/kernel/haoc/iee/iee-token.c | 45 +++++++++++++++++++++---- arch/x86/include/asm/haoc/iee-cred.h | 2 +- arch/x86/include/asm/haoc/iee-token.h | 20 +++++++++-- arch/x86/include/asm/haoc/iee.h | 8 +++++ arch/x86/kernel/haoc/credp/credp.c | 21 ++++++++---- arch/x86/kernel/haoc/iee/iee-func.c | 9 ++--- arch/x86/kernel/haoc/iee/iee-token.c | 16 +++++---- arch/x86/kernel/haoc/iee/iee.c | 20 +++++++++++ kernel/sched/core.c | 2 +- security/keys/process_keys.c | 20 +++++------ 14 files changed, 182 insertions(+), 49 deletions(-) diff --git a/arch/arm64/include/asm/haoc/iee-token.h b/arch/arm64/include/asm/haoc/iee-token.h index 440fcd6e395de..668437622e369 100644 --- a/arch/arm64/include/asm/haoc/iee-token.h +++ b/arch/arm64/include/asm/haoc/iee-token.h @@ -3,7 +3,9 @@ #define _LINUX_IEE_TOKEN_H #include +#include +#ifdef CONFIG_IEE_PTRP extern struct kmem_cache *task_struct_cachep; extern void __init iee_prepare_init_task_token(void); @@ -13,6 +15,20 @@ extern void iee_set_token_page_invalid(unsigned long token_addr, unsigned long token_page, unsigned int order); extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, unsigned int order); +extern void iee_free_task_token_slab(struct kmem_cache *s, struct slab *slab, + unsigned int order); + +#ifndef IEE_TOKEN_BLOCK_SIZE +#define IEE_TOKEN_BLOCK_SIZE 64 +#endif + +#ifndef IEE_TOKEN_ORDER +#define IEE_TOKEN_ORDER(task_order) \ + order_base_2(((1U << (task_order)) * IEE_TOKEN_BLOCK_SIZE) / PAGE_SIZE) +#endif + +extern struct task_token *iee_get_task_token(struct task_struct *task); +#endif /* CONFIG_IEE_PTRP */ #ifdef CONFIG_IEE struct task_token { @@ -27,22 +43,23 @@ struct task_token { }; #endif /* CONFIG_IEE */ +#ifdef CONFIG_IEE_PTRP #ifndef CONFIG_IEE_SIP #include -static inline void iee_verify_token_pgd(struct task_struct *tsk) +static inline void iee_verify_pgd(struct task_struct *tsk) { struct task_token *token; if (tsk == &init_task) return; - token = (struct task_token *)__addr_to_iee(tsk); + token = (struct task_token *)iee_get_task_token(tsk); if (token->pgd != tsk->mm->pgd) panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); } #else -static inline void iee_verify_token_pgd(struct task_struct *tsk) +static inline void iee_verify_pgd(struct task_struct *tsk) { } @@ -67,5 +84,6 @@ static inline void iee_validate_token(struct task_struct *tsk) { iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); } +#endif /* CONFIG_IEE_PTRP */ #endif diff --git a/arch/arm64/kernel/haoc/credp/credp.c b/arch/arm64/kernel/haoc/credp/credp.c index aeec727f9a58c..1d4517f34e8da 100644 --- a/arch/arm64/kernel/haoc/credp/credp.c +++ b/arch/arm64/kernel/haoc/credp/credp.c @@ -211,7 +211,7 @@ void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred struct cred *_new = __ptr_to_iee(new); //struct task_struct *task = current; /* Would verify this field in commit_cred. */ - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = iee_get_task_token(current); if (!uid_eq(current_uid(), init_cred.uid)) panic("IEE: calling prepare_kernel_cred by unprivileged process."); @@ -228,7 +228,7 @@ void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred void _iee_init_copied_cred(unsigned long __unused, struct task_struct *new_task, struct cred *new) { - struct task_token *old_task_token = (struct task_token *)__addr_to_iee(current); + struct task_token *old_task_token = iee_get_task_token(current); //struct task_token *new_task_token = (struct task_token *)__addr_to_iee(new_task); if (old_task_token->new_cred != new) @@ -243,7 +243,7 @@ void _iee_init_copied_cred(unsigned long __unused, void _iee_commit_creds(unsigned long __unused, const struct cred *new) { struct task_struct *task = current; - struct task_token *token = (struct task_token *)__addr_to_iee(task); + struct task_token *token = iee_get_task_token(task); if (token->new_cred != new) panic("IEE: (%s) Invalid cred 0x%llx. token->new_cred 0x%llx", @@ -257,7 +257,7 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) void _iee_abort_cred(unsigned long __unused, const struct cred *cred) { - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = iee_get_task_token(current); token->new_cred = NULL; } @@ -267,7 +267,7 @@ void __iee_code _iee_fill_cred_for_session_keyring(unsigned long __unused, const struct cred *old) { struct cred *_new = __ptr_to_iee(new); - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = iee_get_task_token(current); token->new_cred = new; _new->uid = old->uid; @@ -299,7 +299,7 @@ void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = iee_get_task_token(current); /* Get old cred inside IEE is safer. */ const struct cred *old = current_cred(); /* Would verify this field in commit_cred. */ diff --git a/arch/arm64/kernel/haoc/iee/iee-func.c b/arch/arm64/kernel/haoc/iee/iee-func.c index a625bac775d96..ab600a78a223e 100644 --- a/arch/arm64/kernel/haoc/iee/iee-func.c +++ b/arch/arm64/kernel/haoc/iee/iee-func.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -436,6 +437,16 @@ unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) return order; } +bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, + unsigned int order) +{ +#ifdef CONFIG_IEE_PTRP + if (s == task_struct_cachep) + iee_free_task_token_slab(s, slab, order); +#endif + return false; +} + void iee_set_min_partial(struct kmem_cache *s) { if (!haoc_enabled) { @@ -478,3 +489,22 @@ asmlinkage void notrace iee_bad_mode(struct pt_regs *regs, int reason, panic("bad mode"); } + +#ifdef CONFIG_IEE_PTRP +struct task_token *iee_get_task_token(struct task_struct *task) +{ + unsigned long slab_addr; + unsigned long task_addr; + unsigned int index; + + if (!__is_lm_address((u64)task)) + return (struct task_token *)__kimg_to_iee(task); + + slab_addr = (unsigned long)page_to_virt(virt_to_head_page(task)); + task_addr = (unsigned long)page_to_virt(virt_to_page(task)); + index = (task_addr - slab_addr) / PAGE_SIZE; + + return (struct task_token *)(__virt_to_iee(slab_addr) + + index * IEE_TOKEN_BLOCK_SIZE); +} +#endif diff --git a/arch/arm64/kernel/haoc/iee/iee-si.c b/arch/arm64/kernel/haoc/iee/iee-si.c index 9d85d880eb37a..2a0925a3d2725 100644 --- a/arch/arm64/kernel/haoc/iee/iee-si.c +++ b/arch/arm64/kernel/haoc/iee/iee-si.c @@ -27,7 +27,7 @@ static inline void iee_si_check_ttbr0(void) { u64 old_ttbr0 = read_sysreg(ttbr0_el1); u64 old_phys = (old_ttbr0 & PAGE_MASK) & ~TTBR_ASID_MASK; - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = (struct task_token *)iee_get_task_token(current); /* Phys in TTBR0 shall be the same with current->mm->pgd. */ if (!(current == &init_task) && token->pgd && old_phys != iee_si_reserved_pg_dir) { diff --git a/arch/arm64/kernel/haoc/iee/iee-token.c b/arch/arm64/kernel/haoc/iee/iee-token.c index 0b341f22c9134..2bbe40e425f1c 100644 --- a/arch/arm64/kernel/haoc/iee/iee-token.c +++ b/arch/arm64/kernel/haoc/iee/iee-token.c @@ -108,6 +108,18 @@ static inline void iee_set_token(unsigned long token_addr, unsigned long token_p flush_tlb_kernel_range(token_pages, (token_pages + (PAGE_SIZE << order))); } +static unsigned long iee_token_page(unsigned long token_addr) +{ + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, token_addr); + p4d_t *p4dp = p4d_offset(pgdp, token_addr); + pud_t *pudp = pud_offset(p4dp, token_addr); + pmd_t *pmdp = pmd_offset(pudp, token_addr); + pte_t *ptep = pte_offset_kernel(pmdp, token_addr); + + return (unsigned long)page_address(pte_page(READ_ONCE(*ptep))); +} + /* * iee_set_token_page_valid() - After allocated task token pages, map them to the * corresponding IEE addresses, and enforce RO protection on their linear mappings. @@ -143,8 +155,11 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, return slab; struct folio *folio = slab_folio(slab); + unsigned int task_order = order; + unsigned int token_order = IEE_TOKEN_ORDER(task_order); unsigned long token_addr = __slab_to_iee(slab); - unsigned long alloc_token = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + unsigned long alloc_token = + __get_free_pages(GFP_KERNEL | __GFP_ZERO, token_order); /* Allocation of task_struct and token pages must be done at the same time. */ if (!alloc_token) { @@ -156,15 +171,33 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, /* Make the mapping reset visible before clearing the flag */ smp_wmb(); __folio_clear_slab(folio); - __free_pages((struct page *)folio, order); + __free_pages((struct page *)folio, task_order); return NULL; } /* Map allocated token pages to token addresses. */ - iee_set_token_page_valid(token_addr, alloc_token, order); + iee_set_token_page_valid(token_addr, alloc_token, token_order); return slab; } +void iee_free_task_token_slab(struct kmem_cache *s, struct slab *slab, + unsigned int order) +{ + unsigned int token_order; + unsigned long token_addr; + unsigned long token_page; + + if (!slab || s != task_struct_cachep) + return; + + token_order = IEE_TOKEN_ORDER(order); + token_addr = __slab_to_iee(slab); + token_page = iee_token_page(token_addr); + + iee_set_token_page_invalid(token_addr, token_page, token_order); + free_pages(token_page, token_order); +} + void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk) { /* Do nothing for now. Wait for later update. */ @@ -173,14 +206,14 @@ void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk) void __iee_code _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)iee_get_task_token(tsk); token->pgd = pgd; } void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)iee_get_task_token(tsk); if (token->valid) pr_err("IEE: validate token for multiple times."); @@ -189,7 +222,7 @@ void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct * void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)iee_get_task_token(tsk); token->pgd = NULL; token->valid = false; diff --git a/arch/x86/include/asm/haoc/iee-cred.h b/arch/x86/include/asm/haoc/iee-cred.h index e33cab7449c1c..8da042a49d86d 100644 --- a/arch/x86/include/asm/haoc/iee-cred.h +++ b/arch/x86/include/asm/haoc/iee-cred.h @@ -305,4 +305,4 @@ static void __maybe_unused iee_set_cred_security(struct cred *cred, void *securi } #endif -#endif \ No newline at end of file +#endif diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h index 53e75396577d3..771628ddab444 100644 --- a/arch/x86/include/asm/haoc/iee-token.h +++ b/arch/x86/include/asm/haoc/iee-token.h @@ -3,11 +3,13 @@ #define _LINUX_IEE_TOKEN_H #include #include +#include #include #include extern unsigned long long iee_rw_gate(int flag, ...); +#ifdef CONFIG_IEE_PTRP extern struct kmem_cache *task_struct_cachep; extern void iee_set_token_page_valid(unsigned long token, unsigned long new, @@ -19,6 +21,16 @@ extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, unsigned int order); +#ifndef IEE_TOKEN_BLOCK_SIZE +#define IEE_TOKEN_BLOCK_SIZE 64 +#endif + +#ifndef IEE_TOKEN_ORDER +#define IEE_TOKEN_ORDER(task_order) \ + order_base_2(((1U << (task_order)) * IEE_TOKEN_BLOCK_SIZE) / PAGE_SIZE) +#endif +#endif /* CONFIG_IEE_PTRP */ + #ifdef CONFIG_IEE struct task_token { pgd_t *pgd; /* Logical VA */ @@ -32,14 +44,15 @@ struct task_token { }; #endif /* CONFIG_IEE */ -static inline void iee_verify_token_pgd(struct task_struct *tsk) +#ifdef CONFIG_IEE_PTRP +static inline void iee_verify_pgd(struct task_struct *tsk) { struct task_token *token; if (tsk == &init_task) return; - token = (struct task_token *)__addr_to_iee(tsk); + token = (struct task_token *)__addr_to_token(tsk); if (token->pgd != tsk->mm->pgd) panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); @@ -59,5 +72,6 @@ static inline void iee_validate_token(struct task_struct *tsk) { iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); } +#endif /* CONFIG_IEE_PTRP */ -#endif \ No newline at end of file +#endif diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h index 432d2bc9d27e2..25cfba69d4c3f 100644 --- a/arch/x86/include/asm/haoc/iee.h +++ b/arch/x86/include/asm/haoc/iee.h @@ -40,6 +40,9 @@ extern bool haoc_init_done; __val = (u64)__kimg_to_iee((u64)x); \ __val; \ }) +#ifdef CONFIG_IEE_PTRP +#define __addr_to_token(x) iee_get_task_token(x) +#endif #define IEE_DATA_ORDER (PMD_SHIFT - PAGE_SHIFT) #define IEE_STACK_ORDER 0 @@ -54,4 +57,9 @@ extern void iee_early_init(void); extern void iee_init(void); extern void iee_post_init(void); extern bool iee_init_done; +#ifdef CONFIG_IEE_PTRP +struct task_struct; +struct task_token; +extern struct task_token *iee_get_task_token(struct task_struct *task); +#endif #endif diff --git a/arch/x86/kernel/haoc/credp/credp.c b/arch/x86/kernel/haoc/credp/credp.c index 7e07cfdec5ff6..1b30bf3774855 100644 --- a/arch/x86/kernel/haoc/credp/credp.c +++ b/arch/x86/kernel/haoc/credp/credp.c @@ -11,6 +11,15 @@ extern struct cred init_cred; +static inline struct task_token *credp_task_token(struct task_struct *task) +{ +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) + return (struct task_token *)__addr_to_token(task); +#endif + return (struct task_token *)__addr_to_iee(task); +} + void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) { *((struct rcu_head **)(&(cred->rcu.func))) = rcu; @@ -170,7 +179,7 @@ void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = credp_task_token(current); /* Get old cred inside IEE is safer. */ const struct cred *old = current_cred(); /* Would verify this field in commit_cred. */ @@ -189,7 +198,7 @@ void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); /* Would verify this field in commit_cred. */ - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = credp_task_token(current); if (!uid_eq(current_uid(), init_cred.uid)) panic("IEE: calling prepare_kernel_cred by unprivileged process."); @@ -206,7 +215,7 @@ void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, void _iee_init_copied_cred(unsigned long __unused, struct task_struct *new_task, struct cred *new) { - struct task_token *old_task_token = (struct task_token *)__addr_to_iee(current); + struct task_token *old_task_token = credp_task_token(current); if (old_task_token->new_cred != new) panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", __func__, @@ -220,7 +229,7 @@ void _iee_init_copied_cred(unsigned long __unused, void _iee_commit_creds(unsigned long __unused, const struct cred *new) { struct task_struct *task = current; - struct task_token *token = (struct task_token *)__addr_to_iee(task); + struct task_token *token = credp_task_token(task); if (token->new_cred != new) panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); @@ -233,7 +242,7 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) void _iee_abort_cred(unsigned long __unused, const struct cred *cred) { - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = credp_task_token(current); token->new_cred = NULL; } @@ -241,7 +250,7 @@ void _iee_abort_cred(unsigned long __unused, const struct cred *cred) void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, const struct cred *old) { - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = credp_task_token(current); token->new_cred = new; new->uid = old->uid; diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index aa3a9de793803..dbea682f4bd86 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -67,11 +67,12 @@ static void iee_free_task_struct_slab(struct work_struct *work) container_of(work, struct iee_free_slab_work, work); struct slab *slab = iee_free_slab_work->slab; struct folio *folio = slab_folio(slab); - unsigned int order = folio_order(folio); + unsigned int task_order = folio_order(folio); + unsigned int token_order = IEE_TOKEN_ORDER(task_order); unsigned long token = __slab_to_iee(slab); - // Free token. - iee_set_token_page_invalid(token, 0, order); - __free_pages(&folio->page, order); + + iee_set_token_page_invalid(token, 0, token_order); + __free_pages(&folio->page, task_order); kfree(iee_free_slab_work); } #endif diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c index b39ba03d9c565..a213c21df24b4 100644 --- a/arch/x86/kernel/haoc/iee/iee-token.c +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -91,9 +91,11 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, return slab; struct folio *folio = slab_folio(slab); + unsigned int task_order = order; + unsigned int token_order = IEE_TOKEN_ORDER(task_order); unsigned long token_addr = __slab_to_iee(slab); unsigned long alloc_token = - __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + __get_free_pages(GFP_KERNEL | __GFP_ZERO, token_order); /* Allocation of task_struct and token pages must be done at the same time. */ if (!alloc_token) { @@ -105,26 +107,26 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, /* Make the mapping reset visible before clearing the flag */ smp_wmb(); __folio_clear_slab(folio); - __free_pages((struct page *)folio, order); + __free_pages((struct page *)folio, task_order); return NULL; } /* Map allocated token pages to token addresses. */ - iee_set_token_page_valid(token_addr, alloc_token, order); + iee_set_token_page_valid(token_addr, alloc_token, token_order); return slab; } void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)__addr_to_token(tsk); token->pgd = pgd; } void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)__addr_to_token(tsk); token->pgd = NULL; token->valid = false; @@ -132,9 +134,9 @@ void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)__addr_to_token(tsk); if (token->valid) pr_err("IEE: validate token for multiple times."); token->valid = true; -} \ No newline at end of file +} diff --git a/arch/x86/kernel/haoc/iee/iee.c b/arch/x86/kernel/haoc/iee/iee.c index 35c54c3352ac3..55e50c0b0cb79 100644 --- a/arch/x86/kernel/haoc/iee/iee.c +++ b/arch/x86/kernel/haoc/iee/iee.c @@ -8,6 +8,10 @@ */ #include +#ifdef CONFIG_IEE_PTRP +#include +#endif +#include void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n) { @@ -30,3 +34,19 @@ unsigned long _iee_test_and_clear_bit(unsigned long __unused, long nr, unsigned instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); return arch_test_and_clear_bit(nr, addr); } + +#ifdef CONFIG_IEE_PTRP +struct task_token *iee_get_task_token(struct task_struct *task) +{ + unsigned long slab_addr; + unsigned long task_addr; + unsigned int index; + + slab_addr = (unsigned long)page_to_virt(virt_to_head_page(task)); + task_addr = (unsigned long)page_to_virt(virt_to_page(task)); + index = (task_addr - slab_addr) / PAGE_SIZE; + + return (struct task_token *)((unsigned long)__phys_to_iee(__pa(slab_addr)) + + index * IEE_TOKEN_BLOCK_SIZE); +} +#endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 779aa2efa09db..3b1530044f4f6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5210,7 +5210,7 @@ context_switch(struct rq *rq, struct task_struct *prev, */ #ifdef CONFIG_IEE_PTRP if(haoc_enabled) - iee_verify_token_pgd(next); + iee_verify_pgd(next); #endif switch_mm_irqs_off(prev->active_mm, next->mm, next); lru_gen_use_mm(next->mm); diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index 8517970bb56ce..1993bef2f2f1a 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -951,22 +951,20 @@ static void fill_cred_for_session_keyring(struct cred *new, new->process_keyring = key_get(old->process_keyring); } -/* - * Replace a process's session keyring on behalf of one of its children when - * the target process is about to resume userspace execution. - */ void key_change_session_keyring(struct callback_head *twork) { const struct cred *old = current_cred(); - #ifdef CONFIG_CREDP - struct cred *new =NULL; - if(haoc_enabled) + +#ifdef CONFIG_CREDP + struct cred *new = NULL; + + if (haoc_enabled) new = *(struct cred **)(twork + 1); else new = container_of(twork, struct cred, rcu); - #else +#else struct cred *new = container_of(twork, struct cred, rcu); - #endif +#endif if (unlikely(current->flags & PF_EXITING)) { put_cred(new); @@ -980,11 +978,11 @@ void key_change_session_keyring(struct callback_head *twork) return; } - #ifdef CONFIG_CREDP +#ifdef CONFIG_CREDP if (haoc_enabled) iee_fill_cred_for_session_keyring(new, old); else - #endif +#endif fill_cred_for_session_keyring(new, old); security_transfer_creds(new, old); From 502e51da2df8a540f55ccf08ad3c9b3437f1d23b Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Fri, 26 Jun 2026 00:02:25 +0800 Subject: [PATCH 04/26] HAOC/CREDP: Upgrade: prepare for haoc-4.0 Update CREDP with typed cred pages, task token tracking, and gated subjective cred updates. - Mark static and slab-backed cred pages as IEE_CRED on x86 and arm64, and restore bitmap/protection state when slabs are freed. - Add curr_cred to task tokens, and initialize or clear it when tokens are validated or invalidated. - Verify CREDP writes against the IEE_CRED bitmap type and the current task token before mutating protected cred fields. - Move CREDP token state from task-object IEE aliases to the IEE_PTRP token lookup path. - Route override_creds() and revert_creds() through CREDP gates, and update only the subjective cred pointer. Signed-off-by: Liu Zhehui --- arch/arm64/include/asm/haoc/haoc-bitmap.h | 1 + arch/arm64/include/asm/haoc/haoc-def.h | 2 + arch/arm64/include/asm/haoc/haoc.h | 4 + arch/arm64/include/asm/haoc/iee-cred.h | 70 ++++++- arch/arm64/include/asm/haoc/iee-init.h | 4 + arch/arm64/include/asm/haoc/iee-token.h | 3 +- arch/arm64/kernel/haoc/credp/credp.c | 243 +++++++++++++++++----- arch/arm64/kernel/haoc/haoc-bitmap.c | 6 + arch/arm64/kernel/haoc/haoc.c | 2 + arch/arm64/kernel/haoc/iee/iee-token.c | 8 + arch/arm64/kernel/vmlinux.lds.S | 3 + arch/x86/include/asm/haoc/haoc-bitmap.h | 1 + arch/x86/include/asm/haoc/haoc-def.h | 2 + arch/x86/include/asm/haoc/haoc.h | 7 +- arch/x86/include/asm/haoc/iee-cred.h | 79 ++++++- arch/x86/include/asm/haoc/iee-token.h | 3 +- arch/x86/kernel/haoc/credp/credp.c | 198 +++++++++++++++--- arch/x86/kernel/haoc/haoc.c | 2 + arch/x86/kernel/haoc/iee/iee-init.c | 6 + arch/x86/kernel/haoc/iee/iee-token.c | 8 + arch/x86/kernel/vmlinux.lds.S | 3 + kernel/cred.c | 14 ++ mm/slub.c | 17 +- security/keys/process_keys.c | 70 +++---- 24 files changed, 624 insertions(+), 132 deletions(-) diff --git a/arch/arm64/include/asm/haoc/haoc-bitmap.h b/arch/arm64/include/asm/haoc/haoc-bitmap.h index 3e2ad505877fc..75c855d50a833 100644 --- a/arch/arm64/include/asm/haoc/haoc-bitmap.h +++ b/arch/arm64/include/asm/haoc/haoc-bitmap.h @@ -15,6 +15,7 @@ enum HAOC_BITMAP_TYPE { IEE_KEY, IEE_SELINUX, IEE_VARP, + IEE_CRED, }; #include diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index 6f26d88fe27e3..afd8529eb9620 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -29,6 +29,8 @@ enum { IEE_OP_COMMIT_CRED, IEE_OP_ABORT_CRED, IEE_OP_FILL_SESSION_KEYRING_CRED, + IEE_OP_OVERRIDE_CRED, + IEE_OP_REVERT_CRED, IEE_OP_SET_CRED_UID, IEE_OP_SET_CRED_GID, IEE_OP_SET_CRED_SUID, diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index c06f1d6b094ba..8bb0dcfd11539 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -42,6 +42,10 @@ void _iee_fill_cred_for_session_keyring(unsigned long __unused, void _iee_init_copied_cred(unsigned long iee_offset, struct task_struct *new_task, struct cred *new); void _iee_commit_creds(unsigned long iee_offset, const struct cred *new); +void _iee_override_creds(unsigned long iee_offset, const struct cred *new); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old); +void _iee_revert_creds(unsigned long iee_offset, const struct cred *old); void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); diff --git a/arch/arm64/include/asm/haoc/iee-cred.h b/arch/arm64/include/asm/haoc/iee-cred.h index ca40e034ead5e..9c14fed8da9a3 100644 --- a/arch/arm64/include/asm/haoc/iee-cred.h +++ b/arch/arm64/include/asm/haoc/iee-cred.h @@ -6,9 +6,65 @@ #include #include #include +#include extern unsigned long long iee_rw_gate(int flag, ...); +static inline void iee_detect_cred_uaf(struct cred *cred, const char *caller) +{ + if (!haoc_enabled || !iee_init_done || !haoc_bitmap_ready) + return; + + if (atomic_long_read(&cred->usage) != 0) + panic("IEE: (%s) Cred usage %ld.", caller, + atomic_long_read(&cred->usage)); +} + +#ifdef CONFIG_IEE_PTRP +static inline void iee_verify_cred(void) +{ + struct task_struct *curr_task; + struct task_token *token; + const struct cred *curr_cred; + + if (!haoc_enabled || !iee_init_done || !haoc_bitmap_ready) + return; + + curr_task = current; + if (curr_task == &init_task) + return; + + token = (struct task_token *)iee_get_task_token(curr_task); + curr_cred = current_cred(); + if (!token->valid) + panic("IEE: (%s) Invalid Token.", __func__); + if (token->curr_cred != curr_cred) + panic("IEE: (%s) Task cred corrupted. token cred 0x%llx, task cred 0x%llx", + __func__, (u64)token->curr_cred, (u64)curr_cred); +} + +static inline void iee_verify_update_cred(struct cred *new) +{ + struct task_struct *curr_task; + struct task_token *token; + + if (!haoc_enabled || !iee_init_done || !haoc_bitmap_ready) + return; + + iee_verify_cred(); + curr_task = current; + if (curr_task == &init_task) + return; + + token = (struct task_token *)iee_get_task_token(curr_task); + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); +} +#else +static inline void iee_verify_cred(void) { } +static inline void iee_verify_update_cred(struct cred *new) { } +#endif + static void __maybe_unused iee_copy_cred(struct cred *new) { // if(!haoc_enabled) @@ -37,7 +93,7 @@ static void __maybe_unused iee_abort_creds(struct cred *cred) } static void __maybe_unused iee_fill_cred_for_session_keyring(struct cred *new, - const struct cred *old) + const struct cred *old) { iee_rw_gate(IEE_OP_FILL_SESSION_KEYRING_CRED, new, old); } @@ -46,6 +102,16 @@ static void __maybe_unused iee_commit_creds(const struct cred *new) { iee_rw_gate(IEE_OP_COMMIT_CRED, new); } + +static void __maybe_unused iee_override_creds(const struct cred *new) +{ + iee_rw_gate(IEE_OP_OVERRIDE_CRED, new); +} + +static void __maybe_unused iee_revert_creds(const struct cred *old) +{ + iee_rw_gate(IEE_OP_REVERT_CRED, old); +} static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) { if(!haoc_enabled) @@ -311,4 +377,4 @@ static void __maybe_unused iee_set_cred_security(struct cred *cred, void *securi } #endif -#endif \ No newline at end of file +#endif diff --git a/arch/arm64/include/asm/haoc/iee-init.h b/arch/arm64/include/asm/haoc/iee-init.h index 31467199c11a4..6de6fea0ac425 100644 --- a/arch/arm64/include/asm/haoc/iee-init.h +++ b/arch/arm64/include/asm/haoc/iee-init.h @@ -16,5 +16,9 @@ extern char iee_init_data_begin[]; extern char iee_init_data_end[]; +#ifdef CONFIG_CREDP +extern char __iee_cred_data_start[]; +extern char __iee_cred_data_end[]; +#endif #endif diff --git a/arch/arm64/include/asm/haoc/iee-token.h b/arch/arm64/include/asm/haoc/iee-token.h index 668437622e369..4bde51ae4926a 100644 --- a/arch/arm64/include/asm/haoc/iee-token.h +++ b/arch/arm64/include/asm/haoc/iee-token.h @@ -38,7 +38,8 @@ struct task_token { bool valid; void *kernel_stack; /* VA */ #ifdef CONFIG_CREDP - struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *curr_cred; /* The current subjective credentials. */ #endif }; #endif /* CONFIG_IEE */ diff --git a/arch/arm64/kernel/haoc/credp/credp.c b/arch/arm64/kernel/haoc/credp/credp.c index 1d4517f34e8da..5ca7fc7968de5 100644 --- a/arch/arm64/kernel/haoc/credp/credp.c +++ b/arch/arm64/kernel/haoc/credp/credp.c @@ -5,11 +5,33 @@ #include #include #include +#include extern struct cred init_cred; +static inline void iee_verify_cred_type(const struct cred *cred) +{ + if (!iee_init_done || !haoc_bitmap_ready) + return; + + iee_verify_type((unsigned long)cred, IEE_CRED, "cred"); +} + +static inline void iee_verify_current_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_cred(); +} + +static inline void iee_verify_new_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_update_cred(cred); +} + void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); *((struct rcu_head **)(&(cred->rcu.func))) = rcu; } @@ -17,6 +39,7 @@ void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->security = security; } @@ -24,6 +47,7 @@ void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, struct cred *cred, int flag, int nr) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); switch (flag) { case AT_ADD: { @@ -43,6 +67,7 @@ bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); atomic_long_set(&cred->usage, i); } @@ -50,6 +75,7 @@ void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->non_rcu = non_rcu; } @@ -57,6 +83,8 @@ void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, struct key *session_keyring) { + iee_verify_cred_type(cred); + iee_verify_cred(); cred = __ptr_to_iee(cred); cred->session_keyring = session_keyring; } @@ -64,6 +92,7 @@ void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, struct key *process_keyring) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->process_keyring = process_keyring; } @@ -71,6 +100,7 @@ void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, struct key *thread_keyring) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->thread_keyring = thread_keyring; } @@ -78,6 +108,7 @@ void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, struct key *request_key_auth) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->request_key_auth = request_key_auth; } @@ -85,6 +116,7 @@ void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->jit_keyring = jit_keyring; } @@ -92,6 +124,7 @@ void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, kernel_cap_t cap_inheritable) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_inheritable = cap_inheritable; } @@ -99,6 +132,7 @@ void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, kernel_cap_t cap_permitted) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_permitted = cap_permitted; } @@ -106,6 +140,7 @@ void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, kernel_cap_t cap_effective) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_effective = cap_effective; } @@ -113,6 +148,7 @@ void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_bset = cap_bset; } @@ -120,6 +156,7 @@ void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_ambient = cap_ambient; } @@ -127,6 +164,7 @@ void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, void __iee_code _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, unsigned int securebits) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->securebits = securebits; } @@ -134,6 +172,7 @@ void __iee_code _iee_set_cred_securebits(unsigned long __unused, void __iee_code _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, struct group_info *group_info) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->group_info = group_info; } @@ -141,6 +180,7 @@ void __iee_code _iee_set_cred_group_info(unsigned long __unused, void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->ucounts = ucounts; } @@ -148,6 +188,7 @@ void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, struct user_namespace *user_ns) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->user_ns = user_ns; } @@ -155,6 +196,7 @@ void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->user = user; } @@ -162,6 +204,7 @@ void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->fsgid = fsgid; } @@ -169,6 +212,7 @@ void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->fsuid = fsuid; } @@ -176,6 +220,7 @@ void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->egid = egid; } @@ -183,6 +228,7 @@ void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->euid = euid; } @@ -190,6 +236,7 @@ void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->sgid = sgid; } @@ -197,6 +244,7 @@ void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->suid = suid; } @@ -209,14 +257,23 @@ void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - //struct task_struct *task = current; - /* Would verify this field in commit_cred. */ - struct task_token *token = iee_get_task_token(current); + + iee_verify_cred(); + iee_verify_cred_type(old); + iee_verify_cred_type(new); if (!uid_eq(current_uid(), init_cred.uid)) panic("IEE: calling prepare_kernel_cred by unprivileged process."); - token->new_cred = new; +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -228,14 +285,26 @@ void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred void _iee_init_copied_cred(unsigned long __unused, struct task_struct *new_task, struct cred *new) { - struct task_token *old_task_token = iee_get_task_token(current); - //struct task_token *new_task_token = (struct task_token *)__addr_to_iee(new_task); - - if (old_task_token->new_cred != new) - panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", __func__, - (u64)old_task_token->new_cred, (u64)new); - /* Update token info of new task by current task token. */ - old_task_token->new_cred = NULL; + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *old_task_token = + (struct task_token *)iee_get_task_token(current); + struct task_token *new_task_token = + (struct task_token *)iee_get_task_token(new_task); + + if (!old_task_token->valid || !new_task_token->valid) + panic("IEE: (%s) Invalid Token.", __func__); + if (old_task_token->new_cred != new) + panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", + __func__, (u64)old_task_token->new_cred, (u64)new); + /* Update token info of new task by current task token. */ + old_task_token->new_cred = NULL; + new_task_token->curr_cred = new; + } +#endif new_task->cred = new_task->real_cred = new; } @@ -243,13 +312,22 @@ void _iee_init_copied_cred(unsigned long __unused, void _iee_commit_creds(unsigned long __unused, const struct cred *new) { struct task_struct *task = current; - struct task_token *token = iee_get_task_token(task); - if (token->new_cred != new) - panic("IEE: (%s) Invalid cred 0x%llx. token->new_cred 0x%llx", - __func__, (u64)new, (u64)token->new_cred); - /* task->cred shall be updated once. */ - token->new_cred = NULL; + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(task); + + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx. token->new_cred 0x%llx", + __func__, (u64)new, (u64)token->new_cred); + /* task->cred shall be updated once. */ + token->new_cred = NULL; + token->curr_cred = new; + } +#endif rcu_assign_pointer(task->real_cred, new); rcu_assign_pointer(task->cred, new); @@ -257,53 +335,78 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) void _iee_abort_cred(unsigned long __unused, const struct cred *cred) { - struct task_token *token = iee_get_task_token(current); + iee_verify_cred(); + iee_verify_cred_type(cred); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); - token->new_cred = NULL; + token->new_cred = NULL; + } +#endif } -void __iee_code _iee_fill_cred_for_session_keyring(unsigned long __unused, - struct cred *new, - const struct cred *old) +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old) { - struct cred *_new = __ptr_to_iee(new); - struct task_token *token = iee_get_task_token(current); - - token->new_cred = new; - _new->uid = old->uid; - _new->euid = old->euid; - _new->suid = old->suid; - _new->fsuid = old->fsuid; - _new->gid = old->gid; - _new->egid = old->egid; - _new->sgid = old->sgid; - _new->fsgid = old->fsgid; - _new->user = get_uid(old->user); - _new->ucounts = old->ucounts; - _new->user_ns = get_user_ns(old->user_ns); - _new->group_info = get_group_info(old->group_info); - - _new->securebits = old->securebits; - _new->cap_inheritable = old->cap_inheritable; - _new->cap_permitted = old->cap_permitted; - _new->cap_effective = old->cap_effective; - _new->cap_ambient = old->cap_ambient; - _new->cap_bset = old->cap_bset; - - _new->jit_keyring = old->jit_keyring; - _new->thread_keyring = key_get(old->thread_keyring); - _new->process_keyring = key_get(old->process_keyring); + iee_verify_cred_type(new); + iee_verify_cred_type(old); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + token->new_cred = new; + } +#endif + + new = __ptr_to_iee(new); + new->uid = old->uid; + new->euid = old->euid; + new->suid = old->suid; + new->fsuid = old->fsuid; + new->gid = old->gid; + new->egid = old->egid; + new->sgid = old->sgid; + new->fsgid = old->fsgid; + new->user = get_uid(old->user); + new->ucounts = old->ucounts; + new->user_ns = get_user_ns(old->user_ns); + new->group_info = get_group_info(old->group_info); + + new->securebits = old->securebits; + new->cap_inheritable = old->cap_inheritable; + new->cap_permitted = old->cap_permitted; + new->cap_effective = old->cap_effective; + new->cap_ambient = old->cap_ambient; + new->cap_bset = old->cap_bset; + + new->jit_keyring = old->jit_keyring; + new->thread_keyring = key_get(old->thread_keyring); + new->process_keyring = key_get(old->process_keyring); } void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - struct task_token *token = iee_get_task_token(current); /* Get old cred inside IEE is safer. */ const struct cred *old = current_cred(); - /* Would verify this field in commit_cred. */ - token->new_cred = new; + + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -314,6 +417,7 @@ void _iee_copy_cred(unsigned long __unused, struct cred *new) void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->gid = gid; } @@ -321,6 +425,39 @@ void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->uid = uid; } + +void __iee_code _iee_override_creds(unsigned long __unused, const struct cred *new) +{ + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + token->curr_cred = new; + } +#endif + + rcu_assign_pointer(current->cred, new); +} + +void __iee_code _iee_revert_creds(unsigned long __unused, const struct cred *old) +{ + iee_verify_cred(); + iee_verify_cred_type(old); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + token->curr_cred = old; + } +#endif + + rcu_assign_pointer(current->cred, old); +} diff --git a/arch/arm64/kernel/haoc/haoc-bitmap.c b/arch/arm64/kernel/haoc/haoc-bitmap.c index 694f6256c0fc5..aa82948d46e0a 100644 --- a/arch/arm64/kernel/haoc/haoc-bitmap.c +++ b/arch/arm64/kernel/haoc/haoc-bitmap.c @@ -194,6 +194,12 @@ void __init haoc_bitmap_setup(void) num_pages = (end - start) / PAGE_SIZE; iee_set_bitmap_type(start, num_pages, IEE_SELINUX); #endif + #ifdef CONFIG_CREDP + start = (unsigned long)__iee_cred_data_start; + end = (unsigned long)__iee_cred_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_CRED); + #endif #ifdef CONFIG_VARP /* Map .iee.varp as RO pages, the variable need to be protected can be added */ start = (unsigned long)__iee_varp_data_start; diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index bbb071026ffc2..0567c8caf58f2 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -36,6 +36,8 @@ iee_func iee_funcs[] = { (iee_func)_iee_commit_creds, (iee_func)_iee_abort_cred, (iee_func)_iee_fill_cred_for_session_keyring, + (iee_func)_iee_override_creds, + (iee_func)_iee_revert_creds, (iee_func)_iee_set_cred_uid, (iee_func)_iee_set_cred_gid, (iee_func)_iee_set_cred_suid, diff --git a/arch/arm64/kernel/haoc/iee/iee-token.c b/arch/arm64/kernel/haoc/iee/iee-token.c index 2bbe40e425f1c..c293943b94ae3 100644 --- a/arch/arm64/kernel/haoc/iee/iee-token.c +++ b/arch/arm64/kernel/haoc/iee/iee-token.c @@ -217,6 +217,10 @@ void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct * if (token->valid) pr_err("IEE: validate token for multiple times."); +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = tsk->cred; +#endif token->valid = true; } @@ -225,5 +229,9 @@ void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct struct task_token *token = (struct task_token *)iee_get_task_token(tsk); token->pgd = NULL; +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = NULL; +#endif token->valid = false; } diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 4da97244a2ff9..6b2f766947d98 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -151,7 +151,10 @@ jiffies = jiffies_64; #ifdef CONFIG_CREDP #define CRED_DATA \ . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_start = .; \ *(.iee.cred) \ + . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_end = .; \ . = ALIGN(PAGE_SIZE); #else #define CRED_DATA diff --git a/arch/x86/include/asm/haoc/haoc-bitmap.h b/arch/x86/include/asm/haoc/haoc-bitmap.h index 608defe0feda0..9d6090b20a8b2 100644 --- a/arch/x86/include/asm/haoc/haoc-bitmap.h +++ b/arch/x86/include/asm/haoc/haoc-bitmap.h @@ -20,6 +20,7 @@ enum HAOC_BITMAP_TYPE { IEE_SIP_DATA, IEE_BITMAP, IEE_VARP, + IEE_CRED, }; extern void iee_init_bitmap(void); diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index a3512114ef190..e0063ec6580d2 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -28,6 +28,8 @@ enum { IEE_OP_COMMIT_CRED, IEE_OP_ABORT_CRED, IEE_OP_FILL_SESSION_KEYRING_CRED, + IEE_OP_OVERRIDE_CRED, + IEE_OP_REVERT_CRED, IEE_OP_SET_CRED_UID, IEE_OP_SET_CRED_GID, IEE_OP_SET_CRED_SUID, diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index 44f96b892ec7c..25dc31d85263a 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -34,12 +34,13 @@ void _iee_set_bitmap_type(unsigned long __unused, uint8_t *bitmap_ptr, void _iee_copy_cred(unsigned long __unused, struct cred *new); void _iee_copy_kernel_cred(unsigned long iee_offset, const struct cred *old, struct cred *new); void _iee_abort_cred(unsigned long iee_offset, const struct cred *cred); -void _iee_fill_cred_for_session_keyring(unsigned long __unused, - struct cred *new, - const struct cred *old); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old); void _iee_init_copied_cred(unsigned long iee_offset, struct task_struct *new_task, struct cred *new); void _iee_commit_creds(unsigned long iee_offset, const struct cred *new); +void _iee_override_creds(unsigned long iee_offset, const struct cred *new); +void _iee_revert_creds(unsigned long iee_offset, const struct cred *old); void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); diff --git a/arch/x86/include/asm/haoc/iee-cred.h b/arch/x86/include/asm/haoc/iee-cred.h index 8da042a49d86d..fa6a3a1d81a24 100644 --- a/arch/x86/include/asm/haoc/iee-cred.h +++ b/arch/x86/include/asm/haoc/iee-cred.h @@ -2,11 +2,73 @@ #ifndef _LINUX_IEE_CRED_H #define _LINUX_IEE_CRED_H -#include #include +#include +#include +#include +#ifdef CONFIG_IEE_PTRP +#include +#endif extern unsigned long long iee_rw_gate(int flag, ...); +static inline void iee_detect_cred_uaf(struct cred *cred, const char *caller) +{ + if (unlikely(!haoc_init_done)) + return; + + if (atomic_long_read(&cred->usage) != 0) + panic("IEE: (%s) Cred usage %ld.", caller, + atomic_long_read(&cred->usage)); +} + +#ifdef CONFIG_IEE_PTRP +static inline void __iee_verify_cred(void) +{ + struct task_struct *curr_task; + struct task_token *token; + const struct cred *curr_cred; + + if (unlikely(!haoc_init_done)) + return; + + curr_task = current; + if (curr_task == &init_task) + return; + + token = (struct task_token *)__addr_to_token(curr_task); + curr_cred = current_cred(); + if (!token->valid) + panic("IEE: (%s) Invalid Token.", __func__); + if (token->curr_cred != curr_cred) + panic("IEE: (%s) Task cred corrupted.", __func__); +} + +static inline void iee_verify_cred(void) +{ + __iee_verify_cred(); +} + +static inline void iee_verify_update_cred(struct cred *new) +{ + struct task_token *token; + + if (unlikely(!haoc_init_done)) + return; + + __iee_verify_cred(); + if (current == &init_task) + return; + + token = (struct task_token *)__addr_to_token(current); + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); +} +#else +static inline void iee_verify_cred(void) { } +static inline void iee_verify_update_cred(struct cred *new) { } +#endif + static void __maybe_unused iee_copy_cred(struct cred *new) { iee_rw_gate(IEE_OP_COPY_CRED, new); @@ -40,6 +102,16 @@ static void __maybe_unused iee_commit_creds(const struct cred *new) iee_rw_gate(IEE_OP_COMMIT_CRED, new); } +static void __maybe_unused iee_override_creds(const struct cred *new) +{ + iee_rw_gate(IEE_OP_OVERRIDE_CRED, new); +} + +static void __maybe_unused iee_revert_creds(const struct cred *old) +{ + iee_rw_gate(IEE_OP_REVERT_CRED, old); +} + static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) { if(!haoc_enabled) @@ -305,4 +377,9 @@ static void __maybe_unused iee_set_cred_security(struct cred *cred, void *securi } #endif +extern void iee_free_cred_slab(struct work_struct *work); + +extern unsigned long __iee_cred_data_start[]; +extern unsigned long __iee_cred_data_end[]; + #endif diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h index 771628ddab444..7ddd13acb97fd 100644 --- a/arch/x86/include/asm/haoc/iee-token.h +++ b/arch/x86/include/asm/haoc/iee-token.h @@ -39,7 +39,8 @@ struct task_token { bool valid; void *kernel_stack; /* VA */ #ifdef CONFIG_CREDP - struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *curr_cred; /* The current subjective credentials. */ #endif }; #endif /* CONFIG_IEE */ diff --git a/arch/x86/kernel/haoc/credp/credp.c b/arch/x86/kernel/haoc/credp/credp.c index 1b30bf3774855..f131df77ec255 100644 --- a/arch/x86/kernel/haoc/credp/credp.c +++ b/arch/x86/kernel/haoc/credp/credp.c @@ -3,36 +3,51 @@ #include #include #include -#include +#include #include +#include #include #include +#ifdef CONFIG_IEE_PTRP +#include +#endif #include "slab.h" extern struct cred init_cred; -static inline struct task_token *credp_task_token(struct task_struct *task) +static inline void iee_verify_cred_type(const struct cred *cred) { -#ifdef CONFIG_IEE_PTRP - if (haoc_init_done) - return (struct task_token *)__addr_to_token(task); -#endif - return (struct task_token *)__addr_to_iee(task); + iee_verify_type(__pa(cred), IEE_CRED, "cred"); +} + +static inline void iee_verify_current_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_cred(); +} + +static inline void iee_verify_new_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_cred(); } void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) { + iee_verify_current_cred_type(cred); *((struct rcu_head **)(&(cred->rcu.func))) = rcu; } void _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security) { + iee_verify_current_cred_type(cred); cred->security = security; } unsigned long _iee_set_cred_atomic_op_usage(unsigned long __unused, struct cred *cred, int flag, int nr) { + iee_verify_current_cred_type(cred); switch (flag) { case AT_ADD: { atomic_long_add(nr, &cred->usage); @@ -50,126 +65,149 @@ unsigned long _iee_set_cred_atomic_op_usage(unsigned long __unused, void _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i) { + iee_verify_current_cred_type(cred); atomic_long_set(&cred->usage, i); } void _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu) { + iee_verify_current_cred_type(cred); cred->non_rcu = non_rcu; } void _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, struct key *session_keyring) { + iee_verify_new_cred_type(cred); cred->session_keyring = session_keyring; } void _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, struct key *process_keyring) { + iee_verify_new_cred_type(cred); cred->process_keyring = process_keyring; } void _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, struct key *thread_keyring) { + iee_verify_current_cred_type(cred); cred->thread_keyring = thread_keyring; } void _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, struct key *request_key_auth) { + iee_verify_new_cred_type(cred); cred->request_key_auth = request_key_auth; } void _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring) { + iee_verify_new_cred_type(cred); cred->jit_keyring = jit_keyring; } void _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, kernel_cap_t cap_inheritable) { + iee_verify_new_cred_type(cred); cred->cap_inheritable = cap_inheritable; } void _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, kernel_cap_t cap_permitted) { + iee_verify_new_cred_type(cred); cred->cap_permitted = cap_permitted; } void _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, kernel_cap_t cap_effective) { + iee_verify_new_cred_type(cred); cred->cap_effective = cap_effective; } void _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset) { + iee_verify_new_cred_type(cred); cred->cap_bset = cap_bset; } void _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient) { + iee_verify_new_cred_type(cred); cred->cap_ambient = cap_ambient; } void _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, unsigned int securebits) { + iee_verify_new_cred_type(cred); cred->securebits = securebits; } void _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, struct group_info *group_info) { + iee_verify_current_cred_type(cred); cred->group_info = group_info; } void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts) { + iee_verify_new_cred_type(cred); cred->ucounts = ucounts; } void _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, struct user_namespace *user_ns) { + iee_verify_new_cred_type(cred); cred->user_ns = user_ns; } void _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user) { + iee_verify_new_cred_type(cred); cred->user = user; } void _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid) { + iee_verify_new_cred_type(cred); cred->fsgid = fsgid; } void _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid) { + iee_verify_new_cred_type(cred); cred->fsuid = fsuid; } void _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid) { + iee_verify_new_cred_type(cred); cred->egid = egid; } void _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid) { + iee_verify_new_cred_type(cred); cred->euid = euid; } void _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid) { + iee_verify_new_cred_type(cred); cred->sgid = sgid; } void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid) { + iee_verify_new_cred_type(cred); cred->suid = suid; } @@ -179,11 +217,21 @@ void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - struct task_token *token = credp_task_token(current); /* Get old cred inside IEE is safer. */ const struct cred *old = current_cred(); - /* Would verify this field in commit_cred. */ - token->new_cred = new; + + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -197,13 +245,23 @@ void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - /* Would verify this field in commit_cred. */ - struct task_token *token = credp_task_token(current); + + iee_verify_cred_type(old); + iee_verify_cred_type(new); + iee_verify_cred(); if (!uid_eq(current_uid(), init_cred.uid)) panic("IEE: calling prepare_kernel_cred by unprivileged process."); - token->new_cred = new; +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -215,13 +273,24 @@ void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, void _iee_init_copied_cred(unsigned long __unused, struct task_struct *new_task, struct cred *new) { - struct task_token *old_task_token = credp_task_token(current); + iee_verify_cred_type(new); + iee_verify_cred(); - if (old_task_token->new_cred != new) - panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", __func__, - (u64)old_task_token->new_cred, (u64)new); - /* Update token info of new task by current task token. */ - old_task_token->new_cred = NULL; +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *old_task_token = + (struct task_token *)__addr_to_token(current); + struct task_token *new_task_token = + (struct task_token *)__addr_to_token(new_task); + + if (old_task_token->new_cred != new) + panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", + __func__, (u64)old_task_token->new_cred, (u64)new); + /* Update token info of new task by current task token. */ + old_task_token->new_cred = NULL; + new_task_token->curr_cred = new; + } +#endif new_task->cred = new_task->real_cred = new; } @@ -229,12 +298,21 @@ void _iee_init_copied_cred(unsigned long __unused, void _iee_commit_creds(unsigned long __unused, const struct cred *new) { struct task_struct *task = current; - struct task_token *token = credp_task_token(task); - if (token->new_cred != new) - panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); - /* task->cred shall be updated once. */ - token->new_cred = NULL; + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(task); + + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); + /* task->cred shall be updated once. */ + token->new_cred = NULL; + token->curr_cred = new; + } +#endif rcu_assign_pointer(task->real_cred, new); rcu_assign_pointer(task->cred, new); @@ -242,17 +320,33 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) void _iee_abort_cred(unsigned long __unused, const struct cred *cred) { - struct task_token *token = credp_task_token(current); + iee_verify_cred_type(cred); + iee_verify_cred(); - token->new_cred = NULL; +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + token->new_cred = NULL; + } +#endif } void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, const struct cred *old) { - struct task_token *token = credp_task_token(current); + iee_verify_cred_type(new); + iee_verify_cred_type(old); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + token->new_cred = new; + } +#endif - token->new_cred = new; new->uid = old->uid; new->euid = old->euid; new->suid = old->suid; @@ -280,14 +374,48 @@ void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) { + iee_verify_new_cred_type(cred); cred->gid = gid; } void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid) { + iee_verify_new_cred_type(cred); cred->uid = uid; } +void _iee_override_creds(unsigned long __unused, const struct cred *new) +{ + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + token->curr_cred = new; + } +#endif + + rcu_assign_pointer(current->cred, new); +} + +void _iee_revert_creds(unsigned long __unused, const struct cred *old) +{ + iee_verify_cred_type(old); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + token->curr_cred = old; + } +#endif + + rcu_assign_pointer(current->cred, old); +} + struct iee_free_slab_work { struct work_struct work; struct kmem_cache *s; @@ -302,7 +430,19 @@ void iee_free_cred_slab(struct work_struct *work) struct folio *folio = slab_folio(slab); int order = folio_order(folio); - unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); + unset_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << order); __free_pages(&folio->page, order); kfree(iee_free_slab_work); } + +void __init credp_init(void) +{ + unsigned long start, end; + int num_pages; + + start = (unsigned long)__iee_cred_data_start; + end = (unsigned long)__iee_cred_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_pages(start, num_pages, IEE_CRED); +} diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 9e62a8bc4caab..c8f26ca7cf756 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -28,6 +28,8 @@ iee_func iee_funcs[] = { (iee_func)_iee_commit_creds, (iee_func)_iee_abort_cred, (iee_func)_iee_fill_cred_for_session_keyring, + (iee_func)_iee_override_creds, + (iee_func)_iee_revert_creds, (iee_func)_iee_set_cred_uid, (iee_func)_iee_set_cred_gid, (iee_func)_iee_set_cred_suid, diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index c2aa93bf76af9..cfac8a3333e46 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -21,6 +21,9 @@ #ifdef CONFIG_IEE_SIP #include #endif +#ifdef CONFIG_CREDP +extern void credp_init(void); +#endif /* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ unsigned long IEE_OFFSET = 0x200000000000; @@ -249,6 +252,9 @@ void __init iee_post_init(void) iee_stack_set_ro(); #ifdef CONFIG_IEE_SIP iee_sip_init(); +#endif +#ifdef CONFIG_CREDP + credp_init(); #endif haoc_init_done = true; } diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c index a213c21df24b4..a4f8e3a82c38f 100644 --- a/arch/x86/kernel/haoc/iee/iee-token.c +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -129,6 +129,10 @@ void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) struct task_token *token = (struct task_token *)__addr_to_token(tsk); token->pgd = NULL; +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = NULL; +#endif token->valid = false; } @@ -138,5 +142,9 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) if (token->valid) pr_err("IEE: validate token for multiple times."); +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = tsk->cred; +#endif token->valid = true; } diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index dc736a9101a80..a4cb219a8a92c 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -117,7 +117,10 @@ __iee_si_data_end = .; #ifdef CONFIG_CREDP #define CRED_DATA \ . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_start = .; \ *(.iee.cred) \ + . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_end = .; \ . = ALIGN(PAGE_SIZE); #else #define CRED_DATA diff --git a/kernel/cred.c b/kernel/cred.c index d7042a4633408..7069195fd055b 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -650,7 +650,14 @@ const struct cred *override_creds(const struct cred *new) * visible to other threads under RCU. */ get_new_cred((struct cred *)new); +#ifdef CONFIG_CREDP + if (haoc_enabled) + iee_override_creds(new); + else + rcu_assign_pointer(current->cred, new); +#else rcu_assign_pointer(current->cred, new); +#endif kdebug("override_creds() = %p{%ld}", old, atomic_long_read(&old->usage)); @@ -672,7 +679,14 @@ void revert_creds(const struct cred *old) kdebug("revert_creds(%p{%ld})", old, atomic_long_read(&old->usage)); +#ifdef CONFIG_CREDP + if (haoc_enabled) + iee_revert_creds(old); + else + rcu_assign_pointer(current->cred, old); +#else rcu_assign_pointer(current->cred, old); +#endif put_cred(override); } EXPORT_SYMBOL(revert_creds); diff --git a/mm/slub.c b/mm/slub.c index 14465c4c0ffd8..a75373756a61f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2168,9 +2168,15 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) * pages read-only here would trigger faults on those writes. * Skip enforcing RO for cred slabs when KASAN is enabled. */ - if (haoc_enabled && s == cred_jar && !IS_ENABLED(CONFIG_KASAN)) - set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), - oo_order(oo)); + if (haoc_enabled && s == cred_jar && !IS_ENABLED(CONFIG_KASAN)) { + #ifdef CONFIG_X86_64 + set_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << oo_order(oo), IEE_CRED); + #else + set_iee_page_type((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + oo_order(oo), IEE_CRED); + #endif + } #endif account_slab(slab, oo_order(oo), s, flags); @@ -2237,8 +2243,11 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) iee_free_slab(s, slab, iee_free_cred_slab); return; #else - if (!IS_ENABLED(CONFIG_KASAN)) + if (!IS_ENABLED(CONFIG_KASAN)) { + iee_set_bitmap_type((unsigned long)page_address(folio_page(folio, 0)), + pages, IEE_NORMAL); unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); + } #endif } #endif diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index 1993bef2f2f1a..7494e32c747b9 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -920,12 +920,40 @@ long join_session_keyring(const char *name) } /* - * Copy the credential fields that a session keyring change needs, - * reusing cred_alloc_blank()'s blank allocation. + * Replace a process's session keyring on behalf of one of its children when + * the target process is about to resume userspace execution. */ -static void fill_cred_for_session_keyring(struct cred *new, - const struct cred *old) +void key_change_session_keyring(struct callback_head *twork) { + const struct cred *old = current_cred(); + #ifdef CONFIG_CREDP + struct cred *new =NULL; + if(haoc_enabled) + new = *(struct cred **)(twork + 1); + else + new = container_of(twork, struct cred, rcu); + #else + struct cred *new = container_of(twork, struct cred, rcu); + #endif + + if (unlikely(current->flags & PF_EXITING)) { + put_cred(new); + return; + } + + /* If get_ucounts fails more bits are needed in the refcount */ + if (unlikely(!get_ucounts(old->ucounts))) { + WARN_ONCE(1, "In %s get_ucounts failed\n", __func__); + put_cred(new); + return; + } + + #ifdef CONFIG_CREDP + if (haoc_enabled) + iee_fill_cred_for_session_keyring(new, old); + else + #endif + { new-> uid = old-> uid; new-> euid = old-> euid; new-> suid = old-> suid; @@ -949,42 +977,8 @@ static void fill_cred_for_session_keyring(struct cred *new, new->jit_keyring = old->jit_keyring; new->thread_keyring = key_get(old->thread_keyring); new->process_keyring = key_get(old->process_keyring); -} - -void key_change_session_keyring(struct callback_head *twork) -{ - const struct cred *old = current_cred(); - -#ifdef CONFIG_CREDP - struct cred *new = NULL; - - if (haoc_enabled) - new = *(struct cred **)(twork + 1); - else - new = container_of(twork, struct cred, rcu); -#else - struct cred *new = container_of(twork, struct cred, rcu); -#endif - - if (unlikely(current->flags & PF_EXITING)) { - put_cred(new); - return; - } - - /* If get_ucounts fails more bits are needed in the refcount */ - if (unlikely(!get_ucounts(old->ucounts))) { - WARN_ONCE(1, "In %s get_ucounts failed\n", __func__); - put_cred(new); - return; } -#ifdef CONFIG_CREDP - if (haoc_enabled) - iee_fill_cred_for_session_keyring(new, old); - else -#endif - fill_cred_for_session_keyring(new, old); - security_transfer_creds(new, old); commit_creds(new); From b47bbadae14d99981e03cb57ee0d3e46b62d4ef2 Mon Sep 17 00:00:00 2001 From: Hang Shu Date: Wed, 25 Mar 2026 13:54:13 +0800 Subject: [PATCH 05/26] HAOC3.0/x86_64/PTP: Add support for x86 Page Table Isolation Support page table protection and isolation for x86. Signed-off-by: sh Signed-off-by: Liu Zhehui --- arch/x86/Kconfig | 14 + arch/x86/boot/compressed/ident_map_64.c | 22 + arch/x86/configs/deepin_x86_desktop_defconfig | 2 + arch/x86/include/asm/haoc/haoc-def.h | 15 + arch/x86/include/asm/haoc/haoc.h | 18 + arch/x86/include/asm/haoc/iee-func.h | 5 +- arch/x86/include/asm/haoc/ptp.h | 68 +++ arch/x86/include/asm/pgalloc.h | 11 + arch/x86/include/asm/pgtable.h | 32 ++ arch/x86/include/asm/pgtable_64.h | 45 ++ arch/x86/include/asm/tlb.h | 9 + arch/x86/kernel/alternative.c | 27 + arch/x86/kernel/haoc/Makefile | 1 + arch/x86/kernel/haoc/gate.h | 29 + arch/x86/kernel/haoc/haoc.c | 15 + arch/x86/kernel/haoc/iee/iee-func.c | 12 +- arch/x86/kernel/haoc/iee/iee-gate.S | 89 +-- arch/x86/kernel/haoc/iee/iee-init.c | 23 +- arch/x86/kernel/haoc/iee/iee-token.c | 50 ++ arch/x86/kernel/haoc/ptp/Makefile | 1 + arch/x86/kernel/haoc/ptp/ptp-gate.S | 131 +++++ arch/x86/kernel/haoc/ptp/ptp.c | 528 ++++++++++++++++++ arch/x86/kernel/nmi.c | 36 ++ arch/x86/kernel/paravirt.c | 4 + arch/x86/mm/ident_map.c | 12 + arch/x86/mm/pat/set_memory.c | 12 + arch/x86/mm/pgtable.c | 45 ++ arch/x86/mm/pti.c | 32 ++ arch/x86/mm/ptp_ident_map.c | 182 ++++++ arch/x86/platform/efi/efi_64.c | 24 + fs/exec.c | 17 + include/asm-generic/tlb.h | 33 ++ include/linux/haoc-ptp.h | 10 + include/linux/mm.h | 15 + include/linux/mm_types.h | 3 + include/linux/ptp-cache.h | 39 ++ kernel/fork.c | 17 + kernel/sched/core.c | 20 + mm/Makefile | 1 + mm/debug_vm_pgtable.c | 8 + mm/haoc/Makefile | 1 + mm/haoc/ptp-pg_cache.c | 413 ++++++++++++++ mm/memory.c | 15 + mm/mmap.c | 37 ++ mm/mmu_gather.c | 194 +++++++ mm/mprotect.c | 13 + mm/pgtable-generic.c | 25 + 47 files changed, 2308 insertions(+), 47 deletions(-) create mode 100644 arch/x86/include/asm/haoc/ptp.h create mode 100644 arch/x86/kernel/haoc/gate.h create mode 100644 arch/x86/kernel/haoc/ptp/Makefile create mode 100644 arch/x86/kernel/haoc/ptp/ptp-gate.S create mode 100644 arch/x86/kernel/haoc/ptp/ptp.c create mode 100644 arch/x86/mm/ptp_ident_map.c create mode 100644 include/linux/haoc-ptp.h create mode 100644 include/linux/ptp-cache.h create mode 100644 mm/haoc/Makefile create mode 100644 mm/haoc/ptp-pg_cache.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 91685de954096..e0de378277fae 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1556,6 +1556,20 @@ config IEE Could isolate kernel critical data and enforce all write access made and verified in IEE APIs. +config PTP + bool "Page Table Protection(PTP)" + depends on IEE + help + Protect page tables by routing page-table modification paths + through the IEE gate. This keeps page table updates mediated by + the isolated execution environment instead of allowing direct + writes from normal kernel code. + +config PTP_RESERVE_ORDER + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + depends on PTP + range 9 15 + config IEE_PTRP bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 848a958c639f8..bf3c12a654414 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -23,7 +23,11 @@ /* Use the static base for this part of the boot process */ #undef __PAGE_OFFSET #define __PAGE_OFFSET __PAGE_OFFSET_BASE +#ifdef CONFIG_PTP +#include "../../mm/ptp_ident_map.c" +#else #include "../../mm/ident_map.c" +#endif #define _SETUP #include /* For COMMAND_LINE_SIZE */ @@ -101,9 +105,15 @@ void kernel_add_identity_map(unsigned long start, unsigned long end) return; /* Build the mapping. */ +#ifdef CONFIG_PTP + ret = ptp_kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); + if (ret) + error("Error: kernel_ident_mapping_init_for_iee() failed\n"); +#else ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); if (ret) error("Error: kernel_ident_mapping_init() failed\n"); +#endif } /* Locates and clears a region for a new top level page table. */ @@ -215,7 +225,11 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, /* Populate the PTEs */ for (i = 0; i < PTRS_PER_PMD; i++) { +#ifdef CONFIG_PTP + ptp_set_pte_pre_init(&pte[i], __pte(address | page_flags)); +#else set_pte(&pte[i], __pte(address | page_flags)); +#endif address += PAGE_SIZE; } @@ -229,7 +243,11 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, * of a TLB multihit. */ pmd = __pmd((unsigned long)pte | info->kernpg_flag); +#ifdef CONFIG_PTP + ptp_set_pmd_pre_init(pmdp, pmd); +#else set_pmd(pmdp, pmd); +#endif /* Flush TLB to establish the new PMD */ #ifdef CONFIG_IEE_SIP if(haoc_enabled) @@ -328,7 +346,11 @@ static int set_clr_page_flags(struct x86_mapping_info *info, pte = *ptep; pte = pte_set_flags(pte, set); pte = pte_clear_flags(pte, clr); +#ifdef CONFIG_PTP + ptp_set_pte_pre_init(ptep, pte); +#else set_pte(ptep, pte); +#endif /* * If the encryption attribute is being set, then change the page state to diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 81f8e1d106164..7b573e304318c 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -92,6 +92,8 @@ CONFIG_IEE=y CONFIG_IEE_PTRP=y CONFIG_IEE_SIP=y CONFIG_CREDP=y +CONFIG_PTP=y +CONFIG_PTP_RESERVE_ORDER=12 CONFIG_NUMA=y CONFIG_ARCH_MEMORY_PROBE=y CONFIG_X86_PMEM_LEGACY=m diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index e0063ec6580d2..d317af44ef251 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -58,6 +58,21 @@ enum { IEE_OP_SET_CRED_SECURITY, IEE_OP_SET_CRED_RCU, IEE_OP_SET_CRED_UCOUNTS, +#endif +#ifdef CONFIG_PTP +#ifdef CONFIG_IEE_PTRP + IEE_OP_UNSET_TOKEN, + IEE_OP_SET_TOKEN, +#endif + IEE_OP_SET_PTE, + IEE_OP_SET_PMD, + IEE_OP_SET_PUD, + IEE_OP_SET_P4D, + IEE_OP_SET_PGD, + IEE_OP_SET_PTE_TEXT_POKE, + IEE_OP_SET_PMD_IDENT, + IEE_OP_SET_PUD_IDENT, + IEE_OP_SET_SX_PTE, #endif IEE_FLAG_END }; diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index 25dc31d85263a..f938382c8ad59 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -28,6 +28,24 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); void _iee_set_bitmap_type(unsigned long __unused, uint8_t *bitmap_ptr, enum HAOC_BITMAP_TYPE type, int num_pages); +#ifdef CONFIG_PTP +#ifdef CONFIG_IEE_PTRP +void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token, unsigned int order); +void _iee_set_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token_page, unsigned int order); +#endif +void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte); +void _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd); +void _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud); +void _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d); +void _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd); +void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte); +void _iee_set_pmd_ident(unsigned long __unused, pmd_t *pmdp, pmd_t pmd); +void _iee_set_pud_ident(unsigned long __unused, pud_t *pudp, pud_t pud); +void _iee_set_sx_pte(unsigned long __unused, pte_t *ptep, pte_t pte); +#endif + #ifdef CONFIG_CREDP #include diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h index 5a52de12f2f53..791423c74d45c 100644 --- a/arch/x86/include/asm/haoc/iee-func.h +++ b/arch/x86/include/asm/haoc/iee-func.h @@ -17,8 +17,9 @@ extern void set_iee_pages(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type); extern void unset_iee_pages(unsigned long addr, int num_pages); -extern void set_iee_page(unsigned long addr, unsigned int order); -extern void unset_iee_page(unsigned long addr, unsigned int order); +extern void set_iee_page(unsigned long addr, int num_pages, + enum HAOC_BITMAP_TYPE type); +extern void unset_iee_page(unsigned long addr, int num_pages); extern bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order); extern unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order); diff --git a/arch/x86/include/asm/haoc/ptp.h b/arch/x86/include/asm/haoc/ptp.h new file mode 100644 index 0000000000000..20e04e40fa940 --- /dev/null +++ b/arch/x86/include/asm/haoc/ptp.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PTP_H +#define _LINUX_PTP_H + +#include +#include + +extern unsigned long long ptp_rw_gate(int flag, ...); + +static inline void ptp_set_pte(pte_t *ptep, pte_t pte) +{ + ptp_rw_gate(IEE_OP_SET_PTE, ptep, pte); +} + +static inline void ptp_set_pmd(pmd_t *pmdp, pmd_t pmd) +{ + ptp_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); +} + +static inline void ptp_set_pud(pud_t *pudp, pud_t pud) +{ + ptp_rw_gate(IEE_OP_SET_PUD, pudp, pud); +} + +static inline void ptp_set_p4d(p4d_t *p4dp, p4d_t p4d) +{ + ptp_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); +} + +static inline void ptp_set_pgd(pgd_t *pgdp, pgd_t pgd) +{ + ptp_rw_gate(IEE_OP_SET_PGD, pgdp, pgd); +} + +static inline void ptp_set_pte_text_poke(pte_t *ptep, pte_t pte) +{ + ptp_rw_gate(IEE_OP_SET_PTE_TEXT_POKE, ptep, pte); +} + +static inline void ptp_set_pmd_ident(pmd_t *pmdp, pmd_t pmd) +{ + ptp_rw_gate(IEE_OP_SET_PMD_IDENT, pmdp, pmd); +} + +static inline void ptp_set_pud_ident(pud_t *pudp, pud_t pud) +{ + ptp_rw_gate(IEE_OP_SET_PUD_IDENT, pudp, pud); +} + +static inline void ptp_set_sx_pte(pte_t *ptep, pte_t pte) +{ + ptp_rw_gate(IEE_OP_SET_SX_PTE, ptep, pte); +} + +extern pgprotval_t ptp_xchg(pgprotval_t *pgprotp, pgprotval_t pgprotval); +extern pgprotval_t ptp_try_cmpxchg(pgprotval_t *pgprotp, + pgprotval_t old_pgprot, pgprotval_t new_pgprotval); +extern void haoc_ptp_init(void); +extern struct iee_cache pgd_cache; + +#include +struct iee_cr0 { + /* Writable but considered safe to expose */ + unsigned long wp_disabled_cnt; +}; + +DECLARE_PER_CPU(struct iee_cr0, iee_cr0s); +#endif diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index c7ec5bb88334e..eeebd7345fa82 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -9,6 +9,9 @@ #define __HAVE_ARCH_PTE_ALLOC_ONE #define __HAVE_ARCH_PGD_FREE #include +#ifdef CONFIG_PTP +#include +#endif static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } @@ -153,7 +156,11 @@ static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) if (mm == &init_mm) gfp &= ~__GFP_ACCOUNT; +#ifdef CONFIG_PTP + return (p4d_t *)iee_cache_alloc(&pg_cache, gfp | __GFP_ZERO); +#else return (p4d_t *)get_zeroed_page(gfp); +#endif } static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) @@ -162,7 +169,11 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) return; BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, p4d); +#else free_page((unsigned long)p4d); +#endif } extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 993d49cd379a2..765c1652b557a 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -23,6 +23,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif extern pgd_t early_top_pgt[PTRS_PER_PGD]; bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); @@ -1301,9 +1304,15 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, pte_t old_pte, new_pte; old_pte = READ_ONCE(*ptep); +#ifdef CONFIG_PTP + do { + new_pte = pte_wrprotect(old_pte); + } while (!ptp_try_cmpxchg((long *)ptep, pte_val(old_pte), pte_val(new_pte))); +#else do { new_pte = pte_wrprotect(old_pte); } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte)); +#endif } #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) @@ -1363,9 +1372,15 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, pmd_t old_pmd, new_pmd; old_pmd = READ_ONCE(*pmdp); +#ifdef CONFIG_PTP + do { + new_pmd = pmd_wrprotect(old_pmd); + } while (!ptp_try_cmpxchg((long *)pmdp, pmd_val(old_pmd), pmd_val(new_pmd))); +#else do { new_pmd = pmd_wrprotect(old_pmd); } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd)); +#endif } #ifndef pmdp_establish @@ -1375,10 +1390,18 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, { page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); if (IS_ENABLED(CONFIG_SMP)) { +#ifdef CONFIG_PTP + return native_make_pmd(ptp_xchg((pgprotval_t *)pmdp, pmd_val(pmd))); +#else return xchg(pmdp, pmd); +#endif } else { pmd_t old = *pmdp; +#ifdef CONFIG_PTP + set_pmd(pmdp, pmd); +#else WRITE_ONCE(*pmdp, pmd); +#endif return old; } } @@ -1466,14 +1489,23 @@ static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) */ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) { +#ifdef CONFIG_PTP + iee_memcpy(dst, src, count * sizeof(pgd_t)); +#else memcpy(dst, src, count * sizeof(pgd_t)); +#endif #ifdef CONFIG_PAGE_TABLE_ISOLATION if (!static_cpu_has(X86_FEATURE_PTI)) return; /* Clone the user space pgd as well */ +#ifdef CONFIG_PTP + iee_memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), + count * sizeof(pgd_t)); +#else memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), count * sizeof(pgd_t)); #endif +#endif } #define PTE_SHIFT ilog2(PTRS_PER_PTE) diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index a629b1b9f65a6..6839dd1be00be 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -15,6 +15,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif extern p4d_t level4_kernel_pgt[512]; extern p4d_t level4_ident_pgt[512]; @@ -64,7 +67,12 @@ void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte); static inline void native_set_pte(pte_t *ptep, pte_t pte) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*ptep); + ptp_set_pte(ptep, pte); +#else WRITE_ONCE(*ptep, pte); +#endif } static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, @@ -80,7 +88,12 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*pmdp); + ptp_set_pmd(pmdp, pmd); +#else WRITE_ONCE(*pmdp, pmd); +#endif } static inline void native_pmd_clear(pmd_t *pmd) @@ -91,7 +104,11 @@ static inline void native_pmd_clear(pmd_t *pmd) static inline pte_t native_ptep_get_and_clear(pte_t *xp) { #ifdef CONFIG_SMP +#ifdef CONFIG_PTP + return native_make_pte(ptp_xchg((pgprotval_t *)xp, 0)); +#else return native_make_pte(xchg(&xp->pte, 0)); +#endif #else /* native_local_ptep_get_and_clear, but duplicated because of cyclic dependency */ @@ -104,7 +121,11 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) { #ifdef CONFIG_SMP +#ifdef CONFIG_PTP + return native_make_pmd(ptp_xchg((pgprotval_t *)xp, 0)); +#else return native_make_pmd(xchg(&xp->pmd, 0)); +#endif #else /* native_local_pmdp_get_and_clear, but duplicated because of cyclic dependency */ @@ -116,7 +137,12 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) static inline void native_set_pud(pud_t *pudp, pud_t pud) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*pudp); + ptp_set_pud(pudp, pud); +#else WRITE_ONCE(*pudp, pud); +#endif } static inline void native_pud_clear(pud_t *pud) @@ -127,7 +153,11 @@ static inline void native_pud_clear(pud_t *pud) static inline pud_t native_pudp_get_and_clear(pud_t *xp) { #ifdef CONFIG_SMP +#ifdef CONFIG_PTP + return native_make_pud(ptp_xchg((pgprotval_t *)xp, 0)); +#else return native_make_pud(xchg(&xp->pud, 0)); +#endif #else /* native_local_pudp_get_and_clear, * but duplicated because of cyclic dependency @@ -144,13 +174,23 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) pgd_t pgd; if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*p4dp); + ptp_set_p4d(p4dp, p4d); +#else WRITE_ONCE(*p4dp, p4d); +#endif return; } pgd = native_make_pgd(native_p4d_val(p4d)); pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*p4dp); + ptp_set_p4d(p4dp, native_make_p4d(native_pgd_val(pgd))); +#else WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); +#endif } static inline void native_p4d_clear(p4d_t *p4d) @@ -160,7 +200,12 @@ static inline void native_p4d_clear(p4d_t *p4d) static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*pgdp); + ptp_set_pgd(pgdp, pti_set_user_pgtbl(pgdp, pgd)); +#else WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); +#endif } static inline void native_pgd_clear(pgd_t *pgd) diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h index 4d3c9d00d6b6b..4a08a69b172ba 100644 --- a/arch/x86/include/asm/tlb.h +++ b/arch/x86/include/asm/tlb.h @@ -38,4 +38,13 @@ static inline void invlpg(unsigned long addr) { asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); } + +#ifdef CONFIG_PTP +#include +static inline void __ptp_tlb_remove_table(void *table) +{ + iee_cache_free(&pg_cache, page_to_virt((struct page *)table)); +} +#endif + #endif /* _ASM_X86_TLB_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 18ce2b759dbb8..4e9689479bfce 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1992,6 +1992,25 @@ static void text_poke_memset(void *dst, const void *src, size_t len) typedef void text_poke_f(void *dst, const void *src, size_t len); +#ifdef CONFIG_PTP +static inline void set_ptes_text_poke(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + page_table_check_ptes_set(mm, ptep, pte, nr); + + arch_enter_lazy_mmu_mode(); + for (;;) { + compiletime_assert_rwonce_type(*ptep); + ptp_set_pte_text_poke(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_next_pfn(pte); + } + arch_leave_lazy_mmu_mode(); +} +#endif + static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) { bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; @@ -2043,11 +2062,19 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l local_irq_save(flags); pte = mk_pte(pages[0], pgprot); +#ifdef CONFIG_PTP + set_ptes_text_poke(poking_mm, poking_addr, ptep, pte, 1); +#else set_pte_at(poking_mm, poking_addr, ptep, pte); +#endif if (cross_page_boundary) { pte = mk_pte(pages[1], pgprot); +#ifdef CONFIG_PTP + set_ptes_text_poke(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte, 1); +#else set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); +#endif } /* diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index a1b0b8e5c7162..18294d6ab7448 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -1,3 +1,4 @@ obj-y += haoc.o bitmap.o obj-y += iee/ +obj-$(CONFIG_PTP) += ptp/ obj-$(CONFIG_CREDP) += credp/ diff --git a/arch/x86/kernel/haoc/gate.h b/arch/x86/kernel/haoc/gate.h new file mode 100644 index 0000000000000..c3a8f4f24cf3c --- /dev/null +++ b/arch/x86/kernel/haoc/gate.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) + +/* + * scratch_reg would be changed, caller should dertimine if scratch_reg + * should be saved and restored. + */ +.macro DISABLE_WP scratch_reg:req + /* Disable write protection */ + movq %cr0, %\scratch_reg + andq $(~X86_CR0_WP), %\scratch_reg + movq %\scratch_reg, %cr0 +.endm + +.macro ENABLE_WP scratch_reg:req + /* Enable write protection */ + movq %cr0, %\scratch_reg +1: + orq $X86_CR0_WP, %\scratch_reg + movq %\scratch_reg, %cr0 + testq $X86_CR0_WP, %\scratch_reg + je 1b +.endm diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index c8f26ca7cf756..707ebb9454b05 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -58,6 +58,21 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_cred_security, (iee_func)_iee_set_cred_rcu, (iee_func)_iee_set_cred_ucounts, +#endif +#ifdef CONFIG_PTP +#ifdef CONFIG_IEE_PTRP + (iee_func)_iee_unset_token, + (iee_func)_iee_set_token, +#endif + (iee_func)_iee_set_pte, + (iee_func)_iee_set_pmd, + (iee_func)_iee_set_pud, + (iee_func)_iee_set_p4d, + (iee_func)_iee_set_pgd, + (iee_func)_iee_set_pte_text_poke, + (iee_func)_iee_set_pmd_ident, + (iee_func)_iee_set_pud_ident, + (iee_func)_iee_set_sx_pte, #endif NULL }; diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index dbea682f4bd86..bf23da605cba3 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -15,26 +15,26 @@ #endif extern bool haoc_enabled; -void set_iee_pages(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) +void set_iee_page(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) { set_memory_ro(addr, num_pages); iee_set_bitmap_type(__pa(addr), num_pages, type); } -void unset_iee_pages(unsigned long addr, int num_pages) +void unset_iee_page(unsigned long addr, int num_pages) { iee_set_bitmap_type(__pa(addr), num_pages, IEE_NORMAL); set_memory_rw(addr, num_pages); } -void set_iee_page(unsigned long addr, unsigned int order) +void set_iee_pages(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) { - set_memory_ro(addr, 1 << order); + set_iee_page(addr, num_pages, type); } -void unset_iee_page(unsigned long addr, unsigned int order) +void unset_iee_pages(unsigned long addr, int num_pages) { - set_memory_rw(addr, 1 << order); + unset_iee_page(addr, num_pages); } struct iee_free_slab_work { diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index 491978ab46917..bb0e2742dccaa 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -1,32 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#include -#include -#include -#include -#include - -#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) - -/* - * scratch_reg is clobbered, so callers must decide whether it should be - * saved and restored around these macros. - */ -.macro DISABLE_WP scratch_reg:req - /* Disable write protection. */ - movq %cr0, %\scratch_reg - andq $(~X86_CR0_WP), %\scratch_reg - movq %\scratch_reg, %cr0 -.endm - -.macro ENABLE_WP scratch_reg:req - /* Enable write protection. */ - movq %cr0, %\scratch_reg -1: - orq $X86_CR0_WP, %\scratch_reg - movq %\scratch_reg, %cr0 - testq $X86_CR0_WP, %\scratch_reg - je 1b -.endm +#include "../gate.h" SYM_FUNC_START(iee_rw_gate) cmpb $0, haoc_enabled(%rip) @@ -40,11 +13,25 @@ SYM_FUNC_START(iee_rw_gate) .L_haoc_enabled_rw: /* save Interrupt flag */ pushfq - /* close irq */ + /* close irq*/ cli pushq %r12 +#ifdef CONFIG_PTP + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz rw_wp_enabled + + leaq iee_funcs(%rip), %rax + call *(%rax, %rdi, 8) + + popq %r12 + popfq + jmp __x86_return_thunk + +rw_wp_enabled: +#endif DISABLE_WP r12 /* call iee func */ @@ -55,12 +42,14 @@ SYM_FUNC_START(iee_rw_gate) popq %r12 - /* restore irq */ + /* restore irq*/ popfq jmp __x86_return_thunk SYM_FUNC_END(iee_rw_gate) +#if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) EXPORT_SYMBOL(iee_rw_gate) +#endif #ifdef CONFIG_IEE_SIP SYM_FUNC_START(iee_rwx_gate) @@ -74,34 +63,58 @@ SYM_FUNC_START(iee_rwx_gate) .L_haoc_enabled_rwx: pushq %r12 - /* save Interrupt flag */ + /* save Interrupt flag*/ pushfq /* close irq */ cli +#ifdef CONFIG_PTP + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz rwx_wp_enabled + + movq %cr4, %rax /* rax -> cr4 */ + andq $(~X86_CR4_SMEP), %rax + movq %rax, %cr4 + + call _iee_si_handler + + movq %cr4, %rax /* rax -> cr4 */ +3: orq $X86_CR4_SMEP_SMAP, %rax + movq %rax, %cr4 + andq $X86_CR4_SMEP_SMAP, %rax + cmpq $X86_CR4_SMEP_SMAP, %rax + jnz 3 + + popfq + popq %r12 + jmp __x86_return_thunk /* ret */ + +rwx_wp_enabled: +#endif /* set SMEP=0 to enable supervisor-mode exec user-mode insn */ - movq %cr4, %rax + movq %cr4, %rax /* rax -> cr4 */ andq $(~X86_CR4_SMEP), %rax movq %rax, %cr4 DISABLE_WP r12 - call _iee_si_handler +2: call _iee_si_handler ENABLE_WP r12 /* set SMEP=1 to disable supervisor-mode exec user-mode insn */ - movq %cr4, %rax -1: orq $X86_CR4_SMEP_SMAP, %rax + movq %cr4, %rax /* rax -> cr4 */ +1: orq $X86_CR4_SMEP_SMAP, %rax movq %rax, %cr4 andq $X86_CR4_SMEP_SMAP, %rax cmpq $X86_CR4_SMEP_SMAP, %rax - jnz 1b + jnz 1 - /* restore irq */ + /* restore irq*/ popfq popq %r12 - jmp __x86_return_thunk + jmp __x86_return_thunk /* ret */ SYM_FUNC_END(iee_rwx_gate) #endif diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index cfac8a3333e46..f80cde9113e72 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -21,9 +21,16 @@ #ifdef CONFIG_IEE_SIP #include #endif +#ifdef CONFIG_PTP +#include +#include +#endif #ifdef CONFIG_CREDP extern void credp_init(void); #endif +#ifdef CONFIG_PTP +extern void haoc_ptp_init(void); +#endif /* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ unsigned long IEE_OFFSET = 0x200000000000; @@ -214,8 +221,19 @@ void __init iee_early_init(void) if (!cmd_haoc_enabled) return; - if (!check_haoc_hardware_support()) + if (!check_haoc_hardware_support()) { pr_info("HAOC disabled because SMAP or SMEP is not available"); + return; + } + +#ifdef CONFIG_PTP + iee_cache_init(&pgd_cache, PGD_ALLOCATION_ORDER, 1, IEE_PGTABLE, + CONFIG_PTP_RESERVE_ORDER); + if (pgtable_l5_enabled()) + iee_cache_init(&pg_cache, 0, 4, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); + else + iee_cache_init(&pg_cache, 0, 3, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); +#endif } void __init iee_init(void) @@ -250,6 +268,9 @@ void __init iee_post_init(void) return; iee_stack_set_ro(); +#ifdef CONFIG_PTP + haoc_ptp_init(); +#endif #ifdef CONFIG_IEE_SIP iee_sip_init(); #endif diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c index a4f8e3a82c38f..a25aeac727cc5 100644 --- a/arch/x86/kernel/haoc/iee/iee-token.c +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -25,6 +25,9 @@ void iee_set_token_page_valid(unsigned long token, unsigned long token_page, pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_SET_TOKEN, token_ptep, token_page_ptep, token_page, order); +#else for (int i = 0; i < (0x1 << order); i++) { pte_t pte = READ_ONCE(*token_ptep); @@ -38,6 +41,7 @@ void iee_set_token_page_valid(unsigned long token, unsigned long token_page, token_ptep++; token_page_ptep++; } +#endif flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); flush_tlb_kernel_range(token_page, @@ -65,6 +69,9 @@ void iee_set_token_page_invalid(unsigned long token, unsigned long __unused, pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_UNSET_TOKEN, token_ptep, token_page_ptep, token, order); +#else for (int i = 0; i < (0x1 << order); i++) { pte_t pte = READ_ONCE(*token_ptep); @@ -78,6 +85,7 @@ void iee_set_token_page_invalid(unsigned long token, unsigned long __unused, token_ptep++; token_page_ptep++; } +#endif free_pages(token_page, order); flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); flush_tlb_kernel_range(token_page, @@ -148,3 +156,45 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) #endif token->valid = true; } + +#ifdef CONFIG_PTP +void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token, unsigned int order) +{ + token_ptep = (pte_t *)__addr_to_iee(token_ptep); + token_page_ptep = (pte_t *)__addr_to_iee(token_page_ptep); + + for (int i = 0; i < (0x1 << order); i++) { + pte_t pte = READ_ONCE(*token_ptep); + + pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | + (__phys_to_pfn(__iee_pa(token + i * PAGE_SIZE)) << PAGE_SHIFT)); + WRITE_ONCE(*token_ptep, pte); + pte = READ_ONCE(*token_page_ptep); + pte = __pte(pte_val(pte) | ___D | __RW); + WRITE_ONCE(*token_page_ptep, pte); + token_ptep++; + token_page_ptep++; + } +} + +void _iee_set_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token_page, unsigned int order) +{ + token_ptep = (pte_t *)__addr_to_iee(token_ptep); + token_page_ptep = (pte_t *)__addr_to_iee(token_page_ptep); + + for (int i = 0; i < (0x1 << order); i++) { + pte_t pte = READ_ONCE(*token_ptep); + + pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK)) | + (__phys_to_pfn(__pa(token_page + i * PAGE_SIZE)) << PAGE_SHIFT)); + WRITE_ONCE(*token_ptep, pte); + pte = READ_ONCE(*token_page_ptep); + pte = __pte((pte_val(pte) & ~__RW) & ~___D); + WRITE_ONCE(*token_page_ptep, pte); + token_ptep++; + token_page_ptep++; + } +} +#endif diff --git a/arch/x86/kernel/haoc/ptp/Makefile b/arch/x86/kernel/haoc/ptp/Makefile new file mode 100644 index 0000000000000..ab30a58e0e739 --- /dev/null +++ b/arch/x86/kernel/haoc/ptp/Makefile @@ -0,0 +1 @@ +obj-y += ptp.o ptp-gate.o diff --git a/arch/x86/kernel/haoc/ptp/ptp-gate.S b/arch/x86/kernel/haoc/ptp/ptp-gate.S new file mode 100644 index 0000000000000..e56f184f909e0 --- /dev/null +++ b/arch/x86/kernel/haoc/ptp/ptp-gate.S @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include "../gate.h" + +SYM_FUNC_START(ptp_xchg) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_xchg + +.L_haoc_disabled_xchg: + xchg %rsi, (%rdi) + movq %rsi, %rax + jmp __x86_return_thunk + +.L_haoc_enabled_xchg: + /* save RFLAGS, close irq */ + pushfq + cli + + pushq %r12 + + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz xchg_wp_enabled + + xchg %rsi, (%rdi) + movq %rsi, %rax + + popq %r12 + popfq + jmp __x86_return_thunk + +xchg_wp_enabled: + DISABLE_WP r12 + + xchg %rsi, (%rdi) + movq %rsi, %rax + + ENABLE_WP r12 + + popq %r12 + + /* restore RFLAGS*/ + popfq + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(ptp_xchg) + +SYM_FUNC_START(ptp_try_cmpxchg) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_cmpxchg + +.L_haoc_disabled_cmpxchg: + movq %rsi, %rax + lock cmpxchgq %rdx, (%rdi) + jmp __x86_return_thunk + +.L_haoc_enabled_cmpxchg: + /* save RFLAGS, close irq */ + pushfq + cli + + pushq %r12 + + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz cmpxchg_wp_enabled + + movq %rsi, %rax + lock cmpxchgq %rdx, (%rdi) + + popq %r12 + popfq + jmp __x86_return_thunk + +cmpxchg_wp_enabled: + DISABLE_WP r12 + + movq %rsi, %rax + lock cmpxchgq %rdx, (%rdi) + + ENABLE_WP r12 + + popq %r12 + /* restore RFLAGS*/ + popfq + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(ptp_try_cmpxchg) + +SYM_FUNC_START(ptp_rw_gate) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_rw + +.L_haoc_disabled_rw: + leaq iee_funcs(%rip), %rax + call *(%rax, %rdi, 8) + jmp __x86_return_thunk + +.L_haoc_enabled_rw: + /* save Interrupt flag */ + pushfq + /* close irq*/ + cli + + pushq %r12 + + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz ptprw_wp_enabled + + leaq iee_funcs(%rip), %rax + call *(%rax, %rdi, 8) + + popq %r12 + popfq + jmp __x86_return_thunk + +ptprw_wp_enabled: + DISABLE_WP r12 + + /* call iee func */ + leaq iee_funcs(%rip), %rax + call *(%rax, %rdi, 8) + + ENABLE_WP r12 + + popq %r12 + + /* restore irq*/ + popfq + + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(ptp_rw_gate) +EXPORT_SYMBOL(ptp_rw_gate) diff --git a/arch/x86/kernel/haoc/ptp/ptp.c b/arch/x86/kernel/haoc/ptp/ptp.c new file mode 100644 index 0000000000000..2ead8feb18308 --- /dev/null +++ b/arch/x86/kernel/haoc/ptp/ptp.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define __pte_to_phys(pte) (pte_pfn(pte) << PAGE_SHIFT) +#define __pmd_to_phys(pmd) (__pte_to_phys(__pte(pmd_val(pmd)))) +#define __pud_to_phys(pud) (__pte_to_phys(__pte(pud_val(pud)))) +#define __p4d_to_phys(p4d) (__pte_to_phys(__pte(p4d_val(p4d)))) +#define __pgd_to_phys(pgd) (__pte_to_phys(__pte(pgd_val(pgd)))) + +static void __init _ptp_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end) +{ + pmd_t pmd = READ_ONCE(*pmdp); + unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); +} + +static void __init _ptp_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end) +{ + unsigned long next; + pud_t pud = READ_ONCE(*pudp); + pmd_t *pmdp; + pmd_t pmd; + unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); + pmdp = pmd_offset(pudp, addr); + do { + next = pmd_addr_end(addr, end); + pmd = READ_ONCE(*pmdp); + if (pmd_val(pmd) & _PSE) + continue; + else + _ptp_set_pte_table_ro(pmdp, addr, next); + } while (pmdp++, addr = next, addr != end); +} + +static void __init _ptp_set_pud_table_ro(p4d_t *p4dp, unsigned long addr, unsigned long end) +{ + unsigned long next; + p4d_t p4d = READ_ONCE(*p4dp); + pud_t *pudp; + pud_t pud; + unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); + pudp = pud_offset(p4dp, addr); + do { + next = pud_addr_end(addr, end); + pud = READ_ONCE(*pudp); + if (pud_val(pud) & _PSE) { + // _PSE = 1 means a page, not a table + continue; + } else { + _ptp_set_pmd_table_ro(pudp, addr, next); + } + } while (pudp++, addr = next, addr != end); +} + +static void __init _ptp_set_p4d_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end) +{ + unsigned long next; + pgd_t pgd = READ_ONCE(*pgdp); + p4d_t *p4dp; + p4d_t p4d; + unsigned long logical_addr = (unsigned long)__va(__pgd_to_phys(pgd)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); + p4dp = p4d_offset(pgdp, addr); + do { + next = p4d_addr_end(addr, end); + p4d = READ_ONCE(*p4dp); + /* No 512 GiB huge pages yet */ + _ptp_set_pud_table_ro(p4dp, addr, next); + } while (p4dp++, addr = next, addr != end); +} + +static void __init _ptp_mark_iee_pgtable_for_one_region_ro(pgd_t *pgdir, + unsigned long va_start, unsigned long va_end) +{ + unsigned long addr, end, next; + pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); + + addr = va_start & PAGE_MASK; + end = PAGE_ALIGN(va_end); + + do { + next = pgd_addr_end(addr, end); + _ptp_set_p4d_table_ro(pgdp, addr, next); + } while (pgdp++, addr = next, addr != end); +} + +static void __init ptp_mark_all_pgtable_ro(void) +{ + unsigned long logical_addr; + phys_addr_t start, end; + u64 i; + pgd_t *pgdp; + + // handing 1-level page table swapper_pg_dir + pgdp = swapper_pg_dir; + set_iee_page((unsigned long)swapper_pg_dir, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + + // handling 2/3/4/5-level page table for kernel + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)_text, (unsigned long)_etext); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__start_rodata, (unsigned long)__end_rodata); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)_sdata, (unsigned long)_edata); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__bss_start, (unsigned long)__bss_stop); + + // handling 2/3/4/5-level statically allocated page table + #ifdef CONFIG_X86_5LEVEL + set_iee_page((unsigned long)level4_kernel_pgt, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level4_kernel_pgt)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + #endif + + set_iee_page((unsigned long)level3_kernel_pgt, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level3_kernel_pgt)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + + set_iee_page((unsigned long)level2_kernel_pgt, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level2_kernel_pgt)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + + set_iee_page((unsigned long)level2_fixmap_pgt, FIXMAP_PMD_NUM, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level2_fixmap_pgt)); + set_iee_page(logical_addr, FIXMAP_PMD_NUM, IEE_PGTABLE); + + set_iee_page((unsigned long)level1_fixmap_pgt, FIXMAP_PMD_NUM, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level1_fixmap_pgt)); + set_iee_page(logical_addr, FIXMAP_PMD_NUM, IEE_PGTABLE); + + // handling 2/3/4-level page table for logical mem and iee + for_each_mem_range(i, &start, &end) { + if (start >= end) + break; + /* + * The linear map must allow allocation tags reading/writing + * if MTE is present. Otherwise, it has the same attributes as + * PAGE_KERNEL. + */ + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__va(start), (unsigned long)__va(end)); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); + } + pr_info("PTP: mark all kernel page tables ro done"); +} + +static inline void check_addr_range_ro(unsigned long addr_start, unsigned long addr_end, + unsigned long start, unsigned long end, pte_t pte, const char *msg) +{ + if (unlikely(!haoc_init_done)) + return; + + if (unlikely((addr_start >= start && addr_start < end) + || (addr_end > start && addr_end < end))) + if (unlikely(pte_write(pte))) + panic("IEE Error: Are you trying to write %s: [0x%lx, 0x%lx]?", + msg, addr_start, addr_end); +} + +static inline void check_dep_and_ro(unsigned long image_addr, unsigned long size, pte_t pte) +{ + if (unlikely(!haoc_init_done)) + return; + + if (unlikely(!kernel_set_to_readonly)) + return; + + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)_stext, (unsigned long)_etext, pte, "text"); + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); +} + +static inline bool preserves_existing_writable_mapping(pte_t *ptep, pte_t pte) +{ + pte_t old_pte = READ_ONCE(*ptep); + + return pte_present(old_pte) && pte_write(old_pte) && + pte_pfn(old_pte) == pte_pfn(pte); +} + +static inline bool +pmd_preserves_existing_writable_mapping(pmd_t *pmdp, pmd_t pmd) +{ + pmd_t old_pmd = READ_ONCE(*pmdp); + + return pmd_present(old_pmd) && pmd_leaf(old_pmd) && + pmd_write(old_pmd) && pmd_pfn(old_pmd) == pmd_pfn(pmd); +} + +static inline bool +pud_preserves_existing_writable_mapping(pud_t *pudp, pud_t pud) +{ + pud_t old_pud = READ_ONCE(*pudp); + + return pud_present(old_pud) && pud_leaf(old_pud) && + pud_write(old_pud) && pud_pfn(old_pud) == pud_pfn(pud); +} + +static inline void check_alias_mapping(phys_addr_t start, phys_addr_t end, + bool writable, + bool existing_writable_mapping) +{ + uint8_t type; + phys_addr_t addr; + + if (unlikely(!haoc_init_done)) + return; + + if (unlikely(PHYS_PFN(end) >= max_pfn)) + return; + + addr = start; + while (addr < end) { + type = iee_get_bitmap_type(addr); + if (unlikely(type != IEE_NORMAL && writable && + !existing_writable_mapping)) + panic("Detected Alias mapping of IEE data"); + addr += PAGE_SIZE; + } +} + +static inline int pte_smode(pte_t pte) +{ + return !(pte_flags(pte) & _PAGE_USER); +} + +static inline void check_sx(pte_t *ptep, pte_t pte) +{ + pte_t old_pte; + pteval_t diff; + + if (unlikely(!haoc_init_done)) + return; + old_pte = READ_ONCE(*ptep); + if (unlikely(!pte_present(old_pte))) + return; + + diff = pte_flags(old_pte) ^ pte_flags(pte); + if (pte_exec(pte) && pte_smode(pte) && + ((diff & _PAGE_USER) || (diff & _PAGE_NX))) { + pr_err("PTP: old_pte=%lx, pte=%lx\n", pte_flags(old_pte), pte_flags(pte)); + pr_err("PTP: diff=%lx\n", diff); + panic("Detected new S-page with execute permission"); + } + if (pte_exec(pte) && pte_smode(pte) && pte_write(pte)) + panic("Detected SX-page with write permission"); +} + +static inline void check_pt_integrity(pte_t *ptep, pte_t pte) +{ + pte_t old_pte; + + if (unlikely(!haoc_init_done)) + return; + if (iee_get_bitmap_type(__pa(ptep)) != IEE_FIXED_PGTABLE) + return; + old_pte = READ_ONCE(*ptep); + if (pte_pfn(old_pte) != pte_pfn(pte)) + panic("PTP: Are you trying to break pt integrity? old_pte: 0x%lx, new_pte: 0x%lx", + old_pte.pte, pte.pte); +} + +void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) { + WRITE_ONCE(*ptep, pte); + return; + } + + phys_addr_t phys_addr = __pte_to_phys(pte); + unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; + + check_dep_and_ro(image_addr, PAGE_SIZE, pte); + check_alias_mapping(phys_addr, phys_addr + PAGE_SIZE, pte_write(pte), + preserves_existing_writable_mapping(ptep, pte)); + check_sx(ptep, pte); + check_pt_integrity(ptep, pte); + WRITE_ONCE(*ptep, pte); +} + +void _iee_set_sx_pte(unsigned long __unused, pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) { + WRITE_ONCE(*ptep, pte); + return; + } + + phys_addr_t phys_addr = __pte_to_phys(pte); + unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; + + check_dep_and_ro(image_addr, PAGE_SIZE, pte); + + // FIXME: Writing the PTE during page split may create temporary alias mappings, + // so we cannot perform the check here. + // A different approach is needed in the future. + // check_alias_mapping(phys_addr, phys_addr + PAGE_SIZE, ptep, pte); + WRITE_ONCE(*ptep, pte); +} + +void _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pmd_val(pmd) & _PAGE_PRESENT)) { + WRITE_ONCE(*pmdp, pmd); + return; + } + if (pmd_leaf(pmd)) { + phys_addr = __pmd_to_phys(pmd); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PMD_SIZE, __pte(pmd_val(pmd))); + check_alias_mapping(phys_addr, phys_addr + PMD_SIZE, + pmd_write(pmd), + pmd_preserves_existing_writable_mapping(pmdp, pmd)); + check_sx((pte_t *)pmdp, __pte(pmd_val(pmd))); + } + + WRITE_ONCE(*pmdp, pmd); +} + +void _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pud_val(pud) & _PAGE_PRESENT)) { + WRITE_ONCE(*pudp, pud); + return; + } + if (pud_leaf(pud)) { + phys_addr = __pud_to_phys(pud); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PUD_SIZE, __pte(pud_val(pud))); + check_alias_mapping(phys_addr, phys_addr + PUD_SIZE, + pud_write(pud), + pud_preserves_existing_writable_mapping(pudp, pud)); + check_sx((pte_t *)pudp, __pte(pud_val(pud))); + } + + WRITE_ONCE(*pudp, pud); +} + +void _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d) +{ + WRITE_ONCE(*p4dp, p4d); +} + +void _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd) +{ + WRITE_ONCE(*pgdp, pgd); +} + +static inline void check_text_poke_ro(unsigned long image_addr, unsigned long size, pte_t pte) +{ + if (unlikely(!kernel_set_to_readonly)) + return; + + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); +} + +void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) { + WRITE_ONCE(*ptep, pte); + return; + } + + phys_addr_t phys_addr = __pte_to_phys(pte); + unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; + + check_text_poke_ro(image_addr, PAGE_SIZE, pte); + check_alias_mapping(phys_addr, phys_addr + PAGE_SIZE, pte_write(pte), + preserves_existing_writable_mapping(ptep, pte)); + WRITE_ONCE(*ptep, pte); +} + +void _iee_set_pmd_ident(unsigned long __unused, pmd_t *pmdp, pmd_t pmd) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pmd_val(pmd) & _PAGE_PRESENT)) { + WRITE_ONCE(*pmdp, pmd); + return; + } + if (pmd_leaf(pmd)) { + phys_addr = __pmd_to_phys(pmd); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PMD_SIZE, __pte(pmd_val(pmd))); + } + + WRITE_ONCE(*pmdp, pmd); +} + +void _iee_set_pud_ident(unsigned long __unused, pud_t *pudp, pud_t pud) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pud_val(pud) & _PAGE_PRESENT)) { + WRITE_ONCE(*pudp, pud); + return; + } + if (pud_leaf(pud)) { + phys_addr = __pud_to_phys(pud); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PUD_SIZE, __pte(pud_val(pud))); + } + + WRITE_ONCE(*pudp, pud); +} + +DEFINE_PER_CPU(struct iee_cr0, iee_cr0s); + +void __init haoc_ptp_init(void) +{ + int cpu; + struct iee_cr0 *iee_cr0; + + for_each_possible_cpu(cpu) { + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + iee_cr0->wp_disabled_cnt = 0; + } + + ptp_mark_all_pgtable_ro(); +} + +void ptp_disable_wp(unsigned long *cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + if (iee_cr0->wp_disabled_cnt == 0) { + *cr0 = read_cr0(); + asm volatile("mov %0, %%cr0" : : "r"(*cr0 & ~X86_CR0_WP) : "memory"); + } + iee_cr0->wp_disabled_cnt++; + put_cpu(); + local_irq_restore(irq_flags); +} + +void ptp_restore_wp(unsigned long cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + iee_cr0->wp_disabled_cnt--; + if (iee_cr0->wp_disabled_cnt == 0) + asm volatile("mov %0, %%cr0" : : "r"(cr0 | X86_CR0_WP) : "memory"); + put_cpu(); + local_irq_restore(irq_flags); +} + +void ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + *wp_disabled_cnt = iee_cr0->wp_disabled_cnt; + if (*wp_disabled_cnt > 0) { + *cr0 = read_cr0(); + iee_cr0->wp_disabled_cnt = 0; + asm volatile("mov %0, %%cr0" : : "r"(*cr0 | X86_CR0_WP) : "memory"); + } + put_cpu(); + local_irq_restore(irq_flags); +} + +void ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + if (wp_disabled_cnt > 0) { + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + asm volatile("mov %0, %%cr0" : : "r"(cr0 & ~X86_CR0_WP) : "memory"); + iee_cr0->wp_disabled_cnt = wp_disabled_cnt; + put_cpu(); + } + local_irq_restore(irq_flags); +} diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 35fd5f1444fdb..a3e8d44247dfc 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -35,6 +35,13 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#include +#endif +#ifdef CONFIG_PTP +#include +#endif #define CREATE_TRACE_POINTS #include @@ -364,6 +371,24 @@ static noinstr void default_do_nmi(struct pt_regs *regs) int handled; bool b2b = false; +#ifdef CONFIG_IEE + u64 cr0 = read_cr0(); + bool cr0_wp = cr0 & X86_CR0_WP; +#ifdef CONFIG_PTP + u64 iee_wp_flag = 0; +#endif + + if (!cr0_wp) { +#ifdef CONFIG_PTP + iee_wp_flag = this_cpu_read(iee_cr0s.wp_disabled_cnt); + this_cpu_write(iee_cr0s.wp_disabled_cnt, 0); +#endif + cr0 |= X86_CR0_WP; + // we assume that only IEE gate can disable cr0.wp. + // If cr0.wp = false, set cr0.wp = true in case of nmi happens in IEE gate. + asm volatile("mov %0,%%cr0" : "+r" (cr0) : : "memory"); + } +#endif /* * CPU-specific NMI must be processed before non-CPU-specific * NMI, otherwise we may lose it, because the CPU-specific @@ -475,6 +500,17 @@ static noinstr void default_do_nmi(struct pt_regs *regs) out: instrumentation_end(); +#ifdef CONFIG_IEE + if (!cr0_wp) { + cr0 = read_cr0() & ~X86_CR0_WP; + + asm volatile("mov %0,%%cr0" : "+r" (cr0) : : "memory"); + +#ifdef CONFIG_PTP + this_cpu_write(iee_cr0s.wp_disabled_cnt, iee_wp_flag); +#endif + } +#endif } /* diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 234851fe0ef8e..51aab86e48b9a 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -81,7 +81,11 @@ void __init native_pv_lock_init(void) static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) { +#ifdef CONFIG_PTP + ptp_tlb_remove_page(tlb, table); +#else tlb_remove_page(tlb, table); +#endif } unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c index fe0b2e66ded93..174e0e06d44a2 100644 --- a/arch/x86/mm/ident_map.c +++ b/arch/x86/mm/ident_map.c @@ -14,7 +14,11 @@ static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, if (pmd_present(*pmd)) continue; +#ifdef CONFIG_PTP + ptp_set_pmd_ident(pmd, __pmd((addr - info->offset) | info->page_flag)); +#else set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag)); +#endif } } @@ -52,7 +56,11 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, pud_t pudval; pudval = __pud((addr - info->offset) | info->page_flag); +#ifdef CONFIG_PTP + ptp_set_pud_ident(pud, pudval); +#else set_pud(pud, pudval); +#endif continue; } @@ -65,7 +73,11 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, if (!pmd) return -ENOMEM; ident_pmd_init(info, pmd, addr, next); +#ifdef CONFIG_PTP + ptp_set_pud_ident(pud, __pud(__pa(pmd) | info->kernpg_flag)); +#else set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag)); +#endif } return 0; diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 525794f1eefb3..f39bc18d8daf4 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -835,7 +835,11 @@ EXPORT_SYMBOL_GPL(slow_virt_to_phys); static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) { /* change init_mm */ +#ifdef CONFIG_PTP + ptp_set_sx_pte(kpte, pte); +#else set_pte_atomic(kpte, pte); +#endif #ifdef CONFIG_X86_32 if (!SHARED_KERNEL_PMD) { struct page *page; @@ -1068,7 +1072,11 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, else pr_warn_once("CPA: Cannot fixup static protections for PUD split\n"); set: +#ifdef CONFIG_PTP + ptp_set_sx_pte(pte, pfn_pte(pfn, ref_prot)); +#else set_pte(pte, pfn_pte(pfn, ref_prot)); +#endif } static int @@ -1670,7 +1678,11 @@ static int __change_page_attr(struct cpa_data *cpa, int primary) * Do we really change anything ? */ if (pte_val(old_pte) != pte_val(new_pte)) { +#ifdef CONFIG_PTP + ptp_set_sx_pte(kpte, new_pte); +#else set_pte_atomic(kpte, new_pte); +#endif cpa->flags |= CPA_FLUSHTLB; } cpa->numpages = 1; diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index cada0beeee76c..c90fe90119af1 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -6,6 +6,10 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#include +#endif #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; @@ -418,15 +422,29 @@ static inline void _pgd_free(pgd_t *pgd) } #else +#ifdef CONFIG_PTP +struct iee_cache pgd_cache = { + .object_order = PGD_ALLOCATION_ORDER, +}; +#endif + static inline pgd_t *_pgd_alloc(void) { +#ifdef CONFIG_PTP + return (pgd_t *)iee_cache_alloc(&pgd_cache, GFP_PGTABLE_USER); +#else return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, PGD_ALLOCATION_ORDER); +#endif } static inline void _pgd_free(pgd_t *pgd) { +#ifdef CONFIG_PTP + iee_cache_free(&pgd_cache, pgd); +#else free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); +#endif } #endif /* CONFIG_X86_PAE */ @@ -560,8 +578,13 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, int ret = 0; if (pte_young(*ptep)) +#ifdef CONFIG_PTP + ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *) &ptep->pte); +#else ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, (unsigned long *) &ptep->pte); +#endif return ret; } @@ -573,8 +596,13 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, int ret = 0; if (pmd_young(*pmdp)) +#ifdef CONFIG_PTP + ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pmdp); +#else ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, (unsigned long *)pmdp); +#endif return ret; } @@ -587,8 +615,13 @@ int pudp_test_and_clear_young(struct vm_area_struct *vma, int ret = 0; if (pud_young(*pudp)) +#ifdef CONFIG_PTP + ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pudp); +#else ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, (unsigned long *)pudp); +#endif return ret; } @@ -838,14 +871,22 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) for (i = 0; i < PTRS_PER_PMD; i++) { if (!pmd_none(pmd_sv[i])) { pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pte); +#else free_page((unsigned long)pte); +#endif } } free_page((unsigned long)pmd_sv); pagetable_pmd_dtor(virt_to_ptdesc(pmd)); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pmd); +#else free_page((unsigned long)pmd); +#endif return 1; } @@ -868,7 +909,11 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) /* INVLPG to clear all paging-structure caches */ flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pte); +#else free_page((unsigned long)pte); +#endif return 1; } diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index 916247c1d2a36..a47af6f9d6775 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -139,7 +139,11 @@ pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) * The user page tables get the full PGD, accessible from * userspace: */ +#ifdef CONFIG_PTP + ptp_set_pgd(kernel_to_user_pgdp(pgdp), pgd); +#else kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; +#endif /* * If this is normal user memory, make it NX in the kernel @@ -292,7 +296,11 @@ static void __init pti_setup_vsyscall(void) if (WARN_ON(!target_pte)) return; +#ifdef CONFIG_PTP + set_pte(target_pte, *pte); +#else *target_pte = *pte; +#endif set_vsyscall_pgtable_user_bits(kernel_to_user_pgdp(swapper_pg_dir)); } #else @@ -370,14 +378,22 @@ pti_clone_pgtable(unsigned long start, unsigned long end, * code that only set this bit when supported. */ if (boot_cpu_has(X86_FEATURE_PGE)) +#ifdef CONFIG_PTP + set_pmd(pmd, pmd_set_flags(*pmd, _PAGE_GLOBAL)); +#else *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL); +#endif /* * Copy the PMD. That is, the kernelmode and usermode * tables will share the last-level page tables of this * address range */ +#ifdef CONFIG_PTP + set_pmd(target_pmd, *pmd); +#else *target_pmd = *pmd; +#endif addr = round_up(addr + 1, PMD_SIZE); @@ -401,10 +417,18 @@ pti_clone_pgtable(unsigned long start, unsigned long end, /* Set GLOBAL bit in both PTEs */ if (boot_cpu_has(X86_FEATURE_PGE)) +#ifdef CONFIG_PTP + set_pte(pte, pte_set_flags(*pte, _PAGE_GLOBAL)); +#else *pte = pte_set_flags(*pte, _PAGE_GLOBAL); +#endif /* Clone the PTE */ +#ifdef CONFIG_PTP + set_pte(target_pte, *pte); +#else *target_pte = *pte; +#endif addr = round_up(addr + 1, PAGE_SIZE); @@ -430,7 +454,11 @@ static void __init pti_clone_p4d(unsigned long addr) kernel_pgd = pgd_offset_k(addr); kernel_p4d = p4d_offset(kernel_pgd, addr); +#ifdef CONFIG_PTP + set_p4d(user_p4d, *kernel_p4d); +#else *user_p4d = *kernel_p4d; +#endif } /* @@ -461,7 +489,11 @@ static void __init pti_clone_user_shared(void) if (WARN_ON(!target_pte)) return; +#ifdef CONFIG_PTP + set_pte(target_pte, pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL)); +#else *target_pte = pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL); +#endif } } diff --git a/arch/x86/mm/ptp_ident_map.c b/arch/x86/mm/ptp_ident_map.c new file mode 100644 index 0000000000000..bd2e3e6c82b67 --- /dev/null +++ b/arch/x86/mm/ptp_ident_map.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Helper routines for building identity mapping page tables. This is + * included by both the compressed kernel and the regular kernel. + */ + +static inline void ptp_set_pte_pre_init(pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*ptep, pte); +} + +static inline void ptp_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) +{ + WRITE_ONCE(*pmdp, pmd); +} + +static inline void ptp_set_pud_pre_init(pud_t *pudp, pud_t pud) +{ + WRITE_ONCE(*pudp, pud); +} + +static inline void ptp_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) +{ + pgd_t pgd; + + if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { + WRITE_ONCE(*p4dp, p4d); + return; + } + + pgd = native_make_pgd(native_p4d_val(p4d)); + pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); + WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); +} + +static inline void ptp_set_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ + WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); +} + +static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, + unsigned long addr, unsigned long end) +{ + addr &= PMD_MASK; + for (; addr < end; addr += PMD_SIZE) { + pmd_t *pmd = pmd_page + pmd_index(addr); + + if (pmd_present(*pmd)) + continue; + + ptp_set_pmd_pre_init(pmd, __pmd((addr - info->offset) | info->page_flag)); + } +} + +static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, + unsigned long addr, unsigned long end) +{ + unsigned long next; + + for (; addr < end; addr = next) { + pud_t *pud = pud_page + pud_index(addr); + pmd_t *pmd; + + next = (addr & PUD_MASK) + PUD_SIZE; + if (next > end) + next = end; + + if (info->direct_gbpages) { + pud_t pudval; + + if (pud_present(*pud)) + continue; + + addr &= PUD_MASK; + pudval = __pud((addr - info->offset) | info->page_flag); + ptp_set_pud_pre_init(pud, pudval); + continue; + } + + if (pud_present(*pud)) { + pmd = pmd_offset(pud, 0); + ident_pmd_init(info, pmd, addr, next); + continue; + } + pmd = (pmd_t *)info->alloc_pgt_page(info->context); + if (!pmd) + return -ENOMEM; + ident_pmd_init(info, pmd, addr, next); + ptp_set_pud_pre_init(pud, __pud(__pa(pmd) | info->kernpg_flag)); + } + + return 0; +} + +static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, + unsigned long addr, unsigned long end) +{ + unsigned long next; + int result; + + for (; addr < end; addr = next) { + p4d_t *p4d = p4d_page + p4d_index(addr); + pud_t *pud; + + next = (addr & P4D_MASK) + P4D_SIZE; + if (next > end) + next = end; + + if (p4d_present(*p4d)) { + pud = pud_offset(p4d, 0); + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + + continue; + } + pud = (pud_t *)info->alloc_pgt_page(info->context); + if (!pud) + return -ENOMEM; + + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + + ptp_set_p4d_pre_init(p4d, __p4d(__pa(pud) | info->kernpg_flag)); + } + + return 0; +} + +int ptp_kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, + unsigned long pstart, unsigned long pend) +{ + unsigned long addr = pstart + info->offset; + unsigned long end = pend + info->offset; + unsigned long next; + int result; + + /* Set the default pagetable flags if not supplied */ + if (!info->kernpg_flag) + info->kernpg_flag = _KERNPG_TABLE; + + /* Filter out unsupported __PAGE_KERNEL_* bits: */ + info->kernpg_flag &= __default_kernel_pte_mask; + + for (; addr < end; addr = next) { + pgd_t *pgd = pgd_page + pgd_index(addr); + p4d_t *p4d; + + next = (addr & PGDIR_MASK) + PGDIR_SIZE; + if (next > end) + next = end; + + if (pgd_present(*pgd)) { + p4d = p4d_offset(pgd, 0); + result = ident_p4d_init(info, p4d, addr, next); + if (result) + return result; + continue; + } + + p4d = (p4d_t *)info->alloc_pgt_page(info->context); + if (!p4d) + return -ENOMEM; + result = ident_p4d_init(info, p4d, addr, next); + if (result) + return result; + if (pgtable_l5_enabled()) { + ptp_set_pgd_pre_init(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); + } else { + /* + * With p4d folded, pgd is equal to p4d. + * The pgd entry has to point to the pud page table in this case. + */ + pud_t *pud = pud_offset(p4d, 0); + + ptp_set_pgd_pre_init(pgd, __pgd(__pa(pud) | info->kernpg_flag)); + } + } + + return 0; +} diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 91d31ac422d6c..89de70744406e 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -48,6 +48,10 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#include +#endif /* * We allocate runtime services regions top-down, starting from -4G, i.e. @@ -73,7 +77,11 @@ int __init efi_alloc_page_tables(void) gfp_t gfp_mask; gfp_mask = GFP_KERNEL | __GFP_ZERO; +#ifdef CONFIG_PTP + efi_pgd = iee_cache_alloc(&pgd_cache, gfp_mask); +#else efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER); +#endif if (!efi_pgd) goto fail; @@ -116,7 +124,11 @@ void efi_sync_low_kernel_mappings(void) pgd_k = pgd_offset_k(PAGE_OFFSET); num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET); +#ifdef CONFIG_PTP + iee_memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); +#else memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); +#endif pgd_efi = efi_pgd + pgd_index(EFI_VA_END); pgd_k = pgd_offset_k(EFI_VA_END); @@ -124,7 +136,11 @@ void efi_sync_low_kernel_mappings(void) p4d_k = p4d_offset(pgd_k, 0); num_entries = p4d_index(EFI_VA_END); +#ifdef CONFIG_PTP + iee_memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); +#else memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); +#endif /* * We share all the PUD entries apart from those that map the @@ -139,13 +155,21 @@ void efi_sync_low_kernel_mappings(void) pud_k = pud_offset(p4d_k, 0); num_entries = pud_index(EFI_VA_END); +#ifdef CONFIG_PTP + iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#else memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#endif pud_efi = pud_offset(p4d_efi, EFI_VA_START); pud_k = pud_offset(p4d_k, EFI_VA_START); num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START); +#ifdef CONFIG_PTP + iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#else memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#endif } /* diff --git a/fs/exec.c b/fs/exec.c index 3d1b405d24d41..9466a12eb527c 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -84,6 +84,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_PTP +#include +#endif static int bprm_creds_from_file(struct linux_binprm *bprm); @@ -752,7 +755,11 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) free_pgd_range(&tlb, old_start, old_end, new_end, next ? next->vm_start : USER_PGTABLES_CEILING); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_finish_mmu(&tlb); +#else tlb_finish_mmu(&tlb); +#endif vma_prev(&vmi); /* Shrink the vma to just the new range */ @@ -1957,10 +1964,17 @@ static int do_execveat_common(int fd, struct filename *filename, { struct linux_binprm *bprm; int retval; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif if (IS_ERR(filename)) return PTR_ERR(filename); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif + /* * We move the actual failure in case of RLIMIT_NPROC excess from * set*uid() to execve() because too many poorly written programs @@ -2032,6 +2046,9 @@ static int do_execveat_common(int fd, struct filename *filename, out_ret: putname(filename); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif return retval; } diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 681d25fe36174..fae22f47535fe 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -210,6 +210,9 @@ struct mmu_table_batch { ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) extern void tlb_remove_table(struct mmu_gather *tlb, void *table); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +extern void ptp_tlb_remove_table(struct mmu_gather *tlb, void *table); +#endif #else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */ @@ -218,6 +221,9 @@ extern void tlb_remove_table(struct mmu_gather *tlb, void *table); * page directories and we can use the normal page batching to free them. */ #define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page)) +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +#define ptp_tlb_remove_table(tlb, page) ptp_tlb_remove_page((tlb), (page)) +#endif #endif /* CONFIG_MMU_GATHER_TABLE_FREE */ @@ -376,6 +382,9 @@ struct mmu_gather { }; void tlb_flush_mmu(struct mmu_gather *tlb); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +void ptp_tlb_flush_mmu(struct mmu_gather *tlb); +#endif static inline void __tlb_adjust_range(struct mmu_gather *tlb, unsigned long address, @@ -492,6 +501,15 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb, tlb_flush_mmu(tlb); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static inline void ptp_tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + if (__tlb_remove_page_size(tlb, page, false, page_size)) + ptp_tlb_flush_mmu(tlb); +} +#endif + static __always_inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page, bool delay_rmap) { @@ -507,15 +525,30 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) return tlb_remove_page_size(tlb, page, PAGE_SIZE); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static inline void ptp_tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + return ptp_tlb_remove_page_size(tlb, page, PAGE_SIZE); +} +#endif + static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) { +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_remove_table(tlb, pt); +#else tlb_remove_table(tlb, pt); +#endif } /* Like tlb_remove_ptdesc, but for page-like page directories. */ static inline void tlb_remove_page_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) { +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_remove_table(tlb, ptdesc_page(pt)); +#else tlb_remove_page(tlb, ptdesc_page(pt)); +#endif } static inline void tlb_change_page_size(struct mmu_gather *tlb, diff --git a/include/linux/haoc-ptp.h b/include/linux/haoc-ptp.h new file mode 100644 index 0000000000000..29b4d6b4eaeea --- /dev/null +++ b/include/linux/haoc-ptp.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HAOC_PTP_H +#define _LINUX_HAOC_PTP_H + +extern void ptp_disable_wp(unsigned long *cr0); +extern void ptp_restore_wp(unsigned long cr0); +extern void ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0); +extern void ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0); + +#endif /* _LINUX_HAOC_PTP_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 7e2f930539d94..bb6ef72d75d4f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -31,6 +31,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif struct mempolicy; struct anon_vma; @@ -2933,7 +2936,15 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) */ static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) { +#ifdef CONFIG_PTP + struct page *page; + void *pt_page; + + pt_page = iee_cache_alloc(&pg_cache, gfp); + page = !!pt_page ? virt_to_page(pt_page) : NULL; +#else struct page *page = alloc_pages(gfp | __GFP_COMP, order); +#endif return page_ptdesc(page); } @@ -2949,7 +2960,11 @@ static inline void pagetable_free(struct ptdesc *pt) { struct page *page = ptdesc_page(pt); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, page_address(page)); +#else __free_pages(page, compound_order(page)); +#endif } #if USE_SPLIT_PTE_PTLOCKS diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c6878e5c39684..d87c7ac08e7e4 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1259,6 +1259,9 @@ extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma); extern void tlb_finish_mmu(struct mmu_gather *tlb); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +extern void ptp_tlb_finish_mmu(struct mmu_gather *tlb); +#endif struct vm_fault; diff --git a/include/linux/ptp-cache.h b/include/linux/ptp-cache.h new file mode 100644 index 0000000000000..928f0191473bb --- /dev/null +++ b/include/linux/ptp-cache.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PTP_CACHE_H +#define _LINUX_PTP_CACHE_H +#include +#include +union freelist_aba_t { + struct { + void **freelist; + unsigned long counter; + }; + u128 full; +}; + +struct iee_cache { + union { + struct { + void **freelist; + unsigned long tid; + }; + union freelist_aba_t freelist_tid; + }; + unsigned long reserve_order; + unsigned long reserve_start_addr; + unsigned long reserve_end_addr; + unsigned long object_order; + int levels; + enum HAOC_BITMAP_TYPE name; + atomic_t count; + atomic_t fail_count; + bool init; +}; + +extern struct iee_cache pg_cache; +extern void iee_cache_init(struct iee_cache *cache, unsigned long object_order, + int levels, enum HAOC_BITMAP_TYPE bitmap_type, + unsigned long reserve_order); +extern void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp); +extern void iee_cache_free(struct iee_cache *cache, void *object); +#endif diff --git a/kernel/fork.c b/kernel/fork.c index ec9c97caa8f21..ec554539b3d05 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -117,6 +117,9 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_PTP +#include +#endif /* * Minimum number of threads to boot the kernel @@ -2917,6 +2920,11 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) return copy_process(NULL, 0, node, &args); } +#ifdef CONFIG_PTP +void __weak ptp_disable_wp(unsigned long *cr0) { } +void __weak ptp_restore_wp(unsigned long cr0) { } +#endif + /* * Ok, this is the main fork-routine. * @@ -2933,6 +2941,9 @@ pid_t kernel_clone(struct kernel_clone_args *args) struct task_struct *p; int trace = 0; pid_t nr; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif /* * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument @@ -2966,7 +2977,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) trace = 0; } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif p = copy_process(NULL, trace, NUMA_NO_NODE, args); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif add_latent_entropy(); if (IS_ERR(p)) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3b1530044f4f6..f358980bff455 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -100,6 +100,9 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_PTP +#include +#endif EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); @@ -5164,6 +5167,11 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) calculate_sigpending(); } +#ifdef CONFIG_PTP +void __weak ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0) { } +void __weak ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0) { } +#endif + /* * context_switch - switch to the new MM and the new thread's register state. */ @@ -5227,10 +5235,22 @@ context_switch(struct rq *rq, struct task_struct *prev, prepare_lock_switch(rq, next, rf); + /* Here we just switch the register state and the stack. */ +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + int wp_disabled_cnt = 0; + unsigned long cr0 = 0; + + ptp_context_enable_wp(&wp_disabled_cnt, &cr0); +#endif + /* Here we just switch the register state and the stack. */ switch_to(prev, next, prev); barrier(); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_context_restore_wp(wp_disabled_cnt, cr0); +#endif + return finish_task_switch(prev); } diff --git a/mm/Makefile b/mm/Makefile index 92060644c70ee..bc67de29f6c8b 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -54,6 +54,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ compaction.o show_mem.o shmem_quota.o\ interval_tree.o list_lru.o workingset.o \ debug.o gup.o mmap_lock.o $(mmu-y) +obj-$(CONFIG_PTP) += haoc/ # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 4470336b9cab5..0036ab1a396d6 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -437,7 +437,11 @@ static void __init pmd_huge_tests(struct pgtable_debug_args *args) * X86 defined pmd_set_huge() verifies that the given * PMD is not a populated non-leaf entry. */ +#ifdef CONFIG_PTP + set_pmd(args->pmdp, __pmd(0)); +#else WRITE_ONCE(*args->pmdp, __pmd(0)); +#endif WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); WARN_ON(!pmd_clear_huge(args->pmdp)); pmd = READ_ONCE(*args->pmdp); @@ -457,7 +461,11 @@ static void __init pud_huge_tests(struct pgtable_debug_args *args) * X86 defined pud_set_huge() verifies that the given * PUD is not a populated non-leaf entry. */ +#ifdef CONFIG_PTP + set_pud(args->pudp, __pud(0)); +#else WRITE_ONCE(*args->pudp, __pud(0)); +#endif WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); WARN_ON(!pud_clear_huge(args->pudp)); pud = READ_ONCE(*args->pudp); diff --git a/mm/haoc/Makefile b/mm/haoc/Makefile new file mode 100644 index 0000000000000..ad37f99f562d2 --- /dev/null +++ b/mm/haoc/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_PTP) += ptp-pg_cache.o diff --git a/mm/haoc/ptp-pg_cache.c b/mm/haoc/ptp-pg_cache.c new file mode 100644 index 0000000000000..6351dbe60904d --- /dev/null +++ b/mm/haoc/ptp-pg_cache.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct iee_cache pg_cache; + +#ifdef CONFIG_X86_64 +static LIST_HEAD(iee_cache_pending_protect_list); +static DEFINE_SPINLOCK(iee_cache_pending_protect_lock); +static atomic_t iee_cache_pending_protect_count = ATOMIC_INIT(0); + +enum iee_cache_deferred_op { + IEE_CACHE_DEFER_PROTECT, + IEE_CACHE_DEFER_FREE, +}; + +struct iee_cache_deferred_work { + struct work_struct work; + struct list_head list; + void *object; + unsigned int order; + int num_pages; + enum HAOC_BITMAP_TYPE type; + enum iee_cache_deferred_op op; + bool free_after_protect; +}; + +static void *iee_cache_alloc_pending_protect(struct iee_cache *cache, gfp_t gfp); +static bool iee_cache_mark_pending_protect_free(void *object); +#endif + +static inline void *__get_freepointer(void *object) +{ + return *((void **)object); +}; + +static inline void __set_freepointer(void *object, void *next_object) +{ + *(void **)object = next_object; +}; + +static inline void __iee_set_freepointer(void *object, void *next_object) +{ + iee_set_freeptr(object, next_object); +}; + +static inline bool __update_freelist(struct iee_cache *cache, + void *freelist_old, void *freelist_new, + unsigned long tid) +{ + union freelist_aba_t old = { .freelist = freelist_old, .counter = tid }; + union freelist_aba_t new = { .freelist = freelist_new, .counter = tid + 1 }; + + return try_cmpxchg128(&(cache->freelist_tid.full), &old.full, new.full); +} + +#ifdef CONFIG_ARM64 +static void __ptp_set_iee_pages(unsigned long start_addr, unsigned long end_addr, + struct iee_cache *cache) +{ + unsigned long addr; + + if (start_addr != ALIGN(start_addr, PMD_SIZE)) + panic("IEE: pool (HAOC_BITMAP_TYPE %u) not PMD-aligned.", + (unsigned int)cache->name); + + addr = start_addr; + while (addr < end_addr) { + set_iee_page(addr, PMD_ORDER, cache->name); + addr += PMD_SIZE; + } + flush_tlb_kernel_range(start_addr, end_addr); +} +#endif + +void __init iee_cache_init(struct iee_cache *cache, unsigned long object_order, + int levels, enum HAOC_BITMAP_TYPE bitmap_type, + unsigned long reserve_order) +{ + unsigned long addr; + unsigned long addr_next; + unsigned long reserve_pages; + unsigned long start_addr; + unsigned long end_addr; + unsigned long object_size; + struct page *page; + + object_size = (1 << object_order) * PAGE_SIZE; + while (1) { + reserve_pages = (1 << reserve_order) * levels; + start_addr = (unsigned long)memblock_alloc(reserve_pages * PAGE_SIZE, + reserve_pages * PAGE_SIZE); + if (start_addr) + break; + reserve_order--; + /* Allocate pages in pmd blocks to reduce the mapping cost. */ + if (reserve_order < PMD_ORDER) + panic("IEE: fail to reserve pages for HAOC_BITMAP_TYPE %u", + (unsigned int)bitmap_type); + } + end_addr = start_addr + reserve_pages * PAGE_SIZE; + pr_err("IEE: reserve %lu pages for HAOC_BITMAP_TYPE %u, range[0x%lx, 0x%lx]", + reserve_pages, (unsigned int)bitmap_type, start_addr, end_addr); + + addr = start_addr; + addr_next = addr + object_size; + while (addr < end_addr) { + page = virt_to_page(addr); + set_page_count(page, 1); + __set_freepointer((void *)addr, (void *)addr_next); + addr += object_size; + addr_next += object_size; + } + __set_freepointer((void *)(addr - object_size), NULL); + + cache->object_order = object_order; + cache->reserve_order = reserve_order; + cache->reserve_start_addr = start_addr; + cache->reserve_end_addr = end_addr; + cache->levels = levels; + cache->freelist = (void *)start_addr; + cache->tid = 0; + cache->name = bitmap_type; + cache->init = true; + +#ifdef CONFIG_ARM64 + /* IEE for ARM64 needs to access these data by IEE addresses. */ + __ptp_set_iee_pages(start_addr, end_addr, cache); +#endif +#ifdef DEBUG + atomic_set(&cache->count, reserve_pages); + atomic_set(&cache->fail_count, 0); + pr_info("IEE: HAOC_BITMAP_TYPE %u ready. object size 0x%lx, count %d.", + (unsigned int)cache->name, object_size, atomic_read(&cache->count)); +#endif +} + +#ifdef CONFIG_ARM64 +/* Expand the cache pool with PMD_SIZE for each time. */ +static int __ref iee_cache_expand(struct iee_cache *cache, gfp_t gfp) +{ + unsigned long addr; + unsigned long addr_next; + unsigned long start_addr; + unsigned long end_addr; + unsigned long object_size; + unsigned long tid; + void **freelist; + struct page *page; + + if (slab_is_available()) + start_addr = __get_free_pages(gfp, PMD_ORDER); + else + start_addr = (unsigned long)memblock_alloc(PMD_SIZE, PMD_SIZE); + + if (!start_addr) + return 0; + + addr = start_addr; + object_size = (1 << cache->object_order) * PAGE_SIZE; + addr_next = addr + object_size; + end_addr = start_addr + PMD_SIZE; + while (addr < end_addr) { + page = virt_to_page(addr); + set_page_count(page, 1); + __set_freepointer((void *)addr, (void *)addr_next); + addr += object_size; + addr_next += object_size; + } + __set_freepointer((void *)(addr - object_size), NULL); + + /* IEE for ARM64 needs to access these data by IEE addresses. */ + __ptp_set_iee_pages(start_addr, end_addr, cache); + +#ifdef DEBUG + atomic_add(1 << PMD_ORDER, &cache->count); + pr_info("IEE: HAOC_BITMAP_TYPE %u expand to count %d. Curr failed: %d", + (unsigned int)cache->name, atomic_read(&cache->count), + atomic_read(&cache->fail_count)); +#endif + +/* Fill the new allocated pages into the cache freelist. */ +redo: + tid = READ_ONCE(cache->tid); + barrier(); + freelist = READ_ONCE(cache->freelist); + __iee_set_freepointer((void *)(end_addr - object_size), freelist); + if (unlikely(!__update_freelist(cache, freelist, (void *)start_addr, tid))) + goto redo; + + return 1; +} +#endif + +void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) +{ + unsigned long tid; + size_t object_size; + void *object; + void *next_object; + + if (!cache->init) + return (void *)__get_free_pages(gfp, cache->object_order); + + object_size = (1UL << cache->object_order) * PAGE_SIZE; +redo: + tid = READ_ONCE(cache->tid); + barrier(); + object = READ_ONCE(cache->freelist); + if (unlikely(!object)) { + // slow path alloc +#ifdef CONFIG_ARM64 + if (iee_cache_expand(cache, gfp) || READ_ONCE(cache->freelist)) + goto redo; + + /* If the expandsion failed, alloc a singel object without RO protection to + * avoid block spliting. + */ + object = (void *)__get_free_pages(gfp, cache->object_order); + if (!object) + return NULL; + set_iee_address_valid((unsigned long)object, cache->object_order); + iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, cache->name); +#ifdef DEBUG + WARN_ONCE(1, "IEE: Failed on HAOC_BITMAP_TYPE %u expansion.", + (unsigned int)cache->name); + atomic_add(1 << cache->object_order, &cache->fail_count); + #endif + #else + if (!preemptible()) + return iee_cache_alloc_pending_protect(cache, gfp); + object = (void *)__get_free_pages(gfp, cache->object_order); + if (!object) + return NULL; + set_iee_page((unsigned long)object, 1 << cache->object_order, cache->name); +#endif + } else { + // fast path alloc + next_object = __get_freepointer(object); + if (unlikely(!__update_freelist(cache, object, next_object, tid))) + goto redo; + prefetchw(next_object); + if (gfp & __GFP_ZERO) + iee_memset(object, 0, object_size); + } + return object; +}; + +#ifdef CONFIG_X86_64 +static void iee_cache_deferred_work(struct work_struct *work) +{ + struct iee_cache_deferred_work *deferred = + container_of(work, struct iee_cache_deferred_work, work); + unsigned long addr = (unsigned long)deferred->object; + bool free_after_protect = false; + + if (deferred->op == IEE_CACHE_DEFER_PROTECT) { + set_iee_page(addr, deferred->num_pages, deferred->type); + + spin_lock(&iee_cache_pending_protect_lock); + list_del(&deferred->list); + free_after_protect = deferred->free_after_protect; + atomic_dec(&iee_cache_pending_protect_count); + spin_unlock(&iee_cache_pending_protect_lock); + + if (!free_after_protect) + goto out; + } + + unset_iee_page(addr, deferred->num_pages); + free_pages(addr, deferred->order); + +out: + kfree(deferred); +} + +static void *iee_cache_alloc_pending_protect(struct iee_cache *cache, gfp_t gfp) +{ + struct iee_cache_deferred_work *deferred; + gfp_t atomic_gfp = GFP_ATOMIC; + void *object; + + if (gfp & __GFP_ZERO) + atomic_gfp |= __GFP_ZERO; + if (gfp & __GFP_ACCOUNT) + atomic_gfp |= __GFP_ACCOUNT; + + deferred = kmalloc(sizeof(*deferred), GFP_ATOMIC); + if (!deferred) { + WARN_ONCE(1, + "IEE: failed to allocate pending PTP protect work\n"); + return NULL; + } + + object = (void *)__get_free_pages(atomic_gfp, cache->object_order); + if (!object) { + kfree(deferred); + return NULL; + } + + deferred->object = object; + deferred->order = cache->object_order; + deferred->num_pages = 1 << cache->object_order; + deferred->type = cache->name; + deferred->op = IEE_CACHE_DEFER_PROTECT; + deferred->free_after_protect = false; + INIT_WORK(&deferred->work, iee_cache_deferred_work); + + spin_lock(&iee_cache_pending_protect_lock); + list_add(&deferred->list, &iee_cache_pending_protect_list); + atomic_inc(&iee_cache_pending_protect_count); + spin_unlock(&iee_cache_pending_protect_lock); + + WARN_ONCE(1, + "IEE: returning pending-protect PTP page, count=%d\n", + atomic_read(&iee_cache_pending_protect_count)); + queue_work(system_unbound_wq, &deferred->work); + + return object; +} + +static bool iee_cache_mark_pending_protect_free(void *object) +{ + struct iee_cache_deferred_work *deferred; + bool found = false; + + spin_lock(&iee_cache_pending_protect_lock); + list_for_each_entry(deferred, &iee_cache_pending_protect_list, list) { + if (deferred->object == object) { + deferred->free_after_protect = true; + found = true; + break; + } + } + spin_unlock(&iee_cache_pending_protect_lock); + + return found; +} + +/* + * Pool overflow pages are freed from exit_mmap/zap_* with preemption + * disabled. unset_iee_page() calls set_memory_rw() and must not run there. + */ +static void iee_cache_slow_free_pages(struct iee_cache *cache, void *object) +{ + unsigned long addr = (unsigned long)object; + int num_pages = 1 << cache->object_order; + struct iee_cache_deferred_work *deferred; + + if (preemptible()) { + unset_iee_page(addr, num_pages); + free_pages(addr, cache->object_order); + return; + } + + deferred = kmalloc(sizeof(*deferred), GFP_ATOMIC); + if (!deferred) { + WARN_ONCE(1, "IEE: failed to defer PTP page free, leaking page %px\n", + object); + return; + } + + deferred->object = object; + deferred->order = cache->object_order; + deferred->num_pages = num_pages; + deferred->op = IEE_CACHE_DEFER_FREE; + INIT_WORK(&deferred->work, iee_cache_deferred_work); + queue_work(system_unbound_wq, &deferred->work); +} +#endif + +void iee_cache_free(struct iee_cache *cache, void *object) +{ + unsigned long tid; + void **freelist; + + if (!cache->init) { + free_pages((unsigned long)object, cache->object_order); + return; + } + +#ifdef CONFIG_X86_64 + if (unlikely((unsigned long)object < cache->reserve_start_addr + || (unsigned long)object >= cache->reserve_end_addr)) { + if (iee_cache_mark_pending_protect_free(object)) + return; + iee_cache_slow_free_pages(cache, object); + return; + } +#endif + + // fast path free +redo: + tid = READ_ONCE(cache->tid); + barrier(); + freelist = READ_ONCE(cache->freelist); + __iee_set_freepointer(object, freelist); + if (unlikely(!__update_freelist(cache, freelist, object, tid))) + goto redo; +} diff --git a/mm/memory.c b/mm/memory.c index eaf07a4d1e0d5..d148ec2ee2647 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -90,6 +90,10 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + #include "pgalloc-track.h" #include "internal.h" #include "swap.h" @@ -5695,6 +5699,9 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, /* If the fault handler drops the mmap_lock, vma may be freed */ struct mm_struct *mm = vma->vm_mm; vm_fault_t ret; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif __set_current_state(TASK_RUNNING); @@ -5721,7 +5728,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, if (unlikely(is_vm_hugetlb_page(vma))) ret = hugetlb_fault(vma->vm_mm, vma, address, flags); else +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + { + ptp_disable_wp(&cr0); ret = __handle_mm_fault(vma, address, flags); + ptp_restore_wp(cr0); + } +#else + ret = __handle_mm_fault(vma, address, flags); +#endif lru_gen_exit_fault(); diff --git a/mm/mmap.c b/mm/mmap.c index bac026d841062..245a123c74238 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -54,6 +54,10 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + #define CREATE_TRACE_POINTS #include @@ -2343,10 +2347,17 @@ static void unmap_region(struct mm_struct *mm, struct ma_state *mas, update_hiwater_rss(mm); unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); mas_set(mas, mt_start); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + tlb_flush_mmu(&tlb); +#endif free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, next ? next->vm_start : USER_PGTABLES_CEILING, mm_wr_locked); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_finish_mmu(&tlb); +#else tlb_finish_mmu(&tlb); +#endif } /* @@ -2557,6 +2568,9 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, MA_STATE(mas_detach, &mt_detach, 0, 0); mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); mt_on_stack(mt_detach); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif /* * If we need to split any vma, do it now to save pain later. @@ -2668,8 +2682,15 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, * were isolated before we downgraded mmap_lock. */ mas_set(&mas_detach, 1); + +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif unmap_region(mm, &mas_detach, vma, prev, next, start, end, count, !unlock); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif /* Statistics and freeing VMAs */ mas_set(&mas_detach, 0); remove_mt(mm, &mas_detach); @@ -3358,6 +3379,9 @@ void exit_mmap(struct mm_struct *mm) unsigned long nr_accounted = 0; MA_STATE(mas, &mm->mm_mt, 0, 0); int count = 0; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif /* mm's last user has gone, and its about to be pulled down */ mmu_notifier_release(mm); @@ -3378,7 +3402,13 @@ void exit_mmap(struct mm_struct *mm) tlb_gather_mmu_fullmm(&tlb, mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif unmap_vmas(&tlb, &mas, vma, 0, ULONG_MAX, ULONG_MAX, false); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif mmap_read_unlock(mm); /* @@ -3389,9 +3419,16 @@ void exit_mmap(struct mm_struct *mm) mmap_write_lock(mm); mt_clear_in_rcu(&mm->mm_mt); mas_set(&mas, vma->vm_end); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + tlb_flush_mmu(&tlb); +#endif free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING, true); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_finish_mmu(&tlb); +#else tlb_finish_mmu(&tlb); +#endif /* * Walk the list again, actually closing and freeing it, with preemption diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 04469b481b93f..7d4ee1dbf829b 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -142,6 +142,51 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void __ptp_tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) +{ + struct encoded_page **pages = batch->encoded_pages; + unsigned int i, j, nr, nr_pages; + + while (batch->nr) { + if (!page_poisoning_enabled_static() && !want_init_on_free()) { + nr = min(MAX_NR_FOLIOS_PER_FREE, batch->nr); + + if (unlikely(encoded_page_flags(pages[nr - 1]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr++; + } else { + for (nr = 0, nr_pages = 0; + nr < batch->nr && nr_pages < MAX_NR_FOLIOS_PER_FREE; + nr++) { + if (unlikely(encoded_page_flags(pages[nr]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr_pages += encoded_nr_pages(pages[++nr]); + else + nr_pages++; + } + } + + for (i = 0; i < nr; i++) { + struct encoded_page *encoded = pages[i]; + struct page *page = encoded_page_ptr(encoded); + unsigned int page_count = 1; + + if (unlikely(encoded_page_flags(encoded) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + page_count = encoded_nr_pages(pages[++i]); + + for (j = 0; j < page_count; j++) + __ptp_tlb_remove_table(page + j); + } + pages += nr; + batch->nr -= nr; + + cond_resched(); + } +} +#endif + static void tlb_batch_pages_flush(struct mmu_gather *tlb) { struct mmu_gather_batch *batch; @@ -151,6 +196,17 @@ static void tlb_batch_pages_flush(struct mmu_gather *tlb) tlb->active = &tlb->local; } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void ptp_tlb_batch_pages_flush(struct mmu_gather *tlb) +{ + struct mmu_gather_batch *batch; + + for (batch = &tlb->local; batch && batch->nr; batch = batch->next) + __ptp_tlb_batch_free_encoded_pages(batch); + tlb->active = &tlb->local; +} +#endif + static void tlb_batch_list_free(struct mmu_gather *tlb) { struct mmu_gather_batch *batch, *next; @@ -348,6 +404,90 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) tlb_table_flush(tlb); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void __ptp_tlb_remove_table_free(struct mmu_table_batch *batch) +{ + int i; + + for (i = 0; i < batch->nr; i++) + __ptp_tlb_remove_table(batch->tables[i]); + + free_page((unsigned long)batch); +} + +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE +struct ptp_remove_table_work { + struct work_struct work; + struct mmu_table_batch *batch; +}; + +static void ptp_remove_table(struct work_struct *work) +{ + struct ptp_remove_table_work *ptp_remove_table_work = + container_of(work, struct ptp_remove_table_work, work); + + __ptp_tlb_remove_table_free(ptp_remove_table_work->batch); + kfree(ptp_remove_table_work); +} + +static void ptp_tlb_remove_table_rcu(struct rcu_head *head) +{ + struct ptp_remove_table_work *ptp_remove_table_work = + kmalloc(sizeof(struct ptp_remove_table_work), GFP_ATOMIC); + + ptp_remove_table_work->batch = container_of(head, struct mmu_table_batch, rcu); + INIT_WORK(&ptp_remove_table_work->work, ptp_remove_table); + schedule_work(&ptp_remove_table_work->work); +} + +static void ptp_tlb_remove_table_free(struct mmu_table_batch *batch) +{ + call_rcu(&batch->rcu, ptp_tlb_remove_table_rcu); +} +#else +static void ptp_tlb_remove_table_free(struct mmu_table_batch *batch) +{ + __ptp_tlb_remove_table_free(batch); +} +#endif + +static void ptp_tlb_remove_table_one(void *table) +{ + tlb_remove_table_sync_one(); + __ptp_tlb_remove_table(table); +} + +static void ptp_tlb_table_flush(struct mmu_gather *tlb) +{ + struct mmu_table_batch **batch = &tlb->batch; + + if (*batch) { + tlb_table_invalidate(tlb); + ptp_tlb_remove_table_free(*batch); + *batch = NULL; + } +} + +void ptp_tlb_remove_table(struct mmu_gather *tlb, void *table) +{ + struct mmu_table_batch **batch = &tlb->batch; + + if (*batch == NULL) { + *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); + if (*batch == NULL) { + tlb_table_invalidate(tlb); + ptp_tlb_remove_table_one(table); + return; + } + (*batch)->nr = 0; + } + + (*batch)->tables[(*batch)->nr++] = table; + if ((*batch)->nr == MAX_TABLE_BATCH) + ptp_tlb_table_flush(tlb); +} +#endif + static inline void tlb_table_init(struct mmu_gather *tlb) { tlb->batch = NULL; @@ -502,3 +642,57 @@ void tlb_finish_mmu(struct mmu_gather *tlb) #endif dec_tlb_flush_pending(tlb->mm); } + +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void ptp_tlb_flush_mmu_free(struct mmu_gather *tlb) +{ +#ifdef CONFIG_MMU_GATHER_TABLE_FREE + ptp_tlb_table_flush(tlb); +#endif +#ifndef CONFIG_MMU_GATHER_NO_GATHER + ptp_tlb_batch_pages_flush(tlb); +#endif +} + +void ptp_tlb_flush_mmu(struct mmu_gather *tlb) +{ + tlb_flush_mmu_tlbonly(tlb); + ptp_tlb_flush_mmu_free(tlb); +} + +void ptp_tlb_finish_mmu(struct mmu_gather *tlb) +{ + /* + * If there are parallel threads are doing PTE changes on same range + * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB + * flush by batching, one thread may end up seeing inconsistent PTEs + * and result in having stale TLB entries. So flush TLB forcefully + * if we detect parallel PTE batching threads. + * + * However, some syscalls, e.g. munmap(), may free page tables, this + * needs force flush everything in the given range. Otherwise this + * may result in having stale TLB entries for some architectures, + * e.g. aarch64, that could specify flush what level TLB. + */ + if (mm_tlb_flush_nested(tlb->mm)) { + /* + * The aarch64 yields better performance with fullmm by + * avoiding multiple CPUs spamming TLBI messages at the + * same time. + * + * On x86 non-fullmm doesn't yield significant difference + * against fullmm. + */ + tlb->fullmm = 1; + __tlb_reset_range(tlb); + tlb->freed_tables = 1; + } + + ptp_tlb_flush_mmu(tlb); + +#ifndef CONFIG_MMU_GATHER_NO_GATHER + tlb_batch_list_free(tlb); +#endif + dec_tlb_flush_pending(tlb->mm); +} +#endif diff --git a/mm/mprotect.c b/mm/mprotect.c index 76381f6f91521..59eca516a1ff5 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -37,6 +37,10 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + #include "internal.h" bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, @@ -584,6 +588,9 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif if (newflags == oldflags) { *pprev = vma; @@ -644,7 +651,13 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; vma_set_page_prot(vma); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif change_protection(tlb, vma, start, end, mm_cp_flags); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index a78a4adf711ac..aa393f6c5780e 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -234,6 +234,22 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, } #endif +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +struct ptp_pte_free_now_work { + struct work_struct work; + struct page *page; +}; + +static void ptp_pte_free_now(struct work_struct *work) +{ + struct ptp_pte_free_now_work *ptp_work = + container_of(work, struct ptp_pte_free_now_work, work); + + pte_free(NULL, ptp_work->page); + kfree(ptp_work); +} +#endif + /* arch define pte_free_defer in asm/pgalloc.h for its own implementation */ #ifndef pte_free_defer static void pte_free_now(struct rcu_head *head) @@ -241,7 +257,16 @@ static void pte_free_now(struct rcu_head *head) struct page *page; page = container_of(head, struct page, rcu_head); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + struct ptp_pte_free_now_work *ptp_work = + kmalloc(sizeof(struct ptp_pte_free_now_work), GFP_ATOMIC); + + ptp_work->page = page; + INIT_WORK(&ptp_work->work, ptp_pte_free_now); + schedule_work(&ptp_work->work); +#else pte_free(NULL /* mm not passed and not used */, (pgtable_t)page); +#endif } void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) From 7445c123e6c28ab9a97ca59366aa1031c79ceaa4 Mon Sep 17 00:00:00 2001 From: zhangshiyangB7 Date: Fri, 3 Apr 2026 16:59:39 +0800 Subject: [PATCH 06/26] HAOC3.0/arm64/PTP: Add support for arm64 Page Table Isolation Support page table protection and isolation for arm64. Signed-off-by: zhangshiyangB7 Signed-off-by: Liu Zhehui --- .../configs/deepin_arm64_desktop_defconfig | 2 + arch/arm64/include/asm/fixmap.h | 4 + arch/arm64/include/asm/haoc/haoc-def.h | 15 + arch/arm64/include/asm/haoc/haoc.h | 24 + arch/arm64/include/asm/haoc/iee-fixmap.h | 23 + arch/arm64/include/asm/haoc/iee-func.h | 4 +- arch/arm64/include/asm/haoc/iee-init.h | 8 + arch/arm64/include/asm/haoc/iee-mmu.h | 7 +- arch/arm64/include/asm/haoc/iee-pgtable.h | 2059 +++++++++++++++++ arch/arm64/include/asm/haoc/iee-ptp-init.h | 56 + arch/arm64/include/asm/haoc/iee.h | 16 +- arch/arm64/include/asm/pgalloc.h | 15 + arch/arm64/include/asm/pgtable.h | 6 + arch/arm64/include/asm/tlb.h | 8 + arch/arm64/kernel/cpufeature.c | 8 + arch/arm64/kernel/haoc/Kconfig | 14 + arch/arm64/kernel/haoc/Makefile | 1 + arch/arm64/kernel/haoc/haoc-bitmap.c | 3 +- arch/arm64/kernel/haoc/haoc.c | 15 + arch/arm64/kernel/haoc/iee/iee-func.c | 21 +- arch/arm64/kernel/haoc/iee/iee-init.c | 1 - arch/arm64/kernel/haoc/iee/iee-mmu.c | 226 +- arch/arm64/kernel/haoc/iee/iee-token.c | 5 +- arch/arm64/kernel/haoc/ptp/Makefile | 1 + arch/arm64/kernel/haoc/ptp/iee-ptp-init.c | 408 ++++ arch/arm64/kernel/haoc/ptp/ptp.c | 301 +++ arch/arm64/kernel/setup.c | 7 + arch/arm64/kernel/vmlinux.lds.S | 12 + arch/arm64/mm/fault.c | 4 + arch/arm64/mm/fixmap.c | 55 +- arch/arm64/mm/init.c | 6 + arch/arm64/mm/mmu.c | 83 +- arch/arm64/mm/pgd.c | 40 +- arch/arm64/mm/trans_pgd.c | 4 + drivers/firmware/efi/arm-runtime.c | 4 + drivers/tty/serial/earlycon.c | 4 + drivers/usb/early/ehci-dbgp.c | 4 + include/linux/efi.h | 4 + mm/early_ioremap.c | 22 + mm/haoc/ptp-pg_cache.c | 30 +- mm/huge_memory.c | 10 + mm/sparse-vmemmap.c | 12 + 42 files changed, 3459 insertions(+), 93 deletions(-) create mode 100644 arch/arm64/include/asm/haoc/iee-fixmap.h create mode 100644 arch/arm64/include/asm/haoc/iee-pgtable.h create mode 100644 arch/arm64/include/asm/haoc/iee-ptp-init.h create mode 100644 arch/arm64/kernel/haoc/ptp/Makefile create mode 100644 arch/arm64/kernel/haoc/ptp/iee-ptp-init.c create mode 100644 arch/arm64/kernel/haoc/ptp/ptp.c diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index c2d0cd3be5e38..5c9dc3868a0f4 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -165,6 +165,8 @@ CONFIG_IEE=y CONFIG_IEE_PTRP=y CONFIG_IEE_SIP=y CONFIG_CREDP=y +CONFIG_PTP=y +CONFIG_PTP_RESERVE_ORDER=12 CONFIG_KPROBES=y CONFIG_SHADOW_CALL_STACK=y CONFIG_MODULES=y diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 58c294a966768..85d92fb420249 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -109,6 +109,10 @@ void __init fixmap_copy(pgd_t *pgdir); extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); +#ifdef CONFIG_PTP +#include +#endif + #include #endif /* !__ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index afd8529eb9620..15ccc77282dd6 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -59,6 +59,21 @@ enum { IEE_OP_SET_CRED_SECURITY, IEE_OP_SET_CRED_RCU, IEE_OP_SET_CRED_UCOUNTS, +#endif +#ifdef CONFIG_PTP + IEE_OP_SET_TRAMP_PGD, + IEE_OP_SET_BM_PTE, + IEE_OP_SET_PTE, + IEE_OP_SET_PMD, + IEE_OP_SET_PUD, + IEE_OP_SET_P4D, + IEE_OP_SET_SWAPPER_PGD, + IEE_OP_SET_XCHG, + IEE_OP_SET_PMD_XCHG, + IEE_OP_SET_CMPXCHG, + IEE_OP_SET_PMD_CMPXCHG, + IEE_OP_SET_SENSITIVE_PTE, + IEE_OP_UNSET_SENSITIVE_PTE, #endif IEE_FLAG_END }; diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index 8bb0dcfd11539..70d173918638e 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -91,4 +91,28 @@ void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_hea void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); #endif + +#ifdef CONFIG_PTP +#include + +void __iee_code _iee_set_static_pgd(int flag, pgd_t *pgdp, pgd_t pgd); +void __iee_code _iee_set_bm_pte(int flag, pte_t *ptep, pte_t pte); +void __iee_code _iee_set_pte(int flag, pte_t *ptep, pte_t pte); +void __iee_code _iee_set_pmd(int flag, pmd_t *pmdp, pmd_t pmd); +void __iee_code _iee_set_pud(int flag, pud_t *pudp, pud_t pud); +void __iee_code _iee_set_p4d(int flag, p4d_t *p4dp, p4d_t p4d); +void __iee_code _iee_set_swapper_pgd(int flag, pgd_t *pgdp, pgd_t pgd); +pteval_t __iee_code _iee_set_xchg_relaxed(int flag, pte_t *ptep, + pteval_t pteval); +pmdval_t __iee_code _iee_set_pmd_xchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t pmdval); +pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, + pteval_t old_pteval, pteval_t new_pteval); +pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t old_pmdval, pmdval_t new_pmdval); +void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, + pte_t *iee_ptep, int order, int use_block_pmd); +void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, + pte_t *iee_ptep, int order, int use_block_pmd); +#endif #endif diff --git a/arch/arm64/include/asm/haoc/iee-fixmap.h b/arch/arm64/include/asm/haoc/iee-fixmap.h new file mode 100644 index 0000000000000..628ff4a5c6875 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-fixmap.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_FIXMAP_H +#define _LINUX_IEE_FIXMAP_H + +#include + +#ifndef clear_fixmap_pre_init +#define clear_fixmap_pre_init(idx) \ + __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR) +#endif + +#define __iee_set_fixmap_offset_pre_init(idx, phys, flags) \ +({ \ + unsigned long ________addr; \ + __iee_set_fixmap_pre_init(idx, phys, flags); \ + ________addr = fix_to_virt(idx) + ((phys) & (PAGE_SIZE - 1)); \ + ________addr; \ +}) + +#define iee_set_fixmap_offset_pre_init(idx, phys) \ + __iee_set_fixmap_offset_pre_init(idx, phys, FIXMAP_PAGE_NORMAL) + +#endif diff --git a/arch/arm64/include/asm/haoc/iee-func.h b/arch/arm64/include/asm/haoc/iee-func.h index 11bb9750afa8f..16cde421e66be 100644 --- a/arch/arm64/include/asm/haoc/iee-func.h +++ b/arch/arm64/include/asm/haoc/iee-func.h @@ -13,11 +13,13 @@ #include extern void set_iee_address(unsigned long addr, unsigned int order, bool valid); +extern void set_iee_address_valid(unsigned long lm_addr, unsigned int order); +extern void set_iee_address_invalid(unsigned long lm_addr, unsigned int order); extern void iee_set_logical_mem(unsigned long addr, unsigned int order, bool prot); extern void put_pages_into_iee(unsigned long addr, int order); extern void set_iee_page_type(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type); -extern void set_iee_page(unsigned long addr, int order); +extern void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type); extern void unset_iee_page(unsigned long addr, int order); #endif diff --git a/arch/arm64/include/asm/haoc/iee-init.h b/arch/arm64/include/asm/haoc/iee-init.h index 6de6fea0ac425..2a724f05b763b 100644 --- a/arch/arm64/include/asm/haoc/iee-init.h +++ b/arch/arm64/include/asm/haoc/iee-init.h @@ -13,12 +13,20 @@ #define NO_BLOCK_MAPPINGS BIT(0) #define NO_CONT_MAPPINGS BIT(1) #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ +#define IS_IEE_MAPPINGS BIT(3) + +#define TOP_PAGE_TABLE(level) ((level) == CONFIG_PGTABLE_LEVELS) extern char iee_init_data_begin[]; extern char iee_init_data_end[]; +#ifdef CONFIG_PTP +extern char __iee_ptp_data_start[]; +extern char __iee_ptp_data_end[]; +#endif #ifdef CONFIG_CREDP extern char __iee_cred_data_start[]; extern char __iee_cred_data_end[]; #endif +extern struct mutex fixmap_lock; #endif diff --git a/arch/arm64/include/asm/haoc/iee-mmu.h b/arch/arm64/include/asm/haoc/iee-mmu.h index bc43314bd3446..33f291ee72f29 100644 --- a/arch/arm64/include/asm/haoc/iee-mmu.h +++ b/arch/arm64/include/asm/haoc/iee-mmu.h @@ -10,8 +10,13 @@ #ifndef _LINUX_IEE_MMU_H #define _LINUX_IEE_MMU_H +#ifdef CONFIG_PTP +#include +#endif + extern phys_addr_t __init early_iee_stack_alloc(int order); -extern phys_addr_t __init early_iee_data_alloc(int shift); +extern phys_addr_t __init early_iee_data_alloc(int order); +extern phys_addr_t __init early_iee_pgtable_alloc(int shift); extern void __iee_create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, diff --git a/arch/arm64/include/asm/haoc/iee-pgtable.h b/arch/arm64/include/asm/haoc/iee-pgtable.h new file mode 100644 index 0000000000000..8788ebd23388a --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-pgtable.h @@ -0,0 +1,2059 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_IEE_PGTABLE_H +#define __ASM_IEE_PGTABLE_H + +#include +#include + +#include +#include +#include +#include +#include + +/* + * VMALLOC range. + * + * VMALLOC_START: beginning of the kernel vmalloc space + * VMALLOC_END: extends to the available space below vmemmap + */ +#define VMALLOC_START (MODULES_END) +#if VA_BITS == VA_BITS_MIN +#define VMALLOC_END (VMEMMAP_START - SZ_8M) +#else +#define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT) +#define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * \ + sizeof(struct page) - SZ_8M) +#endif + +#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef CYDP_CLEAR_YOUNG +typedef unsigned int cydp_t; +#define CYDP_CLEAR_YOUNG (1U << 0) +#define CYDP_CLEAR_DIRTY (1U << 1) +#endif + +extern pgd_t tramp_pg_dir[]; +extern pgd_t idmap_pg_dir[]; +extern pgd_t init_pg_dir[]; +extern pgd_t init_pg_end[]; + +extern int __pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty); +extern bool haoc_enabled; +extern bool iee_init_done; + +static inline bool iee_pgtable_ready(void) +{ + return haoc_enabled && iee_init_done; +} + +static inline bool in_swapper_pgdir(void *addr); +static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd); + +static inline bool in_tramp_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)tramp_pg_dir & PAGE_MASK); +} + +static inline bool in_idmap_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)idmap_pg_dir & PAGE_MASK); +} + +static inline void iee_set_pte_pre_init(pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*ptep, pte); + + /* + * Only if the new pte is valid and kernel, otherwise TLB maintenance + * or update_mmu_cache() have the necessary barriers. + */ + dsb(ishst); + isb(); +} + +static inline void iee_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) +{ +#ifdef __PAGETABLE_PMD_FOLDED + if (in_swapper_pgdir(pmdp)) { + iee_set_swapper_pgd_pre_init((pgd_t *)pmdp, __pgd(pmd_val(pmd))); + return; + } +#endif /* __PAGETABLE_PMD_FOLDED */ + + WRITE_ONCE(*pmdp, pmd); + + dsb(ishst); + isb(); +} + +static inline void iee_set_pud_pre_init(pud_t *pudp, pud_t pud) +{ + #ifdef __PAGETABLE_PUD_FOLDED + if (in_swapper_pgdir(pudp)) { + iee_set_swapper_pgd_pre_init((pgd_t *)pudp, __pgd(pud_val(pud))); + return; + } + #endif + + WRITE_ONCE(*pudp, pud); + + dsb(ishst); + isb(); +} + +static inline void __maybe_unused iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) +{ + if (in_swapper_pgdir(p4dp)) { + iee_set_swapper_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + return; + } + + WRITE_ONCE(*p4dp, p4d); + dsb(ishst); + isb(); +} + +static inline void iee_set_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ + if (in_swapper_pgdir(pgdp)) { + iee_set_swapper_pgd_pre_init(pgdp, __pgd(pgd_val(pgd))); + return; + } + + WRITE_ONCE(*pgdp, pgd); + dsb(ishst); + isb(); +} + +#define pte_set_fixmap_pre_init(addr) \ + ((pte_t *)iee_set_fixmap_offset_pre_init(FIX_PTE, addr)) +#define pte_set_fixmap_offset_pre_init(pmd, addr) \ + pte_set_fixmap_pre_init(pte_offset_phys(pmd, addr)) +#define pte_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PTE) + +#define pmd_set_fixmap_pre_init(addr) \ + ((pmd_t *)iee_set_fixmap_offset_pre_init(FIX_PMD, addr)) +#define pmd_set_fixmap_offset_pre_init(pud, addr) \ + pmd_set_fixmap_pre_init(pmd_offset_phys(pud, addr)) +#define pmd_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PMD) + +#define pud_set_fixmap_pre_init(addr) \ + ((pud_t *)iee_set_fixmap_offset_pre_init(FIX_PUD, addr)) +#define pud_set_fixmap_offset_pre_init(p4d, addr)\ + pud_set_fixmap_pre_init(pud_offset_phys(p4d, addr)) +#define pud_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PUD) + +#define pgd_set_fixmap_pre_init(addr) \ + ((pgd_t *)iee_set_fixmap_offset_pre_init(FIX_PGD, addr)) +#define pgd_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PGD) + +static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ + WRITE_ONCE(*pgdp, pgd); + dsb(ishst); + isb(); +} + +static inline pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval) +{ + pteval_t ret; + + if (iee_pgtable_ready()) + ret = iee_rw_gate(IEE_OP_SET_XCHG, ptep, pteval); + else + ret = xchg_relaxed((pteval_t *)ptep, pteval); + return (pteval_t)ret; +} + +static inline pmdval_t iee_set_pmd_xchg_relaxed(pmd_t *pmdp, pmdval_t pmdval) +{ + pmdval_t ret; + + if (iee_pgtable_ready()) + ret = iee_rw_gate(IEE_OP_SET_PMD_XCHG, pmdp, pmdval); + else + ret = xchg_relaxed((pmdval_t *)pmdp, pmdval); + return (pmdval_t)ret; +} + +static inline pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, + pteval_t new_pteval) +{ + pteval_t ret; + + if (iee_pgtable_ready()) + ret = iee_rw_gate(IEE_OP_SET_CMPXCHG, ptep, old_pteval, new_pteval); + else + ret = cmpxchg_relaxed((pteval_t *)ptep, old_pteval, new_pteval); + return ret; +} + +static inline pmdval_t iee_set_pmd_cmpxchg_relaxed(pmd_t *pmdp, pmdval_t old_pmdval, + pmdval_t new_pmdval) +{ + pmdval_t ret; + + if (iee_pgtable_ready()) + ret = iee_rw_gate(IEE_OP_SET_PMD_CMPXCHG, pmdp, old_pmdval, new_pmdval); + else + ret = cmpxchg_relaxed((pmdval_t *)pmdp, old_pmdval, new_pmdval); + return ret; +} + +static inline void iee_set_static_pgd(pgd_t *pgdp, pgd_t pgd) +{ + iee_rw_gate(IEE_OP_SET_TRAMP_PGD, pgdp, pgd); +} + +static inline void iee_set_bm_pte(pte_t *ptep, pte_t pte) +{ + iee_rw_gate(IEE_OP_SET_BM_PTE, ptep, pte); + + /* + * Only if the new pte is valid and kernel, otherwise TLB maintenance + * or update_mmu_cache() have the necessary barriers. + */ + dsb(ishst); + isb(); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE + +/* Set stride and tlb_level in flush_*_tlb_range */ +#define flush_pmd_tlb_range(vma, addr, end) \ + __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2) +#define flush_pud_tlb_range(vma, addr, end) \ + __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +/* + * Outside of a few very special situations (e.g. hibernation), we always + * use broadcast TLB invalidation instructions, therefore a spurious page + * fault on one CPU which has been handled concurrently by another CPU + * does not need to perform additional invalidation. + */ +#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) + +/* + * ZERO_PAGE is a global shared page that is always zero: used + * for zero-mapped memory areas etc.. + */ +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page)) + +#define pte_ERROR(e) \ + pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) + +/* + * Macros to convert between a physical address and its placement in a + * page table entry, taking care of 52-bit addresses. + */ +#ifdef CONFIG_ARM64_PA_BITS_52 +static inline phys_addr_t __pte_to_phys(pte_t pte) +{ + pte_val(pte) &= ~PTE_MAYBE_SHARED; + return (pte_val(pte) & PTE_ADDR_LOW) | + ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT); +} +static inline pteval_t __phys_to_pte_val(phys_addr_t phys) +{ + return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK; +} +#else +#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK) +#define __phys_to_pte_val(phys) (phys) +#endif + +#define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT) +#define pfn_pte(pfn, prot) \ + __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) + +#define pte_none(pte) (!pte_val(pte)) +#define __pte_clear(mm, addr, ptep) \ + __set_pte(ptep, __pte(0)) +#define pte_page(pte) (pfn_to_page(pte_pfn(pte))) + +/* + * The following only work if pte_present(). Undefined behaviour otherwise. + */ +#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE))) +#define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) +#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) +#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) +#define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY)) +#define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) +#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) +#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) +#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP)) +#define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \ + PTE_ATTRINDX(MT_NORMAL_TAGGED)) + +#define pte_cont_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +#define pmd_cont_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +#define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte)) +#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) +#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) + +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) +/* + * Execute-only user mappings do not have the PTE_USER bit set. All valid + * kernel mappings have the PTE_UXN bit set. + */ +#define pte_valid_not_user(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN)) +/* + * Returns true if the pte is valid and has the contiguous bit set. + */ +#define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) +/* + * Could the pte be present in the TLB? We must check mm_tlb_flush_pending + * so that we don't erroneously return false for pages that have been + * remapped as PROT_NONE but are yet to be flushed from the TLB. + * Note that we can't make any assumptions based on the state of the access + * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the + * TLB. + */ +#define pte_accessible(mm, pte) \ + (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) + +/* + * p??_access_permitted() is true for valid user mappings (PTE_USER + * bit set, subject to the write permission check). For execute-only + * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits + * not set) must return false. PROT_NONE mappings do not have the + * PTE_VALID bit set. + */ +#define pte_access_permitted(pte, write) \ + (((pte_val(pte) & (PTE_VALID | PTE_USER)) ==\ + (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte))) +#define pmd_access_permitted(pmd, write) \ + (pte_access_permitted(pmd_pte(pmd), (write))) +#define pud_access_permitted(pud, write) \ + (pte_access_permitted(pud_pte(pud), (write))) + +static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) +{ + pte_val(pte) &= ~pgprot_val(prot); + return pte; +} + +static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) +{ + pte_val(pte) |= pgprot_val(prot); + return pte; +} + +static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot) +{ + pmd_val(pmd) &= ~pgprot_val(prot); + return pmd; +} + +static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) +{ + pmd_val(pmd) |= pgprot_val(prot); + return pmd; +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); + if (pte_sw_dirty(pte)) + pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); + return pte; +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY)); + pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); + + return pte; +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + + if (pte_write(pte)) + pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); + + return pte; +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + /* + * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY + * clear), set the PTE_DIRTY bit. + */ + if (pte_hw_dirty(pte)) + pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + + pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); + pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); + return pte; +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_AF)); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_AF)); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); +} + +static inline pte_t pte_mkcont(pte_t pte) +{ + pte = set_pte_bit(pte, __pgprot(PTE_CONT)); + return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE)); +} + +static inline pte_t pte_mknoncont(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_CONT)); +} + +static inline pte_t pte_mkpresent(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_VALID)); +} + +static inline pmd_t pmd_mkcont(pmd_t pmd) +{ + return __pmd(pmd_val(pmd) | PMD_SECT_CONT); +} + +static inline pte_t pte_mkdevmap(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL)); +} + +static inline void set_pte_nosync(pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*ptep, pte); +} + +static inline void __set_pte(pte_t *ptep, pte_t pte) +{ + if (iee_pgtable_ready()) + iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); + else + WRITE_ONCE(*ptep, pte); + + dsb(ishst); + isb(); +} + +static inline pte_t __ptep_get(pte_t *ptep) +{ + return READ_ONCE(*ptep); +} + +extern void __sync_icache_dcache(pte_t pteval); +bool pgattr_change_is_safe(u64 old, u64 new); + +/* + * PTE bits configuration in the presence of hardware Dirty Bit Management + * (PTE_WRITE == PTE_DBM): + * + * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) + * 0 0 | 1 0 0 + * 0 1 | 1 1 0 + * 1 0 | 1 0 1 + * 1 1 | 0 1 x + * + * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via + * the page fault mechanism. Checking the dirty status of a pte becomes: + * + * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY) + */ + +static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, + pte_t pte) +{ + pte_t old_pte; + + if (!IS_ENABLED(CONFIG_DEBUG_VM)) + return; + + old_pte = __ptep_get(ptep); + + if (!pte_valid(old_pte) || !pte_valid(pte)) + return; + if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1) + return; + + /* + * Check for potential race with hardware updates of the pte + * (__ptep_set_access_flags safely changes valid ptes without going + * through an invalid entry). + */ + VM_WARN_ONCE(!pte_young(pte), + "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", + __func__, pte_val(old_pte), pte_val(pte)); + VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), + "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", + __func__, pte_val(old_pte), pte_val(pte)); + VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)), + "%s: unsafe attribute change: 0x%016llx -> 0x%016llx", + __func__, pte_val(old_pte), pte_val(pte)); +} + +static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages) +{ + if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) + __sync_icache_dcache(pte); + + /* + * If the PTE would provide user space access to the tags associated + * with it then ensure that the MTE tags are synchronised. Although + * pte_access_permitted() returns false for exec only mappings, they + * don't expose tags (instruction fetches don't check tags). + */ + if (system_supports_mte() && pte_access_permitted(pte, false) && + !pte_special(pte) && pte_tagged(pte)) + mte_sync_tags(pte, nr_pages); +} + +/* + * Select all bits except the pfn + */ +#define pte_pgprot pte_pgprot +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + +#define pte_advance_pfn pte_advance_pfn +static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) +{ + return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); +} + +static inline void __set_ptes(struct mm_struct *mm, + unsigned long __always_unused addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + page_table_check_ptes_set(mm, ptep, pte, nr); + __sync_cache_and_tags(pte, nr); + + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn(pte, 1); + } +} + +/* + * Huge pte definitions. + */ +#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT)) + +/* + * Hugetlb definitions. + */ +#define HUGE_MAX_HSTATE 4 +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +static inline pte_t pgd_pte(pgd_t pgd) +{ + return __pte(pgd_val(pgd)); +} + +static inline pte_t p4d_pte(p4d_t p4d) +{ + return __pte(p4d_val(p4d)); +} + +static inline pte_t pud_pte(pud_t pud) +{ + return __pte(pud_val(pud)); +} + +static inline pud_t pte_pud(pte_t pte) +{ + return __pud(pte_val(pte)); +} + +static inline pmd_t pud_pmd(pud_t pud) +{ + return __pmd(pud_val(pud)); +} + +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + +static inline pgprot_t mk_pud_sect_prot(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT); +} + +static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT); +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); +} + +static inline int pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & PTE_SWP_EXCLUSIVE; +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); +} + +#ifdef CONFIG_NUMA_BALANCING +/* + * See the comment in include/linux/pgtable.h + */ +static inline int pte_protnone(pte_t pte) +{ + return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE; +} + +static inline int pmd_protnone(pmd_t pmd) +{ + return pte_protnone(pmd_pte(pmd)); +} +#endif + +#define pmd_present_invalid(pmd) (!!(pmd_val(pmd) & PMD_PRESENT_INVALID)) + +static inline int pmd_present(pmd_t pmd) +{ + return pte_present(pmd_pte(pmd)) || pmd_present_invalid(pmd); +} + +/* + * THP definitions. + */ + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pmd_trans_huge(pmd_t pmd) +{ + return pmd_val(pmd) && pmd_present(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) +#define pmd_young(pmd) pte_young(pmd_pte(pmd)) +#define pmd_valid(pmd) pte_valid(pmd_pte(pmd)) +#define pmd_user(pmd) pte_user(pmd_pte(pmd)) +#define pmd_user_exec(pmd) pte_user_exec(pmd_pte(pmd)) +#define pmd_cont(pmd) pte_cont(pmd_pte(pmd)) +#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) +#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) +#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))) +#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) +#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) +#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) + +static inline pmd_t pmd_mkinvalid(pmd_t pmd) +{ + pmd = set_pmd_bit(pmd, __pgprot(PMD_PRESENT_INVALID)); + pmd = clear_pmd_bit(pmd, __pgprot(PMD_SECT_VALID)); + + return pmd; +} + +#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) + +#define pmd_write(pmd) pte_write(pmd_pte(pmd)) + +#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd)) +#endif +static inline pmd_t pmd_mkdevmap(pmd_t pmd) +{ + return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP))); +} + +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP +#define pmd_special(pte) (!!((pmd_val(pte) & PTE_SPECIAL))) +static inline pmd_t pmd_mkspecial(pmd_t pmd) +{ + return set_pmd_bit(pmd, __pgprot(PTE_SPECIAL)); +} +#endif + +#define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd)) +#define __phys_to_pmd_val(phys) __phys_to_pte_val(phys) +#define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT) +#define pfn_pmd(pfn, prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT)\ + | pgprot_val(prot)) +#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) + +#define pud_young(pud) pte_young(pud_pte(pud)) +#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) +#define pud_write(pud) pte_write(pud_pte(pud)) + +#define pud_mkhuge(pud) (__pud(pud_val(pud) & ~PUD_TABLE_BIT)) + +#define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud)) +#define __phys_to_pud_val(phys) __phys_to_pte_val(phys) +#define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT) +#define pfn_pud(pfn, prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT)\ + | pgprot_val(prot)) + +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t pmd) +{ + unsigned long pfn = pmd_pfn(pmd); + + return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd)); +} + +#define pud_pgprot pud_pgprot +static inline pgprot_t pud_pgprot(pud_t pud) +{ + unsigned long pfn = pud_pfn(pud); + + return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud)); +} + +static inline void __set_pte_at(struct mm_struct *mm, + unsigned long __always_unused addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + __sync_cache_and_tags(pte, nr); + __check_safe_pte_update(mm, ptep, pte); + __set_pte(ptep, pte); +} + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + page_table_check_pmd_set(mm, pmdp, pmd); + return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd), + PMD_SIZE >> PAGE_SHIFT); +} + +static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t pud) +{ + page_table_check_pud_set(mm, pudp, pud); + return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud), + PUD_SIZE >> PAGE_SHIFT); +} + +#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d)) +#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys) + +#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd)) +#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys) + +#define __pgprot_modify(prot, mask, bits) \ + __pgprot((pgprot_val(prot) & ~(mask)) | (bits)) + +#define pgprot_nx(prot) \ + __pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN) + +#ifdef PROT_NS_SHARED +#define pgprot_decrypted(prot) \ + __pgprot_modify(prot, PROT_NS_SHARED, PROT_NS_SHARED) +#define pgprot_encrypted(prot) \ + __pgprot_modify(prot, PROT_NS_SHARED, 0) +#endif + +/* + * Mark the prot value as uncacheable and unbufferable. + */ +#define pgprot_noncached(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN) +#define pgprot_writecombine(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) +#define pgprot_device(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN) +#define pgprot_tagged(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED)) +#define pgprot_mhp pgprot_tagged +/* + * DMA allocations for non-coherent devices use what the Arm architecture calls + * "Normal non-cacheable" memory, which permits speculation, unaligned accesses + * and merging of writes. This is different from "Device-nGnR[nE]" memory which + * is intended for MMIO and thus forbids speculation, preserves access size, + * requires strict alignment and can also force write responses to come from the + * endpoint. + */ +#define pgprot_dmacoherent(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, \ + PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) + +#define __HAVE_PHYS_MEM_ACCESS_PROT +struct file; +extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t vma_prot); + +#define pmd_none(pmd) (!pmd_val(pmd)) + +#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ + PMD_TYPE_TABLE) +#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ + PMD_TYPE_SECT) +#define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd)) +#define pmd_bad(pmd) (!pmd_table(pmd)) + +#define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE) +#define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE) + +#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3 +static inline bool pud_sect(pud_t pud) { return false; } +static inline bool pud_table(pud_t pud) { return true; } +#else +#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ + PUD_TYPE_SECT) +#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ + PUD_TYPE_TABLE) +#endif + +extern pgd_t init_pg_dir[PTRS_PER_PGD]; +extern pgd_t init_pg_end[]; +extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; +extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; +extern pgd_t tramp_pg_dir[PTRS_PER_PGD]; +extern pgd_t reserved_pg_dir[PTRS_PER_PGD]; + +extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); + +static inline bool in_swapper_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)swapper_pg_dir & PAGE_MASK); +} + +static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) +{ +#ifdef __PAGETABLE_PMD_FOLDED + if (in_swapper_pgdir(pmdp)) { + set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); + return; + } +#endif /* __PAGETABLE_PMD_FOLDED */ + + if (iee_pgtable_ready()) + iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); + else + WRITE_ONCE(*pmdp, pmd); + + if (pmd_valid(pmd)) { + dsb(ishst); + isb(); + } +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(0)); +} + +static inline phys_addr_t pmd_page_paddr(pmd_t pmd) +{ + return __pmd_to_phys(pmd); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)__va(pmd_page_paddr(pmd)); +} + +/* Find an entry in the third-level page table. */ +#define pte_offset_phys(dir, addr) (pmd_page_paddr(READ_ONCE(*(dir))) \ + + pte_index(addr) * sizeof(pte_t)) + +#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) +#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) +#define pte_clear_fixmap() clear_fixmap(FIX_PTE) + +#define pmd_page(pmd) phys_to_page(__pmd_to_phys(pmd)) + +/* use ONLY for statically allocated translation tables */ +#define pte_offset_kimg(dir, addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr)))) + +/* + * Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + */ +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) + +#if CONFIG_PGTABLE_LEVELS > 2 + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) + +#define pud_none(pud) (!pud_val(pud)) +#define pud_bad(pud) (!pud_table(pud)) +#define pud_present(pud) pte_present(pud_pte(pud)) +#define pud_leaf(pud) (pud_present(pud) && !pud_table(pud)) +#define pud_valid(pud) pte_valid(pud_pte(pud)) +#define pud_user(pud) pte_user(pud_pte(pud)) +#define pud_user_exec(pud) pte_user_exec(pud_pte(pud)) + +static inline void set_pud(pud_t *pudp, pud_t pud) +{ +#ifdef __PAGETABLE_PUD_FOLDED + if (in_swapper_pgdir(pudp)) { + set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); + return; + } +#endif /* __PAGETABLE_PUD_FOLDED */ + + if (iee_pgtable_ready()) + iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); + else + WRITE_ONCE(*pudp, pud); + + if (pud_valid(pud)) { + dsb(ishst); + isb(); + } +} + +static inline void pud_clear(pud_t *pudp) +{ + set_pud(pudp, __pud(0)); +} + +static inline phys_addr_t pud_page_paddr(pud_t pud) +{ + return __pud_to_phys(pud); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)__va(pud_page_paddr(pud)); +} + +/* Find an entry in the second-level page table. */ +#define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) \ + + pmd_index(addr) * sizeof(pmd_t)) + +#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) +#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) +#define pmd_clear_fixmap() clear_fixmap(FIX_PMD) + +#define pud_page(pud) phys_to_page(__pud_to_phys(pud)) + +/* use ONLY for statically allocated translation tables */ +#define pmd_offset_kimg(dir, addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr)))) + +#else + +#define pud_page_paddr(pud) ({ BUILD_BUG(); 0; }) +#define pud_user_exec(pud) pud_user(pud) /* Always 0 with folding */ + +/* Match pmd_offset folding in */ +#define pmd_set_fixmap(addr) NULL +#define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp) +#define pmd_clear_fixmap() + +#define pmd_offset_kimg(dir, addr) ((pmd_t *)dir) + +#endif /* CONFIG_PGTABLE_LEVELS > 2 */ + +#if CONFIG_PGTABLE_LEVELS > 3 + +static __always_inline bool pgtable_l4_enabled(void) +{ +#if defined(CONFIG_ARM64_LPA2) && defined(ARM64_HAS_VA52) + if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) + return true; + if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) + return vabits_actual == VA_BITS; + return alternative_has_cap_unlikely(ARM64_HAS_VA52); +#else + return true; +#endif +} + +static inline bool mm_pud_folded(const struct mm_struct *mm) +{ + return !pgtable_l4_enabled(); +} +#define mm_pud_folded mm_pud_folded + +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e)) + +#define p4d_none(p4d) (pgtable_l4_enabled() && !p4d_val(p4d)) +#define p4d_bad(p4d) (pgtable_l4_enabled() && !(p4d_val(p4d) & 2)) +#define p4d_present(p4d) (!p4d_none(p4d)) +extern bool check_addr_in_iee_valid(unsigned long addr); +static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) +{ + if (in_swapper_pgdir(p4dp)) { + set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + return; + } + + if (in_tramp_pgdir(p4dp)) { + if (iee_pgtable_ready()) + iee_set_static_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + else { + WRITE_ONCE(*(pgd_t *)p4dp, __pgd(p4d_val(p4d))); + dsb(ishst); + isb(); + } + return; + } + + if (in_idmap_pgdir(p4dp)) { + if (iee_pgtable_ready()) + iee_set_static_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + else { + WRITE_ONCE(*(pgd_t *)p4dp, __pgd(p4d_val(p4d))); + dsb(ishst); + isb(); + } + return; + } + + if (iee_pgtable_ready()) + iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); + else + WRITE_ONCE(*p4dp, p4d); + + dsb(ishst); + isb(); +} + +static inline void p4d_clear(p4d_t *p4dp) +{ + if (pgtable_l4_enabled()) + set_p4d(p4dp, __p4d(0)); +} + +static inline phys_addr_t p4d_page_paddr(p4d_t p4d) +{ + return __p4d_to_phys(p4d); +} + +#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) + +static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) +{ + return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr); +} + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + return (pud_t *)__va(p4d_page_paddr(p4d)); +} + +/* Find an entry in the first-level page table. */ +static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) +{ + BUG_ON(!pgtable_l4_enabled()); + + return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); +} + +static inline +pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr) +{ + if (!pgtable_l4_enabled()) + return p4d_to_folded_pud(p4dp, addr); + return (pud_t *)__va(p4d_page_paddr(p4d)) + pud_index(addr); +} +#define pud_offset_lockless pud_offset_lockless + +static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long addr) +{ + return pud_offset_lockless(p4dp, READ_ONCE(*p4dp), addr); +} +#define pud_offset pud_offset + +static inline pud_t *pud_set_fixmap(unsigned long addr) +{ + if (!pgtable_l4_enabled()) + return NULL; + return (pud_t *)set_fixmap_offset(FIX_PUD, addr); +} + +static inline pud_t *pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr) +{ + if (!pgtable_l4_enabled()) + return p4d_to_folded_pud(p4dp, addr); + return pud_set_fixmap(pud_offset_phys(p4dp, addr)); +} + +static inline void pud_clear_fixmap(void) +{ + if (pgtable_l4_enabled()) + clear_fixmap(FIX_PUD); +} + +#define p4d_page(p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d))) + +/* use ONLY for statically allocated translation tables */ +static inline pud_t *pud_offset_kimg(p4d_t *p4dp, u64 addr) +{ + if (!pgtable_l4_enabled()) + return p4d_to_folded_pud(p4dp, addr); + return (pud_t *)__phys_to_kimg(pud_offset_phys(p4dp, addr)); +} + +#else + +static inline bool pgtable_l4_enabled(void) { return false; } + +#define p4d_page_paddr(p4d) ({ BUILD_BUG(); 0; }) +#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0; }) + +/* Match pud_offset folding in */ +#define pud_set_fixmap(addr) NULL +#define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) +#define pud_clear_fixmap() + +#define pud_offset_kimg(dir, addr) ((pud_t *)dir) + +#endif /* CONFIG_PGTABLE_LEVELS > 3 */ + +#if CONFIG_PGTABLE_LEVELS > 4 + +static __always_inline bool pgtable_l5_enabled(void) +{ +#if defined(ARM64_HAS_VA52) + if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) + return vabits_actual == VA_BITS; + return alternative_has_cap_unlikely(ARM64_HAS_VA52); +#else + return true; +#endif +} + +static inline bool mm_p4d_folded(const struct mm_struct *mm) +{ + return !pgtable_l5_enabled(); +} +#define mm_p4d_folded mm_p4d_folded + +#define p4d_ERROR(e) \ + pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e)) + +#define pgd_none(pgd) (pgtable_l5_enabled() && !pgd_val(pgd)) +#define pgd_bad(pgd) (pgtable_l5_enabled() && !(pgd_val(pgd) & 2)) +#define pgd_present(pgd) (!pgd_none(pgd)) + +static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) +{ + if (in_swapper_pgdir(pgdp)) { + set_swapper_pgd(pgdp, __pgd(pgd_val(pgd))); + return; + } + + WRITE_ONCE(*pgdp, pgd); + dsb(ishst); + isb(); +} + +static inline void pgd_clear(pgd_t *pgdp) +{ + if (pgtable_l5_enabled()) + set_pgd(pgdp, __pgd(0)); +} + +static inline phys_addr_t pgd_page_paddr(pgd_t pgd) +{ + return __pgd_to_phys(pgd); +} + +#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) + +static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr) +{ + return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr); +} + +static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr) +{ + BUG_ON(!pgtable_l5_enabled()); + + return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t); +} + +static inline +p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long addr) +{ + if (!pgtable_l5_enabled()) + return pgd_to_folded_p4d(pgdp, addr); + return (p4d_t *)__va(pgd_page_paddr(pgd)) + p4d_index(addr); +} +#define p4d_offset_lockless p4d_offset_lockless + +static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr) +{ + return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr); +} + +static inline p4d_t *p4d_set_fixmap(unsigned long addr) +{ + if (!pgtable_l5_enabled()) + return NULL; + return (p4d_t *)set_fixmap_offset(FIX_P4D, addr); +} + +static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr) +{ + if (!pgtable_l5_enabled()) + return pgd_to_folded_p4d(pgdp, addr); + return p4d_set_fixmap(p4d_offset_phys(pgdp, addr)); +} + +static inline void p4d_clear_fixmap(void) +{ + if (pgtable_l5_enabled()) + clear_fixmap(FIX_P4D); +} + +/* use ONLY for statically allocated translation tables */ +static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr) +{ + if (!pgtable_l5_enabled()) + return pgd_to_folded_p4d(pgdp, addr); + return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr)); +} + +#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd))) + +#else + +static inline bool pgtable_l5_enabled(void) { return false; } + +/* Match p4d_offset folding in */ +#define p4d_set_fixmap(addr) NULL +#define p4d_set_fixmap_offset(p4dp, addr) ((p4d_t *)p4dp) +#define p4d_clear_fixmap() + +#define p4d_offset_kimg(dir, addr) ((p4d_t *)dir) + +static inline +p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) +{ + /* + * With runtime folding of the pud, pud_offset_lockless() passes + * the 'pgd_t *' we return here to p4d_to_folded_pud(), which + * will offset the pointer assuming that it points into + * a page-table page. However, the fast GUP path passes us a + * pgd_t allocated on the stack and so we must use the original + * pointer in 'pgdp' to construct the p4d pointer instead of + * using the generic p4d_offset_lockless() implementation. + * + * Note: reusing the original pointer means that we may + * dereference the same (live) page-table entry multiple times. + * This is safe because it is still only loaded once in the + * context of each level and the CPU guarantees same-address + * read-after-read ordering. + */ + return p4d_offset(pgdp, addr); +} +#define p4d_offset_lockless p4d_offset_lockless_folded + +#endif /* CONFIG_PGTABLE_LEVELS > 4 */ + +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) + +#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) +#define pgd_clear_fixmap() clear_fixmap(FIX_PGD) + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + /* + * Normal and Normal-Tagged are two different memory types and indices + * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK. + */ + const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | + PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP | + PTE_ATTRINDX_MASK; + /* preserve the hardware dirty information */ + if (pte_hw_dirty(pte)) + pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + + pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); + /* + * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware + * dirtiness again. + */ + if (pte_sw_dirty(pte)) + pte = pte_mkdirty(pte); + return pte; +} + +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); +} + +extern int __ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +static inline int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + return __pmdp_set_access_flags(vma, address, pmdp, + entry, dirty); +} + +static inline int pud_devmap(pud_t pud) +{ + return 0; +} + +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} +#endif + +#ifdef CONFIG_PAGE_TABLE_CHECK +static inline bool pte_user_accessible_page(pte_t pte) +{ + return pte_present(pte) && (pte_user(pte) || pte_user_exec(pte)); +} + +static inline bool pmd_user_accessible_page(pmd_t pmd) +{ + return pmd_leaf(pmd) && !pmd_present_invalid(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd)); +} + +static inline bool pud_user_accessible_page(pud_t pud) +{ + return pud_leaf(pud) && (pud_user(pud) || pud_user_exec(pud)); +} +#endif + +/* + * Atomic pte/pmd modifications. + */ +static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + pte_t old_pte, pte; + + pte = __ptep_get(ptep); + do { + old_pte = pte; + pte = pte_mkold(pte); + pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, + pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); + + return pte_young(pte); +} + +static inline int __ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + int young = __ptep_test_and_clear_young(vma, address, ptep); + + if (young) { + /* + * We can elide the trailing DSB here since the worst that can + * happen is that a CPU continues to use the young entry in its + * TLB and we mistakenly reclaim the associated page. The + * window for such an event is bounded by the next + * context-switch, which provides a DSB to complete the TLB + * invalidation. + */ + flush_tlb_page_nosync(vma, address); + } + + return young; +} + +static inline int __pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + pmd_t old_pmd, pmd; + + pmd = READ_ONCE(*pmdp); + do { + old_pmd = pmd; + pmd = __pmd(pmd_val(pmd) & ~PTE_AF); + pmd_val(pmd) = iee_set_pmd_cmpxchg_relaxed(pmdp, + pmd_val(old_pmd), pmd_val(pmd)); + } while (pmd_val(pmd) != pmd_val(old_pmd)); + + return pmd_young(pmd); +} + +static inline void __pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp, + pmd_t pmd) +{ + pmd_t old_pmd; + + do { + old_pmd = pmd; + pmd = pmd_wrprotect(pmd); + pmd_val(pmd) = iee_set_pmd_cmpxchg_relaxed(pmdp, pmd_val(old_pmd), pmd_val(pmd)); + } while (pmd_val(pmd) != pmd_val(old_pmd)); +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) +#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + /* Operation applies to PMD table entry only if FEAT_HAFT is enabled */ + VM_WARN_ON(pmd_table(READ_ONCE(*pmdp)) && !system_supports_haft()); + return __pmdp_test_and_clear_young(vma, address, pmdp); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ + +static inline pte_t __ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + pteval_t pteval = iee_set_xchg_relaxed((pte_t *)&pte_val(*ptep), (pteval_t)0); + pte_t pte = __pte(pteval); + + page_table_check_pte_clear(mm, pte); + + return pte; +} + +static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full) +{ + for (;;) { + __ptep_get_and_clear(mm, addr, ptep); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} + +static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned int nr, int full) +{ + pte_t pte, tmp_pte; + + pte = __ptep_get_and_clear(mm, addr, ptep); + while (--nr) { + ptep++; + addr += PAGE_SIZE; + tmp_pte = __ptep_get_and_clear(mm, addr, ptep); + if (pte_dirty(tmp_pte)) + pte = pte_mkdirty(pte); + if (pte_young(tmp_pte)) + pte = pte_mkyoung(pte); + } + return pte; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + pteval_t pteval = iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pteval_t)0); + pmd_t pmd = __pmd(pteval); + + page_table_check_pmd_clear(mm, pmd); + + return pmd; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +static inline void ___ptep_set_wrprotect(struct mm_struct *mm, + unsigned long address, pte_t *ptep, + pte_t pte) +{ + pte_t old_pte; + + do { + old_pte = pte; + + pte = pte_wrprotect(pte); + pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); +} + +/* + * __ptep_set_wrprotect - mark read-only while trasferring potential hardware + * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. + */ +static inline void __ptep_set_wrprotect(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + ___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep)); +} + +static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, + pte_t *ptep, unsigned int nr) +{ + unsigned int i; + + for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + __ptep_set_wrprotect(mm, address, ptep); +} + +static inline void __clear_young_dirty_pte(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, cydp_t flags) +{ + pte_t old_pte; + + do { + old_pte = pte; + + if (flags & CYDP_CLEAR_YOUNG) + pte = pte_mkold(pte); + if (flags & CYDP_CLEAR_DIRTY) + pte = pte_mkclean(pte); + + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), + pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); +} + +static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + pte_t pte; + + for (;;) { + pte = __ptep_get(ptep); + + if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY)) + __set_pte(ptep, pte_mkclean(pte_mkold(pte))); + else + __clear_young_dirty_pte(vma, addr, ptep, pte, flags); + + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_SET_WRPROTECT +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + __pmdp_set_wrprotect(mm, address, pmdp, READ_ONCE(*pmdp)); +} + +#define pmdp_establish pmdp_establish +static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + return __pmd(iee_set_pmd_xchg_relaxed((pmd_t *)&pmd_val(*pmdp), pmd_val(pmd))); +} +#endif + +/* + * Encode and decode a swap entry: + * bits 0-1: present (must be zero) + * bits 2: remember PG_anon_exclusive + * bits 3-7: swap type + * bits 8-57: swap offset + * bit 58: PTE_PROT_NONE (must be zero) + */ +#define __SWP_TYPE_SHIFT 3 +#define __SWP_TYPE_BITS 5 +#define __SWP_OFFSET_BITS 50 +#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) +#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) +#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) + +#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) +#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK) +#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) \ + | ((offset) << __SWP_OFFSET_SHIFT) }) + +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) + +#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) +#define __swp_entry_to_pmd(swp) __pmd((swp).val) +#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ + +/* + * Ensure that there are not more swap files than can be encoded in the kernel + * PTEs. + */ +#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) + +#ifdef CONFIG_ARM64_MTE + +#define __HAVE_ARCH_PREPARE_TO_SWAP +static inline int arch_prepare_to_swap(struct page *page) +{ + if (system_supports_mte()) + return mte_save_tags(page); + return 0; +} + +#define __HAVE_ARCH_SWAP_INVALIDATE +static inline void arch_swap_invalidate_page(int type, pgoff_t offset) +{ + if (system_supports_mte()) + mte_invalidate_tags(type, offset); +} + +static inline void arch_swap_invalidate_area(int type) +{ + if (system_supports_mte()) + mte_invalidate_tags_area(type); +} + +#define __HAVE_ARCH_SWAP_RESTORE +static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) +{ + if (system_supports_mte()) + mte_restore_tags(entry, &folio->page); +} + +#endif /* CONFIG_ARM64_MTE */ + +/* + * On AArch64, the cache coherency is handled via the __set_ptes() function. + */ +static inline void update_mmu_cache_range(struct vm_fault *vmf, + struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, + unsigned int nr) +{ + /* + * We don't do anything here, so there's a very small chance of + * us retaking a user fault which we just fixed up. The alternative + * is doing a dsb(ishst), but that penalises the fastpath. + */ +} + +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) +#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) + +#ifdef CONFIG_ARM64_PA_BITS_52 +#define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52) +#else +#define phys_to_ttbr(addr) (addr) +#endif + +/* + * On arm64 without hardware Access Flag, copying from user will fail because + * the pte is old and cannot be marked young. So we always end up with zeroed + * page after fork() + CoW for pfn mappings. We don't always have a + * hardware-managed access flag on arm64. + */ +#define arch_has_hw_pte_young cpu_has_hw_af + +#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG +#define arch_has_hw_nonleaf_pmd_young system_supports_haft +#endif + +/* + * Experimentally, it's cheap to set the access flag in hardware and we + * benefit from prefaulting mappings as 'old' to start with. + */ +#define arch_wants_old_prefaulted_pte cpu_has_hw_af + +/* + * Request exec memory is read into pagecache in at least 64K folios. The + * trade-off here is performance improvement due to storing translations more + * effciently in the iTLB vs the potential for read amplification due to reading + * data from disk that won't be used. The latter is independent of base page + * size, so we set a page-size independent block size of 64K. This size can be + * contpte-mapped when 4K base pages are in use (16 pages into 1 iTLB entry), + * and HPA can coalesce it (4 pages into 1 TLB entry) when 16K base pages are in + * use. + */ +#define arch_wants_exec_folio_order() ilog2(SZ_64K >> PAGE_SHIFT) + +static inline bool pud_sect_supported(void) +{ + return PAGE_SIZE == SZ_4K; +} + + +#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION +#define ptep_modify_prot_start ptep_modify_prot_start +extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); + +#define ptep_modify_prot_commit ptep_modify_prot_commit +extern void ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte); + +#define modify_prot_start_ptes modify_prot_start_ptes +extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr); + +#define modify_prot_commit_ptes modify_prot_commit_ptes +extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep, pte_t old_pte, pte_t pte, + unsigned int nr); + +#ifdef CONFIG_ARM64_CONTPTE + +/* + * The contpte APIs are used to transparently manage the contiguous bit in ptes + * where it is possible and makes sense to do so. The PTE_CONT bit is considered + * a private implementation detail of the public ptep API (see below). + */ +extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); +extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); +extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte); +extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep); +extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr); +extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full); +extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned int nr, int full); +extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr); +extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty); +extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags); + +static __always_inline void contpte_try_fold(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pte) +{ + /* + * Only bother trying if both the virtual and physical addresses are + * aligned and correspond to the last entry in a contig range. The core + * code mostly modifies ranges from low to high, so this is the likely + * the last modification in the contig range, so a good time to fold. + * We can't fold special mappings, because there is no associated folio. + */ + + const unsigned long contmask = CONT_PTES - 1; + bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + + if (unlikely(valign)) { + bool palign = (pte_pfn(pte) & contmask) == contmask; + + if (unlikely(palign && + pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) + __contpte_try_fold(mm, addr, ptep, pte); + } +} + +static __always_inline void contpte_try_unfold(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pte) +{ + if (unlikely(pte_valid_cont(pte))) + __contpte_try_unfold(mm, addr, ptep, pte); +} + +#define pte_batch_hint pte_batch_hint +static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) +{ + if (!pte_valid_cont(pte)) + return 1; + + return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); +} + +/* + * The below functions constitute the public API that arm64 presents to the + * core-mm to manipulate PTE entries within their page tables (or at least this + * is the subset of the API that arm64 needs to implement). These public + * versions will automatically and transparently apply the contiguous bit where + * it makes sense to do so. Therefore any users that are contig-aware (e.g. + * hugetlb, kernel mapper) should NOT use these APIs, but instead use the + * private versions, which are prefixed with double underscore. All of these + * APIs except for ptep_get_lockless() are expected to be called with the PTL + * held. Although the contiguous bit is considered private to the + * implementation, it is deliberately allowed to leak through the getters (e.g. + * ptep_get()), back to core code. This is required so that pte_leaf_size() can + * provide an accurate size for perf_get_pgtable_size(). But this leakage means + * its possible a pte will be passed to a setter with the contiguous bit set, so + * we explicitly clear the contiguous bit in those cases to prevent accidentally + * setting it in the pgtable. + */ + +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + pte_t pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(pte))) + return pte; + + return contpte_ptep_get(ptep, pte); +} + +#define ptep_get_lockless ptep_get_lockless +static inline pte_t ptep_get_lockless(pte_t *ptep) +{ + pte_t pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(pte))) + return pte; + + return contpte_ptep_get_lockless(ptep); +} + +static inline void set_pte(pte_t *ptep, pte_t pte) +{ + /* + * We don't have the mm or vaddr so cannot unfold contig entries (since + * it requires tlb maintenance). set_pte() is not used in core code, so + * this should never even be called. Regardless do our best to service + * any call and emit a warning if there is any attempt to set a pte on + * top of an existing contig range. + */ + pte_t orig_pte = __ptep_get(ptep); + + WARN_ON_ONCE(pte_valid_cont(orig_pte)); + __set_pte(ptep, pte_mknoncont(pte)); +} + +#define set_ptes set_ptes +static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + pte = pte_mknoncont(pte); + + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __set_ptes(mm, addr, ptep, pte, 1); + contpte_try_fold(mm, addr, ptep, pte); + } else { + contpte_set_ptes(mm, addr, ptep, pte, nr); + } +} + +static inline void pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __pte_clear(mm, addr, ptep); +} + +#define clear_full_ptes clear_full_ptes +static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full) +{ + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __clear_full_ptes(mm, addr, ptep, nr, full); + } else { + contpte_clear_full_ptes(mm, addr, ptep, nr, full); + } +} + +#define get_and_clear_full_ptes get_and_clear_full_ptes +static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned int nr, int full) +{ + pte_t pte; + + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full); + } else { + pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full); + } + + return pte; +} + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + return __ptep_get_and_clear(mm, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t orig_pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_test_and_clear_young(vma, addr, ptep); + + return contpte_ptep_test_and_clear_young(vma, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +static inline int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t orig_pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_clear_flush_young(vma, addr, ptep); + + return contpte_ptep_clear_flush_young(vma, addr, ptep); +} + +#define wrprotect_ptes wrprotect_ptes +static __always_inline void wrprotect_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) +{ + if (likely(nr == 1)) { + /* + * Optimization: wrprotect_ptes() can only be called for present + * ptes so we only need to check contig bit as condition for + * unfold, and we can remove the contig bit from the pte we read + * to avoid re-reading. This speeds up fork() which is sensitive + * for order-0 folios. Equivalent to contpte_try_unfold(). + */ + pte_t orig_pte = __ptep_get(ptep); + + if (unlikely(pte_cont(orig_pte))) { + __contpte_try_unfold(mm, addr, ptep, orig_pte); + orig_pte = pte_mknoncont(orig_pte); + } + ___ptep_set_wrprotect(mm, addr, ptep, orig_pte); + } else { + contpte_wrprotect_ptes(mm, addr, ptep, nr); + } +} + +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + wrprotect_ptes(mm, addr, ptep, 1); +} + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty) +{ + pte_t orig_pte = __ptep_get(ptep); + + entry = pte_mknoncont(entry); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_set_access_flags(vma, addr, ptep, entry, dirty); + + return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty); +} + +#define clear_young_dirty_ptes clear_young_dirty_ptes +static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + if (likely(nr == 1 && !pte_cont(__ptep_get(ptep)))) + __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); + else + contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags); +} + +#else /* CONFIG_ARM64_CONTPTE */ + +#define ptep_get __ptep_get +#define set_pte __set_pte +#define set_ptes __set_ptes +#define pte_clear __pte_clear +#define clear_full_ptes __clear_full_ptes +#define get_and_clear_full_ptes __get_and_clear_full_ptes +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +#define ptep_get_and_clear __ptep_get_and_clear +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +#define ptep_test_and_clear_young __ptep_test_and_clear_young +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +#define ptep_clear_flush_young __ptep_clear_flush_young +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +#define ptep_set_wrprotect __ptep_set_wrprotect +#define wrprotect_ptes __wrprotect_ptes +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +#define ptep_set_access_flags __ptep_set_access_flags +#define clear_young_dirty_ptes __clear_young_dirty_ptes + +#endif /* CONFIG_ARM64_CONTPTE */ + +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +void vmemmap_update_pmd(unsigned long addr, pmd_t *pmdp, pte_t *ptep); +#define vmemmap_update_pmd vmemmap_update_pmd +void vmemmap_update_pte(unsigned long addr, pte_t *ptep, pte_t pte); +#define vmemmap_update_pte vmemmap_update_pte +#endif + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/include/asm/haoc/iee-ptp-init.h b/arch/arm64/include/asm/haoc/iee-ptp-init.h new file mode 100644 index 0000000000000..bfd554611d2ab --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-ptp-init.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_PTP_INIT_H +#define _LINUX_IEE_PTP_INIT_H + +#include + +#define NR_BM_PTE_TABLES \ + SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PMD_SHIFT) +#define NR_BM_PMD_TABLES \ + SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PUD_SHIFT) + +static_assert(NR_BM_PMD_TABLES == 1); + +#define __BM_TABLE_IDX(addr, shift) \ + (((addr) >> (shift)) - (FIXADDR_TOT_START >> (shift))) + +#define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) + +extern pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __section(".iee.ptp") __aligned(PAGE_SIZE); +extern pmd_t bm_pmd[PTRS_PER_PMD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +extern pud_t bm_pud[PTRS_PER_PUD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; + +extern void __iee_pgd_populate_pre_init(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot); +extern void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot); +extern void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot); +extern void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, + pmdval_t prot); +extern int iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot); +extern int iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot); + +extern void __iee_set_fixmap_pre_init(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags); + +extern void iee_set_pgtable_pre_init(unsigned long *addr, unsigned long content); +extern void set_iee_address_pre_init(unsigned long addr, bool valid); +extern void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags); +extern void __create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags); + +extern void __init efi_memmap_unmap_after_init(void); + +extern int early_ioremap_debug __initdata; +extern int after_paging_init __initdata; +extern void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; +extern unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; +extern unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; + +extern bool pgattr_change_is_safe(u64 old, u64 new); + +#endif diff --git a/arch/arm64/include/asm/haoc/iee.h b/arch/arm64/include/asm/haoc/iee.h index 7137aa186d29a..c0024ff619ff9 100644 --- a/arch/arm64/include/asm/haoc/iee.h +++ b/arch/arm64/include/asm/haoc/iee.h @@ -34,19 +34,27 @@ extern bool haoc_enabled; /* Support conversion from both kernel and linear addresses. */ #define __ptr_to_iee(x) ({ \ typeof(x) __val; \ - if (__is_lm_address((u64)x)) \ + if (!haoc_enabled) \ + __val = x; \ + else if (__is_lm_address((u64)x)) \ __val = ((typeof(x))(__virt_to_iee((u64)x))); \ - else \ + else if ((u64)x < (u64)KERNEL_END && (u64)x >= (u64)KERNEL_START) \ __val = ((typeof(x))(__kimg_to_iee((u64)x))); \ + else \ + __val = x; \ __val; \ }) #define __addr_to_iee(x) ({ \ u64 __val; \ - if (__is_lm_address((u64)x)) \ + if (!haoc_enabled) \ + __val = (u64)x; \ + else if (__is_lm_address((u64)x)) \ __val = __virt_to_iee((u64)x); \ - else \ + else if ((u64)x < (u64)KERNEL_END && (u64)x >= (u64)KERNEL_START) \ __val = __kimg_to_iee((u64)x); \ + else \ + __val = (u64)x; \ __val; \ }) diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 661964e99b9d5..a171c57f635ba 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -14,6 +14,9 @@ #include #define __HAVE_ARCH_PGD_FREE +#if CONFIG_PGTABLE_LEVELS > 3 +#define __HAVE_ARCH_PUD_FREE +#endif #include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -53,6 +56,18 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; __p4d_populate(p4dp, __pa(pudp), p4dval); } + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + if (!pgtable_l4_enabled()) + return; + BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); +#ifdef CONFIG_PTP + pagetable_free(virt_to_ptdesc(pud)); +#else + free_page((unsigned long)pud); +#endif +} #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) { diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 6e92be4a1fa81..129ea4adf4a19 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -5,6 +5,10 @@ #ifndef __ASM_PGTABLE_H #define __ASM_PGTABLE_H +#ifdef CONFIG_PTP +#include +#else + #include #include @@ -1471,4 +1475,6 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma, #endif /* !__ASSEMBLY__ */ +#endif /* !CONFIG_PTP */ + #endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index ce21e6be73116..030081d8ce9c2 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,10 +10,18 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif static inline void __tlb_remove_table(void *_table) { + #ifdef CONFIG_PTP + // set_iee_address_invalid((u64)page_to_virt((struct page *)_table)); + iee_cache_free(&pg_cache, page_to_virt((struct page *)_table)); + #else free_page_and_swap_cache((struct page *)_table); + #endif } #define tlb_flush tlb_flush diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index ac148814ffbf2..c1e42241256cc 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -80,6 +80,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif #include #include #include @@ -1835,7 +1838,12 @@ static int __init __kpti_install_ng_mappings(void *__unused) remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); if (!cpu) { + #ifdef CONFIG_PTP + /* CPU 0 is never hotpluged so we could call init function here. */ + alloc = (u64)__va(early_iee_pgtable_alloc(0)); + #else alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); + #endif kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE); kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd); diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index 10b43695101e2..e34960c41acde 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -60,4 +60,18 @@ config CREDP by IEE either. If unsure, say N. +config PTP + bool "Page Table Protection(PTP)" + depends on IEE + help + Protect page tables by routing page-table modification paths + through the IEE gate. This keeps page table updates mediated by + the isolated execution environment instead of allowing direct + writes from normal kernel code. + +config PTP_RESERVE_ORDER + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + depends on PTP + range 9 15 + default 12 endmenu # HAOC diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile index 0bcecf0ea9876..f874c584bb86f 100644 --- a/arch/arm64/kernel/haoc/Makefile +++ b/arch/arm64/kernel/haoc/Makefile @@ -2,3 +2,4 @@ obj-y += haoc.o haoc-bitmap.o obj-y += iee/ obj-$(CONFIG_CREDP) += credp/ +obj-$(CONFIG_PTP) += ptp/ diff --git a/arch/arm64/kernel/haoc/haoc-bitmap.c b/arch/arm64/kernel/haoc/haoc-bitmap.c index aa82948d46e0a..d1c162b91ca0a 100644 --- a/arch/arm64/kernel/haoc/haoc-bitmap.c +++ b/arch/arm64/kernel/haoc/haoc-bitmap.c @@ -47,11 +47,12 @@ static pte_t * __init haoc_bitmap_pte_populate(pmd_t *pmd, unsigned long addr) if (!p) return NULL; - entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, SET_NG(PAGE_KERNEL)); #ifdef CONFIG_PTP write_sysreg((read_sysreg(TCR_EL1) | TCR_HPD1) & ~TCR_A1, tcr_el1); isb(); WRITE_ONCE(*__ptr_to_iee(pte), entry); + dsb(ishst); write_sysreg((read_sysreg(TCR_EL1) & ~TCR_HPD1) | TCR_A1, tcr_el1); isb(); #else diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index 0567c8caf58f2..3d2046e452398 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -66,6 +66,21 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_cred_security, (iee_func)_iee_set_cred_rcu, (iee_func)_iee_set_cred_ucounts, +#endif +#ifdef CONFIG_PTP + (iee_func)_iee_set_static_pgd, + (iee_func)_iee_set_bm_pte, + (iee_func)_iee_set_pte, + (iee_func)_iee_set_pmd, + (iee_func)_iee_set_pud, + (iee_func)_iee_set_p4d, + (iee_func)_iee_set_swapper_pgd, + (iee_func)_iee_set_xchg_relaxed, + (iee_func)_iee_set_pmd_xchg_relaxed, + (iee_func)_iee_set_cmpxchg_relaxed, + (iee_func)_iee_set_pmd_cmpxchg_relaxed, + (iee_func)_iee_set_sensitive_pte, + (iee_func)_iee_unset_sensitive_pte, #endif NULL }; diff --git a/arch/arm64/kernel/haoc/iee/iee-func.c b/arch/arm64/kernel/haoc/iee/iee-func.c index ab600a78a223e..8853b3738ceec 100644 --- a/arch/arm64/kernel/haoc/iee/iee-func.c +++ b/arch/arm64/kernel/haoc/iee/iee-func.c @@ -51,6 +51,7 @@ static void iee_may_split_pmd(pud_t *pudp, unsigned long addr, unsigned int orde } } + #ifdef CONFIG_IEE_ALLOW_SPLIT_LM /* May split Block Descriptor. */ if (!iee_support_pmd_block(addr, order)) { struct page *page = pmd_page(*pmdp); @@ -88,6 +89,9 @@ static void iee_may_split_pmd(pud_t *pudp, unsigned long addr, unsigned int orde if (pgtable) pte_free_kernel(&init_mm, pgtable); } + #endif + dsb(ishst); + isb(); } /* @@ -222,6 +226,10 @@ void set_iee_address_invalid(unsigned long lm_addr, unsigned int order) static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) { +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_SET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, + use_block_pmd); +#else int i; if (use_block_pmd) { @@ -251,10 +259,15 @@ static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, } dsb(ishst); isb(); +#endif } static void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) { +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_UNSET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, + use_block_pmd); +#else int i; if (use_block_pmd) { @@ -284,6 +297,7 @@ static void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, } dsb(ishst); isb(); +#endif } /* Only support address range smaller then one PMD block. */ @@ -405,20 +419,21 @@ static void remove_pages_from_iee(unsigned long addr, int order) /* See put_pages_into_iee(). */ void set_iee_page_type(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) { - put_pages_into_iee(addr, order); - iee_set_bitmap_type(addr, 1UL << order, type); + set_iee_page(addr, order, type); } /* See put_pages_into_iee(). */ -void set_iee_page(unsigned long addr, int order) +void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) { put_pages_into_iee(addr, order); + iee_set_bitmap_type(addr, 1UL << order, type); } /* See remove_pages_from_iee(). */ void unset_iee_page(unsigned long addr, int order) { remove_pages_from_iee(addr, order); + iee_set_bitmap_type(addr, 1UL << order, IEE_NORMAL); } unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) diff --git a/arch/arm64/kernel/haoc/iee/iee-init.c b/arch/arm64/kernel/haoc/iee/iee-init.c index f328e715f58bd..d9e4b57faade0 100644 --- a/arch/arm64/kernel/haoc/iee/iee-init.c +++ b/arch/arm64/kernel/haoc/iee/iee-init.c @@ -91,7 +91,6 @@ void __init iee_init_post(void) iee_si_init(); #endif } - void __init iee_stack_init(void) { if (!haoc_enabled) diff --git a/arch/arm64/kernel/haoc/iee/iee-mmu.c b/arch/arm64/kernel/haoc/iee/iee-mmu.c index 91ad698d612e4..e3b5d675ea0fe 100644 --- a/arch/arm64/kernel/haoc/iee/iee-mmu.c +++ b/arch/arm64/kernel/haoc/iee/iee-mmu.c @@ -17,6 +17,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif #define IEE_EARLY_BLOCK_NR 64 @@ -44,7 +47,14 @@ static struct iee_early_alloc iee_stack = { .curr_block_nr = -1 }; -static DEFINE_MUTEX(fixmap_lock); +#ifdef CONFIG_PTP +static struct iee_early_alloc iee_pgtable = { + .name = "iee_early_pgtable", + .curr_block_nr = -1 +}; +#endif + +DEFINE_MUTEX(fixmap_lock); __aligned(PAGE_SIZE) DECLARE_PER_CPU(u64*[(PAGE_SIZE/8)], iee_cpu_stack_ptr); @@ -52,6 +62,17 @@ __aligned(PAGE_SIZE) DECLARE_PER_CPU(u64*[(PAGE_SIZE/8)], /* reserve 8 pages for iee init stack. */ __aligned(IEE_STACK_SIZE) __initdata u64 iee_init_stack[IEE_STACK_SIZE/8]; +#ifdef CONFIG_PTP +void __init iee_init_tcr(void) +{ + unsigned long ptr = (unsigned long)pte_set_fixmap_pre_init(__pa_symbol(&kernel_tcr)); + *((u64 *)ptr) = read_sysreg(tcr_el1) & IEE_TCR_MASK & ~TCR_HPD1; + pte_clear_fixmap_pre_init(); + ptr = (unsigned long)pte_set_fixmap_pre_init(__pa_symbol(&iee_tcr)); + *((u64 *)ptr) = (kernel_tcr | TCR_HPD1) & ~TCR_A1; + pte_clear_fixmap_pre_init(); +} +#else /* Setup global values used in verifications of TCR_EL1 to protect IEE switch gate. * Use fixmap functions as these globals are put inside IEE text section. */ @@ -69,6 +90,7 @@ void __init iee_init_tcr(void) *((u64 *)ptr) = (kernel_tcr | TCR_HPD1) & ~TCR_A1; clear_fixmap(FIX_PTE); } +#endif static void __init iee_setup_bootcpu_stack(void) { @@ -106,7 +128,11 @@ static phys_addr_t __init iee_mem_pool_early_alloc(struct iee_early_alloc *cache * any level of table. */ for (i = 0; i < (1 << (order)); i++) { +#ifdef CONFIG_PTP + ptr = pte_set_fixmap_pre_init(phys + i * PAGE_SIZE); +#else ptr = pte_set_fixmap(phys + i * PAGE_SIZE); +#endif memset(ptr, 0, PAGE_SIZE); @@ -114,7 +140,11 @@ static phys_addr_t __init iee_mem_pool_early_alloc(struct iee_early_alloc *cache * Implicit barriers also ensure the zeroed page is visible to the page * table walker */ +#ifdef CONFIG_PTP + pte_clear_fixmap_pre_init(); +#else pte_clear_fixmap(); +#endif } cache->begin = phys; @@ -157,11 +187,14 @@ void __init early_iee_data_cache_init(void) /* Calculate IEE stack alloc block size. */ iee_mem_pool_early_alloc(&iee_stack, IEE_DATA_ORDER); /* Calculate IEE data alloc block size. */ - iee_mem_pool_early_alloc(&iee_data, get_iee_alloc_order(1)); + iee_mem_pool_early_alloc(&iee_data, get_iee_alloc_order(0)); +#ifdef CONFIG_PTP + iee_mem_pool_early_alloc(&iee_pgtable, get_iee_alloc_order(1)); +#endif } phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, - int order) + int order, enum HAOC_BITMAP_TYPE type) { phys_addr_t phys; phys_addr_t expand_phys; @@ -179,7 +212,9 @@ phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, /* Put the expanded memory into IEE if late enough. */ block_order = cache->blocks[cache->curr_block_nr].order; - if (iee_init_done) + if (haoc_bitmap_ready) + set_iee_page((unsigned long)__va(expand_phys), block_order, type); + else if (iee_init_done) put_pages_into_iee((unsigned long)__va(expand_phys), block_order); goto redo; } @@ -194,14 +229,15 @@ phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, */ phys_addr_t __init early_iee_stack_alloc(int order) { - return iee_early_alloc(&iee_stack, order); + return iee_early_alloc(&iee_stack, order, IEE_DATA); } -phys_addr_t __init early_iee_data_alloc(int shift) +phys_addr_t __init early_iee_data_alloc(int order) { - return iee_early_alloc(&iee_data, 0); + return iee_early_alloc(&iee_data, order, IEE_DATA); } +#ifndef CONFIG_PTP static void iee_clear_pgtable(phys_addr_t phys) { void *table = pte_set_fixmap(phys); @@ -209,9 +245,20 @@ static void iee_clear_pgtable(phys_addr_t phys) clear_page(table); pte_clear_fixmap(); } +#endif + +#ifdef CONFIG_PTP +phys_addr_t __init early_iee_pgtable_alloc(int shift) +{ + return iee_early_alloc(&iee_pgtable, 0, IEE_PGTABLE); +} +#endif -static phys_addr_t __init early_pgtable_alloc(int shift) +static phys_addr_t __init iee_early_pgtable_alloc(int shift) { +#ifdef CONFIG_PTP + return early_iee_pgtable_alloc(shift); +#else phys_addr_t phys; phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, @@ -220,8 +267,10 @@ static phys_addr_t __init early_pgtable_alloc(int shift) panic("Failed to allocate page table page\n"); return phys; +#endif } +#ifndef CONFIG_PTP static void iee_init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot) { @@ -257,7 +306,7 @@ static void iee_alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, IEE_CHECK(pmd_sect(pmd)); if (pmd_none(pmd)) { - pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_SECT_AF; + pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF; phys_addr_t pte_phys; if (flags & NO_EXEC_MAPPINGS) @@ -336,11 +385,13 @@ static void iee_alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, */ IEE_CHECK(pud_sect(pud)); if (pud_none(pud)) { - pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PMD_SECT_AF; + pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF; phys_addr_t pmd_phys; if (flags & NO_EXEC_MAPPINGS) pudval |= PUD_TABLE_PXN; + if ((flags & IS_IEE_MAPPINGS) && TOP_PAGE_TABLE(3)) + pudval |= PGD_APTABLE_RO; IEE_CHECK(!pgtable_alloc); pmd_phys = pgtable_alloc(PMD_SHIFT); iee_clear_pgtable(pmd_phys); @@ -376,11 +427,13 @@ static void iee_alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long en p4d_t p4d = READ_ONCE(*p4dp); if (p4d_none(p4d)) { - p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | PMD_SECT_AF; + p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF; phys_addr_t pud_phys; if (flags & NO_EXEC_MAPPINGS) p4dval |= P4D_TABLE_PXN; + if ((flags & IS_IEE_MAPPINGS) && TOP_PAGE_TABLE(4)) + p4dval |= PGD_APTABLE_RO; IEE_CHECK(!pgtable_alloc); pud_phys = pgtable_alloc(PUD_SHIFT); iee_clear_pgtable(pud_phys); @@ -405,6 +458,7 @@ static void iee_alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long en pud_clear_fixmap(); } +#endif /* This function is almost the same with __create_pgd_mapping_locked() * but not permitting block descriptors larger than pmd block to simplify @@ -432,53 +486,13 @@ void __iee_create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, do { next = pgd_addr_end(addr, end); - iee_alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, +#ifdef CONFIG_PTP + iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, flags); - phys += next - addr; - } while (pgdp++, addr = next, addr != end); -} - -/* Mark the pgd entry of IEE address ranges with APTable to setup isolation. */ -static void __init __create_pgd_mapping_for_iee_locked(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(int), - int flags) -{ - unsigned long addr, end, next; - pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); - p4d_t *p4dp; - p4d_t p4d; - - /* - * If the virtual and physical address don't have the same offset - * within a page, we cannot map the region as the caller expects. - */ - if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) - return; - - phys &= PAGE_MASK; - addr = virt & PAGE_MASK; - end = PAGE_ALIGN(virt + size); - - do { - next = pgd_addr_end(addr, end); - p4dp = p4d_offset(pgdp, addr); - p4d = READ_ONCE(*p4dp); - if (!p4d_none(p4d) && !(p4d_val(p4d) & PGD_APTABLE_RO)) { - phys += next - addr; - continue; - } +#else iee_alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, flags); - - /* Set APTable RO on pgd entries of IEE mappings to prevent kernel access - * when TCR.HPD1 == 0. - */ - p4d = READ_ONCE(*p4dp); - __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APTABLE_RO | PGD_PXNTABLE | - PGD_UXNTABLE | PUD_TYPE_TABLE)); - +#endif phys += next - addr; } while (pgdp++, addr = next, addr != end); } @@ -490,7 +504,7 @@ static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, int flags) { mutex_lock(&fixmap_lock); - __create_pgd_mapping_for_iee_locked(pgdir, phys, virt, size, prot, + __iee_create_pgd_mapping_locked(pgdir, phys, virt, size, prot, pgtable_alloc, flags); mutex_unlock(&fixmap_lock); } @@ -499,7 +513,7 @@ static void __init __map_memblock_for_iee(pgd_t *pgdp, phys_addr_t start, phys_addr_t end, pgprot_t prot, int flags) { __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, - prot, early_pgtable_alloc, flags); + prot, iee_early_pgtable_alloc, flags); } /* @@ -532,7 +546,7 @@ void __init iee_init_mappings(pgd_t *pgdp) * Not allowing block or continuous mappings on IEE for faster page * attribution modification. */ - flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; + flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | IS_IEE_MAPPINGS; /* map all the memory banks non-executable but invalid on iee addresses. */ for_each_mem_range(i, &start, &end) { @@ -561,7 +575,24 @@ void __init setup_iee_early_data_bitmap(void) { setup_iee_data_cache_bitmap(&iee_data, IEE_DATA); setup_iee_data_cache_bitmap(&iee_stack, IEE_DATA); +#ifdef CONFIG_PTP + setup_iee_data_cache_bitmap(&iee_pgtable, IEE_PGTABLE); +#endif +} + +#ifdef CONFIG_PTP +static void setup_iee_early_address(struct iee_early_alloc *cache) +{ + int i, j; + + for (j = 0; j < cache->curr_block_nr + 1; j++) { + for (i = 0; i < (1 << cache->blocks[j].order); i++) { + set_iee_address_pre_init(__phys_to_iee(cache->blocks[j].start + + i * PAGE_SIZE), true); + } + } } +#endif static void prot_iee_early_data_cache(struct iee_early_alloc *cache) { @@ -576,12 +607,89 @@ static void prot_iee_early_data_cache(struct iee_early_alloc *cache) /* Put early allocated pages into IEE. */ void __init init_early_iee_data(void) { - u64 i; + int i; + if (!haoc_enabled) return; +#ifdef CONFIG_PTP + /* Setup iee mappings of early allocated IEE objects to enable IEE. */ + for (i = 0; ((unsigned long)idmap_pg_dir + i * PAGE_SIZE) < + (unsigned long)iee_init_data_end; i++) { + set_iee_address_pre_init(__phys_to_iee(__pa_symbol((unsigned long)idmap_pg_dir + + i * PAGE_SIZE)), true); + } + for (i = 0; ((unsigned long)init_pg_dir + i * PAGE_SIZE) < + (unsigned long)init_pg_end; i++) { + set_iee_address_pre_init(__phys_to_iee(__pa_symbol((unsigned long)init_pg_dir + + i * PAGE_SIZE)), true); + } + setup_iee_early_address(&iee_pgtable); + setup_iee_early_address(&iee_data); + prot_iee_early_data_cache(&iee_pgtable); +#else for (i = 0; (iee_init_data_begin + i * PAGE_SIZE) < iee_init_data_end; i++) set_iee_address(__phys_to_iee(__pa_symbol(iee_init_data_begin + i * PAGE_SIZE)), 0, true); +#endif + prot_iee_early_data_cache(&iee_data); prot_iee_early_data_cache(&iee_stack); } + +#ifdef CONFIG_PTP +int __pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + pmdval_t old_pmdval, pmdval; + pmd_t pmd = READ_ONCE(*pmdp); + + if (pmd_same(pmd, entry)) + return 0; + + /* only preserve the access flags and write permission */ + pmd_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; + + /* + * Setting the flags must be done atomically to avoid racing with the + * hardware update of the access/dirty state. The PTE_RDONLY bit must + * be set to the most permissive (lowest value) of *ptep and entry + * (calculated as: a & b == ~(~a | ~b)). + */ + pmd_val(entry) ^= PTE_RDONLY; + pmdval = pmd_val(pmd); + do { + old_pmdval = pmdval; + pmdval ^= PTE_RDONLY; + pmdval |= pmd_val(entry); + pmdval ^= PTE_RDONLY; + pmdval = iee_set_pmd_cmpxchg_relaxed(pmdp, old_pmdval, pmdval); + } while (pmdval != old_pmdval); + + /* Invalidate a stale read-only entry */ + if (dirty) + flush_tlb_page(vma, address); + return 1; +} + +void * __ref __ptp_vmemmap_alloc_block(unsigned long size, int node) +{ + int order = get_order(size); + void *p; + + if (!haoc_enabled) { + p = vmemmap_alloc_block(size, node); + if (p) + memset(p, 0, size); + return p; + } + + /* If the main allocator is up use that, fallback to bootmem. */ + if (slab_is_available()) + return iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); + + if (order != 0) + panic("PTP: Unsupport vmemmap alloc."); + return __va(early_iee_pgtable_alloc(0)); +} +#endif diff --git a/arch/arm64/kernel/haoc/iee/iee-token.c b/arch/arm64/kernel/haoc/iee/iee-token.c index c293943b94ae3..28e9506d1ff83 100644 --- a/arch/arm64/kernel/haoc/iee/iee-token.c +++ b/arch/arm64/kernel/haoc/iee/iee-token.c @@ -36,10 +36,11 @@ void __init iee_prepare_init_task_token(void) | __phys_to_pte_val(init_token_page)); /* Manaully go through IEE gates to bypass PTP checks. */ #ifdef CONFIG_PTP - write_sysreg(read_sysreg(TCR_EL1) | TCR_HPD1 | TCR_A1, tcr_el1); + write_sysreg((read_sysreg(TCR_EL1) | TCR_HPD1) & ~TCR_A1, tcr_el1); isb(); WRITE_ONCE(*__ptr_to_iee(ptep), pte); - write_sysreg(read_sysreg(TCR_EL1) & ~(TCR_HPD1 | TCR_A1), tcr_el1); + dsb(ishst); + write_sysreg((read_sysreg(TCR_EL1) & ~TCR_HPD1) | TCR_A1, tcr_el1); isb(); #else __set_pte(ptep, pte); diff --git a/arch/arm64/kernel/haoc/ptp/Makefile b/arch/arm64/kernel/haoc/ptp/Makefile new file mode 100644 index 0000000000000..c2887cf5c18ce --- /dev/null +++ b/arch/arm64/kernel/haoc/ptp/Makefile @@ -0,0 +1 @@ +obj-y += ptp.o iee-ptp-init.o diff --git a/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c b/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c new file mode 100644 index 0000000000000..cb971ca2e6611 --- /dev/null +++ b/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#define PTP_CHECK(condition) do { \ + if (unlikely(condition)) \ + panic("PTP check failed on %s.", __func__); \ +} while (0) + +#if CONFIG_PGTABLE_LEVELS > 3 +void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) +{ + iee_set_p4d_pre_init(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); +} +#else +void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) +{ + BUILD_BUG(); +} +#endif + +void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) +{ + iee_set_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); +} + +void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, + pmdval_t prot) +{ + iee_set_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); +} + +int iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot) +{ + pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); + + /* Only allow permission changes for now */ + if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), + pud_val(new_pud))) + return 0; + + WARN_ON_ONCE(phys & ~PUD_MASK); + iee_set_pud_pre_init(pudp, new_pud); + return 1; +} + +int iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) +{ + pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); + + /* Only allow permission changes for now */ + if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), + pmd_val(new_pmd))) + return 0; + + WARN_ON_ONCE(phys & ~PMD_MASK); + iee_set_pmd_pre_init(pmdp, new_pmd); + return 1; +} + +static inline pte_t *fixmap_pte(unsigned long addr) +{ + return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; +} + +void __iee_set_fixmap_pre_init(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *ptep; + + PTP_CHECK(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + + ptep = fixmap_pte(addr); + + if (pgprot_val(flags)) { + iee_set_pte_pre_init(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); + } else { + iee_set_pte_pre_init(ptep, __pte(0)); + flush_tlb_kernel_range(addr, addr+PAGE_SIZE); + } +} + +void iee_set_pgtable_pre_init(unsigned long *addr, unsigned long content) +{ + WRITE_ONCE(*addr, content); +} + +void set_iee_address_pre_init(unsigned long addr, bool valid) +{ + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + + p4d_t *p4dp = p4d_offset(pgdp, addr); + + pud_t *pudp = pud_offset(p4dp, addr); + + pmd_t *pmdp = pmd_offset(pudp, addr); + + pte_t *ptep = pte_offset_kernel(pmdp, addr); + pte_t pte = READ_ONCE(*ptep); + + if (valid) + pte = __pte(pte_val(pte) | PTE_VALID); + else + pte = __pte(pte_val(pte) & ~PTE_VALID); + iee_set_pgtable_pre_init((unsigned long *)ptep, (unsigned long)pte.pte); + dsb(ishst); + isb(); +} + +static void iee_init_pte_pre_init(pmd_t *pmdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot) +{ + pte_t *ptep; + + ptep = pte_set_fixmap_offset_pre_init(pmdp, addr); + do { + pte_t old_pte = __ptep_get(ptep); + + iee_set_pgtable_pre_init((unsigned long *)ptep, + (unsigned long)(pfn_pte(__phys_to_pfn(phys), prot).pte)); + + /* + * After the PTE entry has been populated once, we + * only allow updates to the permission attributes. + */ + PTP_CHECK(!pgattr_change_is_safe(pte_val(old_pte), + pte_val(__ptep_get(ptep)))); + + phys += PAGE_SIZE; + } while (ptep++, addr += PAGE_SIZE, addr != end); + + pte_clear_fixmap_pre_init(); +} + +static void iee_alloc_init_cont_pte_pre_init(pmd_t *pmdp, unsigned long addr, + unsigned long end, phys_addr_t phys, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + unsigned long next; + pmd_t pmd = READ_ONCE(*pmdp); + + PTP_CHECK(pmd_sect(pmd)); + if (pmd_none(pmd)) { + pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF; + phys_addr_t pte_phys; + + if (flags & NO_EXEC_MAPPINGS) + pmdval |= PMD_TABLE_PXN; + PTP_CHECK(!pgtable_alloc); + pte_phys = pgtable_alloc(PAGE_SHIFT); + iee_set_pgtable_pre_init((unsigned long *)pmdp, + (unsigned long)(__phys_to_pmd_val(pte_phys) | pmdval)); + pmd = READ_ONCE(*pmdp); + } + PTP_CHECK(pmd_bad(pmd)); + + do { + pgprot_t __prot = prot; + + next = pte_cont_addr_end(addr, end); + + /* use a contiguous mapping if the range is suitably aligned */ + if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && + (flags & NO_CONT_MAPPINGS) == 0) + __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + + iee_init_pte_pre_init(pmdp, addr, next, phys, __prot); + + phys += next - addr; + } while (addr = next, addr != end); +} + +static void iee_init_pmd_pre_init(pud_t *pudp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), int flags) +{ + unsigned long next; + pmd_t *pmdp; + + pmdp = pmd_set_fixmap_offset_pre_init(pudp, addr); + do { + pmd_t old_pmd = READ_ONCE(*pmdp); + + next = pmd_addr_end(addr, end); + + /* try section mapping first */ + if (((addr | next | phys) & ~PMD_MASK) == 0 && + (flags & NO_BLOCK_MAPPINGS) == 0) { + iee_set_pgtable_pre_init((unsigned long *)pmdp, + (unsigned long)(pfn_pmd(__phys_to_pfn(phys), + mk_pmd_sect_prot(prot)).pmd)); + + /* + * After the PMD entry has been populated once, we + * only allow updates to the permission attributes. + */ + PTP_CHECK(!pgattr_change_is_safe(pmd_val(old_pmd), + READ_ONCE(pmd_val(*pmdp)))); + } else { + iee_alloc_init_cont_pte_pre_init(pmdp, addr, next, phys, prot, + pgtable_alloc, flags); + + PTP_CHECK(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + } + phys += next - addr; + } while (pmdp++, addr = next, addr != end); + + pmd_clear_fixmap_pre_init(); +} + +static void iee_alloc_init_cont_pmd_pre_init(pud_t *pudp, unsigned long addr, + unsigned long end, phys_addr_t phys, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), int flags) +{ + unsigned long next; + pud_t pud = READ_ONCE(*pudp); + + /* + * Check for initial section mappings in the pgd/pud. + */ + PTP_CHECK(pud_sect(pud)); + if (pud_none(pud)) { + pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF; + phys_addr_t pmd_phys; + + if (flags & NO_EXEC_MAPPINGS) + pudval |= PUD_TABLE_PXN; + PTP_CHECK(!pgtable_alloc); + pmd_phys = pgtable_alloc(PMD_SHIFT); + iee_set_pgtable_pre_init((unsigned long *)pudp, + (unsigned long)(__phys_to_pud_val(pmd_phys) | pudval)); + pud = READ_ONCE(*pudp); + } + PTP_CHECK(pud_bad(pud)); + + do { + pgprot_t __prot = prot; + + next = pmd_cont_addr_end(addr, end); + + /* use a contiguous mapping if the range is suitably aligned */ + if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && + (flags & NO_CONT_MAPPINGS) == 0) + __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + + iee_init_pmd_pre_init(pudp, addr, next, phys, __prot, pgtable_alloc, flags); + + phys += next - addr; + } while (addr = next, addr != end); +} + +void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + unsigned long next; + pud_t *pudp; + p4d_t *p4dp = p4d_offset(pgdp, addr); + p4d_t p4d = READ_ONCE(*p4dp); + + if (p4d_none(p4d)) { + p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF; + phys_addr_t pud_phys; + + if (flags & NO_EXEC_MAPPINGS) + p4dval |= P4D_TABLE_PXN; + PTP_CHECK(!pgtable_alloc); + pud_phys = pgtable_alloc(PUD_SHIFT); + iee_set_pgtable_pre_init((unsigned long *)p4dp, + (unsigned long)(__phys_to_p4d_val(pud_phys) | p4dval)); + p4d = READ_ONCE(*p4dp); + } + PTP_CHECK(p4d_bad(p4d)); + + pudp = pud_set_fixmap_offset_pre_init(p4dp, addr); + do { + pud_t old_pud = READ_ONCE(*pudp); + + next = pud_addr_end(addr, end); + + iee_alloc_init_cont_pmd_pre_init(pudp, addr, next, phys, prot, + pgtable_alloc, flags); + + PTP_CHECK(pud_val(old_pud) != 0 && + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); + phys += next - addr; + } while (pudp++, addr = next, addr != end); + + pud_clear_fixmap_pre_init(); +} + +static void __create_pgd_mapping_locked_pre_init(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + unsigned long addr, end, next; + pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); + + /* + * If the virtual and physical address don't have the same offset + * within a page, we cannot map the region as the caller expects. + */ + if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) + return; + + phys &= PAGE_MASK; + addr = virt & PAGE_MASK; + end = PAGE_ALIGN(virt + size); + + do { + next = pgd_addr_end(addr, end); + iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, + flags); + phys += next - addr; + } while (pgdp++, addr = next, addr != end); +} + +void __create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + mutex_lock(&fixmap_lock); + __create_pgd_mapping_locked_pre_init(pgdir, phys, virt, size, prot, + pgtable_alloc, flags); + mutex_unlock(&fixmap_lock); +} + +static void __init early_iounmap_after_init(void __iomem *addr, unsigned long size) +{ + unsigned long virt_addr; + unsigned long offset; + unsigned int nrpages; + enum fixed_addresses idx; + int i, slot; + + slot = -1; + for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { + if (prev_map[i] == addr) { + slot = i; + break; + } + } + + if (WARN(slot < 0, "early_iounmap(%p, %08lx) not found slot\n", + addr, size)) + return; + + if (WARN(prev_size[slot] != size, + "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n", + addr, size, slot, prev_size[slot])) + return; + + WARN(early_ioremap_debug, "early_iounmap(%p, %08lx) [%d]\n", + addr, size, slot); + + virt_addr = (unsigned long)addr; + if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN))) + return; + + offset = offset_in_page(virt_addr); + nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT; + + idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; + while (nrpages > 0) { + if (after_paging_init) + __late_clear_fixmap(idx); + else + __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); + --idx; + --nrpages; + } + prev_map[slot] = NULL; +} + +void __init efi_memmap_unmap_after_init(void) +{ + if (!efi_enabled(EFI_MEMMAP)) + return; + + if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) { + unsigned long size; + + size = efi.memmap.desc_size * efi.memmap.nr_map; + early_iounmap_after_init((__force void __iomem *)efi.memmap.map, size); + } else { + memunmap(efi.memmap.map); + } + + efi.memmap.map = NULL; + clear_bit(EFI_MEMMAP, &efi.flags); +} diff --git a/arch/arm64/kernel/haoc/ptp/ptp.c b/arch/arm64/kernel/haoc/ptp/ptp.c new file mode 100644 index 0000000000000..0c2a2e50e33bb --- /dev/null +++ b/arch/arm64/kernel/haoc/ptp/ptp.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +bool check_addr_in_iee_valid(unsigned long addr) +{ + pgd_t *pgdir = swapper_pg_dir; + + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + p4d_t *p4dp = p4d_offset(pgdp, addr); + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + if (!(p4d_val(READ_ONCE(*p4dp)) & PTE_VALID)) + return false; + + pudp = pud_offset(p4dp, addr); + + if (!(pud_val(READ_ONCE(*pudp)) & PTE_VALID)) + return false; + + pmdp = pmd_offset(pudp, addr); + + if (!(pmd_val(READ_ONCE(*pmdp)) & PTE_VALID)) + return false; + + ptep = pte_offset_kernel(pmdp, addr); + + return (pte_val(READ_ONCE(*ptep)) & PTE_VALID); +} + +// Return true if it is only changing prot of a pte. +static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte) +{ + if (((pte_val(*ptep) ^ pte_val(pte)) & PTE_ADDR_MASK) == 0) + return true; + else + return false; +} + +// Return true if the modify does not break DEP. +static inline bool check_pte_dep(char *addr, pte_t pte) +{ + // DEP for kernel code and readonly data + // _text: .text start addr, __init_begin: .rodata end addr + if (addr >= _stext && addr < _etext) { + if ((PTE_WRITE & pte_val(pte)) // DBM == 1 --> writable + || !(PTE_RDONLY & pte_val(pte))) { // DBM == 0 && AP[2] = 0 --> writable + panic("Can't make kernel's text/readonly page as writable!\n" + "addr = 0x%16llx, pte_val = 0x%16llx", + (u64)addr, pte_val(pte)); + } + } + return true; +} + +// Return true if the modify does not break DEP. +static inline bool check_pmd_dep(char *addr, pmd_t pmd) +{ + // DEP for kernel code and readonly data + // _text: .text start addr, __init_begin: .rodata end addr + if (addr >= _stext && addr < _etext) { + if ((PTE_WRITE & pmd_val(pmd)) || // DBM == 1 --> writable + !(PTE_RDONLY & pmd_val(pmd))) { // DBM == 0 && AP[2] = 0 --> writable + panic("Can't make kernel's text/readonly page as writable!\n" + "addr = 0x%16llx, pmd_val = 0x%16llx", + (u64)addr, pmd_val(pmd)); + } + } + return true; +} + +void __iee_code _iee_set_static_pgd(int flag, pgd_t *pgdp, pgd_t pgd) +{ + if (haoc_enabled && (pgd_val(pgd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(pgdp), pgd); +} + +void __iee_code _iee_set_bm_pte(int flag, pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*__ptr_to_iee(ptep), pte); +} + +void __iee_code _iee_set_pte(int flag, pte_t *ptep, pte_t pte) +{ + char *addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); + + if (!(pte_val(pte) & PTE_VALID)) { + WRITE_ONCE(*__ptr_to_iee(ptep), pte); + return; + } + + // Avoid mapping a new VA to IEE PA. + if (haoc_enabled && !is_changing_pte_prot(ptep, pte) && + check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) + panic("You are remmaping IEE page to other VA.\n"); + + // Avoid mapping a writable VA to kernel code PA. + if (!check_pte_dep(addr, pte)) + return; + + WRITE_ONCE(*__ptr_to_iee(ptep), pte); +} + +void __iee_code _iee_set_pmd(int flag, pmd_t *pmdp, pmd_t pmd) +{ + char *addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); + + if (!(pmd_val(pmd) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(pmdp), pmd); + return; + } + + // Check if the pte table is legally allocated. + if (haoc_enabled && (pmd_val(pmd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) + panic("You can't use non-iee-pgtable\n"); + + // Avoid mapping a huge pmd to IEE physical page. + // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) + // panic("Mapping IEE physical page to a huge pmd.\n"); + + if (!check_pmd_dep(addr, pmd)) + return; + + WRITE_ONCE(*__ptr_to_iee(pmdp), pmd); +} + +void __iee_code _iee_set_pud(int flag, pud_t *pudp, pud_t pud) +{ + if (!(pud_val(pud) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(pudp), pud); + return; + } + + if (haoc_enabled && (pud_val(pud) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(pudp), pud); +} + +void __iee_code _iee_set_p4d(int flag, p4d_t *p4dp, p4d_t p4d) +{ + if (!(p4d_val(p4d) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(p4dp), p4d); + return; + } + + if (haoc_enabled && (p4d_val(p4d) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(p4dp), p4d); +} + +void __iee_code _iee_set_swapper_pgd(int flag, pgd_t *pgdp, pgd_t pgd) +{ + if (!(pgd_val(pgd) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(pgdp), pgd); + return; + } + + if (haoc_enabled && (pgd_val(pgd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(pgdp), pgd); +} + +pteval_t __iee_code _iee_set_xchg_relaxed(int flag, pte_t *ptep, pteval_t pteval) +{ + pteval_t ret = xchg_relaxed((pteval_t *)__ptr_to_iee(ptep), pteval); + return ret; +} + +pmdval_t __iee_code _iee_set_pmd_xchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t pmdval) +{ + pmdval_t ret = xchg_relaxed((pmdval_t *)__ptr_to_iee(pmdp), pmdval); + return ret; +} + +pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, + pteval_t old_pteval, pteval_t new_pteval) +{ + pteval_t pteval = cmpxchg_relaxed((pteval_t *)__ptr_to_iee(ptep), + old_pteval, new_pteval); + + return pteval; +} + +pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t old_pmdval, pmdval_t new_pmdval) +{ + pmdval_t pmdval = cmpxchg_relaxed((pmdval_t *)__ptr_to_iee(pmdp), + old_pmdval, new_pmdval); + + return pmdval; +} + +void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, + int order, int use_block_pmd) +{ + int i; + + lm_ptep = __ptr_to_iee(lm_ptep); + iee_ptep = __ptr_to_iee(iee_ptep); + if (use_block_pmd) { + #ifdef CONFIG_IEE_ALLOW_SPLIT_LM + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + #else + if (order == IEE_DATA_ORDER) { + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + } else { + /* Give up RO protection on linear mapping to avoid page table spliting. */ + // pr_err("IEE: give up RO protection to avoid page table split."); + } + #endif + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*iee_ptep); + + pte = __pte(pte_val(pte) | PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + iee_ptep++; + } + } else { + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*lm_ptep); + + pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); + WRITE_ONCE(*lm_ptep, pte); + pte = READ_ONCE(*iee_ptep); + pte = __pte(pte_val(pte) | PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + lm_ptep++; + iee_ptep++; + } + } +} + +void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, + int order, int use_block_pmd) +{ + int i; + + lm_ptep = __ptr_to_iee(lm_ptep); + iee_ptep = __ptr_to_iee(iee_ptep); + if (use_block_pmd) { + #ifdef CONFIG_IEE_ALLOW_SPLIT_LM + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + pmd = __pmd(pmd_val(pmd) | PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + #else + if (order == IEE_DATA_ORDER) { + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + pmd = __pmd(pmd_val(pmd) | PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + } else { + /* Give up RO protection on linear mapping to avoid page table spliting. */ + // pr_err("IEE: give up RO protection to avoid page table split."); + } + #endif + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*iee_ptep); + + pte = __pte(pte_val(pte) & ~PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + iee_ptep++; + } + } else { + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*lm_ptep); + + pte = __pte(pte_val(pte) | PTE_DBM); + WRITE_ONCE(*lm_ptep, pte); + pte = READ_ONCE(*iee_ptep); + pte = __pte(pte_val(pte) & ~PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + lm_ptep++; + iee_ptep++; + } + } +} diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index c6adf753e6d00..68aae0e4caa54 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -371,6 +371,13 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) unflatten_device_tree(); bootmem_init(); +#ifdef CONFIG_PTP + /* Prepare page pool for page tables. */ + if (haoc_enabled) { + iee_cache_init(&pg_cache, 0, CONFIG_PGTABLE_LEVELS, IEE_PGTABLE, + CONFIG_PTP_RESERVE_ORDER); + } +#endif kasan_init(); diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 6b2f766947d98..40b75a1c486da 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -148,6 +148,17 @@ jiffies = jiffies_64; #define IEE_TEXT #endif +#ifdef CONFIG_PTP +#define PTP_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_ptp_data_start = .; \ + *(.iee.ptp) \ + . = ALIGN(PAGE_SIZE); \ + __iee_ptp_data_end = .; +#else +#define PTP_DATA +#endif + #ifdef CONFIG_CREDP #define CRED_DATA \ . = ALIGN(PAGE_SIZE); \ @@ -165,6 +176,7 @@ jiffies = jiffies_64; .iee.data : { \ . = ALIGN(PAGE_SIZE); \ iee_init_data_begin = .; \ + PTP_DATA \ CRED_DATA \ . = ALIGN(PAGE_SIZE); \ iee_init_data_end = .; \ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2edb093fe8b66..5ea9f63d559eb 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -238,7 +238,11 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, pteval ^= PTE_RDONLY; pteval |= pte_val(entry); pteval ^= PTE_RDONLY; + #ifdef CONFIG_PTP + pteval = iee_set_cmpxchg_relaxed(ptep, old_pteval, pteval); + #else pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); + #endif } while (pteval != old_pteval); /* Invalidate a stale read-only entry */ diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index fbf904db2442f..4374f608973cc 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -16,6 +16,11 @@ #include #include +#ifdef CONFIG_PTP +#include +#include +#endif + /* ensure that the fixmap region does not grow down into the PCI I/O region */ static_assert(FIXADDR_TOT_START > PCI_IO_END); @@ -31,15 +36,35 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) +#ifdef CONFIG_PTP +pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __section(".iee.ptp") __aligned(PAGE_SIZE); +pmd_t bm_pmd[PTRS_PER_PMD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +pud_t bm_pud[PTRS_PER_PUD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +#else static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +#endif static inline pte_t *fixmap_pte(unsigned long addr) { return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; } +#ifdef CONFIG_PTP +static inline void set_bm_fixmap_pte(pte_t *ptep, pte_t pte) +{ + if (haoc_enabled && iee_init_done) { + iee_set_bm_pte(ptep, pte); + return; + } + + WRITE_ONCE(*ptep, pte); + dsb(ishst); + isb(); +} +#endif + static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) { pmd_t pmd = READ_ONCE(*pmdp); @@ -47,8 +72,12 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) if (pmd_none(pmd)) { ptep = bm_pte[BM_PTE_TABLE_IDX(addr)]; +#ifdef CONFIG_PTP + __iee_pmd_populate_pre_init(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE); +#else __pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE | PMD_TABLE_AF); +#endif } } @@ -59,9 +88,14 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, pud_t pud = READ_ONCE(*pudp); pmd_t *pmdp; - if (pud_none(pud)) + if (pud_none(pud)) { +#ifdef CONFIG_PTP + __iee_pud_populate_pre_init(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); +#else __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE | PUD_TABLE_AF); +#endif + } pmdp = pmd_offset_kimg(pudp, addr); do { @@ -87,9 +121,14 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); } - if (p4d_none(p4d)) + if (p4d_none(p4d)) { +#ifdef CONFIG_PTP + __iee_p4d_populate_pre_init(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); +#else __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE | P4D_TABLE_AF); +#endif + } pudp = pud_offset_kimg(p4dp, addr); early_fixmap_init_pmd(pudp, addr, end); @@ -107,7 +146,11 @@ void __init early_fixmap_init(void) unsigned long end = FIXADDR_TOP; pgd_t *pgdp = pgd_offset_k(addr); +#ifdef CONFIG_PTP + p4d_t *p4dp = p4d_offset_kimg(pgdp, addr); +#else p4d_t *p4dp = p4d_offset(pgdp, addr); +#endif early_fixmap_init_pud(p4dp, addr, end); } @@ -127,9 +170,17 @@ void __set_fixmap(enum fixed_addresses idx, ptep = fixmap_pte(addr); if (pgprot_val(flags)) { +#ifdef CONFIG_PTP + set_bm_fixmap_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); +#else __set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); +#endif } else { +#ifdef CONFIG_PTP + set_bm_fixmap_pte(ptep, __pte(0)); +#else __pte_clear(&init_mm, addr, ptep); +#endif flush_tlb_kernel_range(addr, addr+PAGE_SIZE); } } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index a5284c19c285e..511de939cd970 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -50,6 +50,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif /* * We need to be able to catch inadvertent references to memstart_addr @@ -549,6 +552,9 @@ void __init mem_init(void) void free_initmem(void) { + #ifdef CONFIG_IEE + if (!haoc_enabled) + #endif free_reserved_area(lm_alias(__init_begin), lm_alias(__init_end), POISON_FREE_INITMEM, "unused kernel"); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d3ca3fc09b265..e56da8dbcec5a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -42,12 +42,16 @@ #include #ifdef CONFIG_IEE #include +#include #include #include #ifdef CONFIG_IEE_SIP #include #endif #endif +#ifdef CONFIG_PTP +#include +#endif #define NO_BLOCK_MAPPINGS BIT(0) #define NO_CONT_MAPPINGS BIT(1) @@ -82,7 +86,9 @@ unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_ EXPORT_SYMBOL(empty_zero_page); static DEFINE_SPINLOCK(swapper_pgdir_lock); +#ifndef CONFIG_IEE static DEFINE_MUTEX(fixmap_lock); +#endif void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) { @@ -115,6 +121,11 @@ static phys_addr_t __init early_pgtable_alloc(int shift) { phys_addr_t phys; +#ifdef CONFIG_PTP + if (haoc_enabled) + return early_iee_pgtable_alloc(shift); +#endif + phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, MEMBLOCK_ALLOC_NOLEAKTRACE); if (!phys) @@ -421,7 +432,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, { mutex_lock(&fixmap_lock); #ifdef CONFIG_IEE - if (haoc_enabled) + if (haoc_enabled && !iee_init_done) __iee_create_pgd_mapping_locked(pgdir, phys, virt, size, prot, pgtable_alloc, flags); else @@ -443,10 +454,15 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, static phys_addr_t __pgd_pgtable_alloc(int shift) { - /* Page is zeroed by init_clear_pgtable() so don't duplicate effort. */ - void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL & ~__GFP_ZERO); - + #ifdef CONFIG_PTP + void *ptr = iee_cache_alloc(&pg_cache, GFP_PGTABLE_KERNEL); + #else + void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); + #endif BUG_ON(!ptr); + + /* Ensure the zeroed page is visible to the page table walker */ + dsb(ishst); return __pa(ptr); } @@ -484,8 +500,13 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } + #ifdef CONFIG_PTP + __create_pgd_mapping_pre_init(init_mm.pgd, phys, virt, size, prot, NULL, + NO_CONT_MAPPINGS); + #else __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, NO_CONT_MAPPINGS); + #endif } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -522,6 +543,14 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, phys_addr_t end, pgprot_t prot, int flags) { +#ifdef CONFIG_PTP + if (haoc_enabled) { + __create_pgd_mapping_pre_init(pgdp, start, __phys_to_virt(start), + end - start, prot, + early_pgtable_alloc, flags); + return; + } +#endif __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, prot, early_pgtable_alloc, flags); } @@ -534,6 +563,12 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), (unsigned long)__init_begin - (unsigned long)_stext, PAGE_KERNEL_RO); + #ifdef CONFIG_PTP + if (haoc_enabled) + update_mapping_prot(__pa_symbol(init_pg_dir), (unsigned long)lm_alias(init_pg_dir), + (unsigned long)init_pg_end - (unsigned long)init_pg_dir, + PAGE_KERNEL_RO); + #endif } #ifdef CONFIG_KFENCE @@ -595,6 +630,10 @@ static void __init map_mem(pgd_t *pgdp) static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_stext); phys_addr_t kernel_end = __pa_symbol(__init_begin); +#ifdef CONFIG_PTP + phys_addr_t init_pgtable_start = __pa_symbol(init_pg_dir); + phys_addr_t init_pgtable_end = __pa_symbol(init_pg_end); +#endif phys_addr_t start, end; phys_addr_t early_kfence_pool; int flags = NO_EXEC_MAPPINGS; @@ -621,6 +660,9 @@ static void __init map_mem(pgd_t *pgdp) * the following for-loop */ memblock_mark_nomap(kernel_start, kernel_end - kernel_start); + #ifdef CONFIG_PTP + memblock_mark_nomap(init_pgtable_start, init_pgtable_end - init_pgtable_start); + #endif /* map all the memory banks */ for_each_mem_range(i, &start, &end) { @@ -648,6 +690,11 @@ static void __init map_mem(pgd_t *pgdp) __map_memblock(pgdp, kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); + #ifdef CONFIG_PTP + __map_memblock(pgdp, init_pgtable_start, init_pgtable_end, + PAGE_KERNEL, NO_CONT_MAPPINGS); + memblock_clear_nomap(init_pgtable_start, init_pgtable_end - init_pgtable_start); + #endif arm64_kfence_map_pool(early_kfence_pool, pgdp); } @@ -662,6 +709,13 @@ void mark_rodata_ro(void) section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, section_size, PAGE_KERNEL_RO); + #ifdef CONFIG_PTP + if (haoc_enabled) { + section_size = (unsigned long)init_pg_end - (unsigned long)init_pg_dir; + update_mapping_prot(__pa_symbol(init_pg_dir), (unsigned long)init_pg_dir, + section_size, PAGE_KERNEL_RO); + } + #endif debug_checkwx(); } @@ -708,7 +762,14 @@ static int __init map_entry_trampoline(void) pgprot_val(prot) &= ~PTE_NG; /* Map only the text into the trampoline page table */ + #ifdef CONFIG_PTP + if (haoc_enabled) + iee_memset(__va(__pa_symbol(tramp_pg_dir)), 0, PGD_SIZE); + else + memset(tramp_pg_dir, 0, PGD_SIZE); + #else memset(tramp_pg_dir, 0, PGD_SIZE); + #endif __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, entry_tramp_text_size(), prot, __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); @@ -807,7 +868,8 @@ static void __init map_kernel(pgd_t *pgdp) // 只有当IEE数据段非空时才创建分离的映射 map_kernel_segment(pgdp, _data, iee_init_data_end, PAGE_KERNEL, &vmlinux_iee_init_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); - map_kernel_segment(pgdp, iee_init_data_end, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); + map_kernel_segment(pgdp, iee_init_data_end, _end, PAGE_KERNEL, + &vmlinux_data, NO_CONT_MAPPINGS, 0); } else { pr_info("IEE: No IEE init data, using standard data mapping\n"); map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); @@ -879,8 +941,11 @@ void __init paging_init(void) init_early_iee_data(); #endif - memblock_phys_free(__pa_symbol(init_pg_dir), - __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); + #ifdef CONFIG_IEE + if (!haoc_enabled) + #endif + memblock_phys_free(__pa_symbol(init_pg_dir), + __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); memblock_allow_resize(); @@ -905,7 +970,11 @@ static void free_hotplug_page_range(struct page *page, size_t size, static void free_hotplug_pgtable_page(struct page *page) { +#ifdef CONFIG_PTP + pagetable_free(virt_to_ptdesc(page_to_virt(page))); +#else free_hotplug_page_range(page, PAGE_SIZE, NULL); +#endif } static bool pgtable_range_aligned(unsigned long start, unsigned long end, diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index 4a64089e5771c..356eae166d31d 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -15,29 +15,55 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + static struct kmem_cache *pgd_cache __ro_after_init; +static bool pgdir_is_page_size(void) +{ + if (PGD_SIZE == PAGE_SIZE) + return true; + if (CONFIG_PGTABLE_LEVELS == 4) + return !pgtable_l4_enabled(); + if (CONFIG_PGTABLE_LEVELS == 5) + return !pgtable_l5_enabled(); + return false; +} + pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; - if (PGD_SIZE == PAGE_SIZE) + if (pgdir_is_page_size()) { +#ifdef CONFIG_PTP + return (pgd_t *)iee_cache_alloc(&pg_cache, gfp); +#else return (pgd_t *)__get_free_page(gfp); - else - return kmem_cache_alloc(pgd_cache, gfp); +#endif + } + + return kmem_cache_alloc(pgd_cache, gfp); } void pgd_free(struct mm_struct *mm, pgd_t *pgd) { - if (PGD_SIZE == PAGE_SIZE) + if (pgdir_is_page_size()) { +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pgd); +#else free_page((unsigned long)pgd); - else - kmem_cache_free(pgd_cache, pgd); +#endif + return; + } + + kmem_cache_free(pgd_cache, pgd); } void __init pgtable_cache_init(void) { - if (PGD_SIZE == PAGE_SIZE) + if (pgdir_is_page_size()) return; #ifdef CONFIG_ARM64_PA_BITS_52 diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c index 5139a28130c08..c7a356adb51ba 100644 --- a/arch/arm64/mm/trans_pgd.c +++ b/arch/arm64/mm/trans_pgd.c @@ -252,7 +252,11 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, level_mask = GENMASK_ULL(level_msb, level_lsb); index = (dst_addr & level_mask) >> level_lsb; + #ifdef CONFIG_PTP + set_pte((pte_t *)(levels[this_level] + index), __pte(prev_level_entry)); + #else *(levels[this_level] + index) = prev_level_entry; + #endif pfn = virt_to_pfn(levels[this_level]); prev_level_entry = pte_val(pfn_pte(pfn, diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c index 83092d93f36a6..fb12e7d0660ad 100644 --- a/drivers/firmware/efi/arm-runtime.c +++ b/drivers/firmware/efi/arm-runtime.c @@ -94,7 +94,11 @@ static int __init arm_enable_runtime_services(void) return 0; } + #ifdef CONFIG_PTP + efi_memmap_unmap_after_init(); + #else efi_memmap_unmap(); + #endif mapsize = efi.memmap.desc_size * efi.memmap.nr_map; diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c index a5fbb6ed38aed..93f80a19c1947 100644 --- a/drivers/tty/serial/earlycon.c +++ b/drivers/tty/serial/earlycon.c @@ -40,7 +40,11 @@ static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size) { void __iomem *base; #ifdef CONFIG_FIX_EARLYCON_MEM + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK, FIXMAP_PAGE_IO); + #else set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK); + #endif base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE); base += paddr & ~PAGE_MASK; #else diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c index 45b42d8f64532..ac3b1d2689696 100644 --- a/drivers/usb/early/ehci-dbgp.c +++ b/drivers/usb/early/ehci-dbgp.c @@ -879,7 +879,11 @@ int __init early_dbgp_init(char *s) * FIXME I don't have the bar size so just guess PAGE_SIZE is more * than enough. 1K is the biggest I have seen. */ + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(FIX_DBGP_BASE, bar_val & PAGE_MASK, FIXMAP_PAGE_NOCACHE); + #else set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK); + #endif ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE); ehci_bar += bar_val & ~PAGE_MASK; dbgp_printk("ehci_bar: %p\n", ehci_bar); diff --git a/include/linux/efi.h b/include/linux/efi.h index b5c69570c9be6..1a65f01c8207d 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -28,6 +28,10 @@ #include +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +#include +#endif + struct screen_info; #define EFI_SUCCESS 0 diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c index ce06b2884789a..3ee71ba8e7823 100644 --- a/mm/early_ioremap.c +++ b/mm/early_ioremap.c @@ -20,7 +20,11 @@ #include "internal.h" #ifdef CONFIG_MMU +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +int early_ioremap_debug __initdata; +#else static int early_ioremap_debug __initdata; +#endif static int __init early_ioremap_debug_setup(char *str) { @@ -30,7 +34,11 @@ static int __init early_ioremap_debug_setup(char *str) } early_param("early_ioremap_debug", early_ioremap_debug_setup); +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +int after_paging_init __initdata; +#else static int after_paging_init __initdata; +#endif pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr, unsigned long size, @@ -64,9 +72,15 @@ static inline void __init __late_clear_fixmap(enum fixed_addresses idx) } #endif +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; +unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; +unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; +#else static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; +#endif void __init early_ioremap_setup(void) { @@ -147,7 +161,11 @@ __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot) if (after_paging_init) __late_set_fixmap(idx, phys_addr, prot); else + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(idx, phys_addr, prot); + #else __early_set_fixmap(idx, phys_addr, prot); + #endif phys_addr += PAGE_SIZE; --idx; --nrpages; @@ -199,7 +217,11 @@ void __init early_iounmap(void __iomem *addr, unsigned long size) if (after_paging_init) __late_clear_fixmap(idx); else + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR); + #else __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); + #endif --idx; --nrpages; } diff --git a/mm/haoc/ptp-pg_cache.c b/mm/haoc/ptp-pg_cache.c index 6351dbe60904d..cde38ee6b0b8f 100644 --- a/mm/haoc/ptp-pg_cache.c +++ b/mm/haoc/ptp-pg_cache.c @@ -17,6 +17,7 @@ struct iee_cache pg_cache; +int expend_num; #ifdef CONFIG_X86_64 static LIST_HEAD(iee_cache_pending_protect_list); static DEFINE_SPINLOCK(iee_cache_pending_protect_lock); @@ -73,6 +74,9 @@ static void __ptp_set_iee_pages(unsigned long start_addr, unsigned long end_addr { unsigned long addr; + if (!haoc_enabled) + return; + if (start_addr != ALIGN(start_addr, PMD_SIZE)) panic("IEE: pool (HAOC_BITMAP_TYPE %u) not PMD-aligned.", (unsigned int)cache->name); @@ -150,7 +154,7 @@ void __init iee_cache_init(struct iee_cache *cache, unsigned long object_order, #ifdef CONFIG_ARM64 /* Expand the cache pool with PMD_SIZE for each time. */ -static int __ref iee_cache_expand(struct iee_cache *cache, gfp_t gfp) +static __maybe_unused int __ref iee_cache_expand(struct iee_cache *cache, gfp_t gfp) { unsigned long addr; unsigned long addr_next; @@ -201,6 +205,9 @@ static int __ref iee_cache_expand(struct iee_cache *cache, gfp_t gfp) if (unlikely(!__update_freelist(cache, freelist, (void *)start_addr, tid))) goto redo; + expend_num++; + pr_alert("gwm %s num %d , start_addr: 0x%lx, end_addr: 0x%lx, start_addr-end_addr: 0x%lx\n", + __func__, expend_num, start_addr, end_addr, start_addr - end_addr); return 1; } #endif @@ -223,8 +230,8 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) if (unlikely(!object)) { // slow path alloc #ifdef CONFIG_ARM64 - if (iee_cache_expand(cache, gfp) || READ_ONCE(cache->freelist)) - goto redo; + // if (iee_cache_expand(cache, gfp) || READ_ONCE(cache->freelist)) + // goto redo; /* If the expandsion failed, alloc a singel object without RO protection to * avoid block spliting. @@ -233,13 +240,13 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) if (!object) return NULL; set_iee_address_valid((unsigned long)object, cache->object_order); - iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, cache->name); + iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, IEE_PGTABLE); #ifdef DEBUG WARN_ONCE(1, "IEE: Failed on HAOC_BITMAP_TYPE %u expansion.", (unsigned int)cache->name); atomic_add(1 << cache->object_order, &cache->fail_count); #endif - #else +#else if (!preemptible()) return iee_cache_alloc_pending_protect(cache, gfp); object = (void *)__get_free_pages(gfp, cache->object_order); @@ -402,12 +409,25 @@ void iee_cache_free(struct iee_cache *cache, void *object) } #endif +#ifdef CONFIG_ARM64 + if (unlikely((unsigned long)object < cache->reserve_start_addr + || (unsigned long)object >= cache->reserve_end_addr)) { + + set_iee_address_invalid((unsigned long)object, cache->object_order); + iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, IEE_NORMAL); + free_pages((unsigned long)object, cache->object_order); + return; + } +#endif // fast path free redo: tid = READ_ONCE(cache->tid); barrier(); freelist = READ_ONCE(cache->freelist); __iee_set_freepointer(object, freelist); +#ifdef CONFIG_ARM64 + dsb(sy); +#endif if (unlikely(!__update_freelist(cache, freelist, object, tid))) goto redo; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index d9e69d0439bd5..26282c45dade4 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2247,7 +2247,12 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); pgtable = pgtable_trans_huge_withdraw(mm, pmd); + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + WRITE_ONCE(*(&_pmd), __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) + | (PMD_TYPE_TABLE | PMD_TABLE_PXN))); + #else pmd_populate(mm, &_pmd, pgtable); + #endif pte = pte_offset_map(&_pmd, haddr); VM_BUG_ON(!pte); @@ -2421,7 +2426,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, * This's critical for some architectures (Power). */ pgtable = pgtable_trans_huge_withdraw(mm, pmd); + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + WRITE_ONCE(*(&_pmd), __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) + | (PMD_TYPE_TABLE | PMD_TABLE_PXN))); + #else pmd_populate(mm, &_pmd, pgtable); + #endif pte = pte_offset_map(&_pmd, haddr); VM_BUG_ON(!pte); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index c3353cd442a5d..9ff858b7768d0 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -31,6 +31,10 @@ #include +#ifdef CONFIG_PTP +extern void *__ptp_vmemmap_alloc_block(unsigned long size, int node); +#endif + /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. @@ -167,6 +171,7 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, get_page(reuse); p = page_to_virt(reuse); } + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } @@ -175,11 +180,18 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) { + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + void *p = __ptp_vmemmap_alloc_block(size, node); + + if (!p) + return NULL; + #else void *p = vmemmap_alloc_block(size, node); if (!p) return NULL; memset(p, 0, size); + #endif return p; } From a7d69f16d9ddda86bc9c5cba46d1a1bfacbc45a9 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 13 Aug 2026 16:34:01 +0800 Subject: [PATCH 07/26] HAOC3.0/arm64/PTP: Add protected __set_pte_nosync helper Signed-off-by: Liu Zhehui --- arch/arm64/include/asm/haoc/iee-pgtable.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/include/asm/haoc/iee-pgtable.h b/arch/arm64/include/asm/haoc/iee-pgtable.h index 8788ebd23388a..90fd6d2edf0f5 100644 --- a/arch/arm64/include/asm/haoc/iee-pgtable.h +++ b/arch/arm64/include/asm/haoc/iee-pgtable.h @@ -467,6 +467,14 @@ static inline void set_pte_nosync(pte_t *ptep, pte_t pte) WRITE_ONCE(*ptep, pte); } +static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) +{ + if (iee_pgtable_ready()) + iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); + else + WRITE_ONCE(*ptep, pte); +} + static inline void __set_pte(pte_t *ptep, pte_t pte) { if (iee_pgtable_ready()) From d059fc267ff8a81fe71b8bc2aadb61db45b0bf18 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 9 Jul 2026 17:56:51 +0800 Subject: [PATCH 08/26] HAOC3.0/x86_64/KEYP: Add support for x86 Key Protection Support key protection for x86. Co-authored-by: hubing Signed-off-by: Liu Zhehui --- arch/x86/Kconfig | 9 + arch/x86/configs/deepin_x86_desktop_defconfig | 1 + arch/x86/include/asm/haoc/haoc-def.h | 40 +++ arch/x86/include/asm/haoc/haoc.h | 35 +++ arch/x86/include/asm/haoc/iee-key.h | 151 ++++++++++ arch/x86/kernel/haoc/Makefile | 1 + arch/x86/kernel/haoc/haoc.c | 29 ++ arch/x86/kernel/haoc/keyp/Makefile | 1 + arch/x86/kernel/haoc/keyp/keyp.c | 236 +++++++++++++++ block/sed-opal.c | 8 + certs/blacklist.c | 7 + certs/system_keyring.c | 18 ++ crypto/af_alg.c | 16 ++ crypto/asymmetric_keys/asymmetric_type.c | 21 ++ crypto/asymmetric_keys/public_key.c | 20 ++ crypto/asymmetric_keys/signature.c | 10 + drivers/md/dm-crypt.c | 20 ++ drivers/md/dm-verity-verify-sig.c | 8 + drivers/nvdimm/security.c | 24 ++ fs/crypto/keyring.c | 28 ++ fs/crypto/keysetup_v1.c | 12 + fs/ecryptfs/ecryptfs_kernel.h | 4 + fs/ecryptfs/keystore.c | 52 ++++ fs/ecryptfs/main.c | 4 + fs/nfs/nfs4idmap.c | 21 +- fs/smb/client/cifs_spnego.c | 26 +- fs/smb/client/cifsacl.c | 51 +++- fs/smb/client/connect.c | 8 + fs/smb/client/sess.c | 4 + fs/smb/client/smb2pdu.c | 4 + fs/ubifs/auth.c | 8 + fs/verity/signature.c | 5 + include/keys/asymmetric-subtype.h | 4 + include/keys/asymmetric-type.h | 8 + include/keys/request_key_auth-type.h | 4 + include/linux/key.h | 76 +++++ lib/digsig.c | 8 + mm/slab.h | 22 +- mm/slab_common.c | 12 + mm/slub.c | 50 +++- net/ceph/ceph_common.c | 4 + net/ceph/crypto.c | 4 + net/dns_resolver/dns_key.c | 22 +- net/rxrpc/af_rxrpc.c | 4 + net/rxrpc/conn_event.c | 5 + net/rxrpc/key.c | 19 ++ net/rxrpc/rxkad.c | 42 +++ net/rxrpc/security.c | 10 + net/rxrpc/sendmsg.c | 4 + net/rxrpc/server_key.c | 8 + security/integrity/evm/evm_crypto.c | 9 + security/keys/big_key.c | 20 ++ security/keys/dh.c | 8 + security/keys/encrypted-keys/encrypted.c | 29 ++ .../keys/encrypted-keys/masterkey_trusted.c | 5 + security/keys/gc.c | 54 ++++ security/keys/internal.h | 8 + security/keys/key.c | 250 ++++++++++++++++ security/keys/keyctl.c | 60 ++++ security/keys/keyring.c | 270 ++++++++++++++++++ security/keys/proc.c | 28 ++ security/keys/process_keys.c | 39 +++ security/keys/request_key.c | 15 + security/keys/request_key_auth.c | 38 ++- security/keys/trusted-keys/trusted_core.c | 8 + security/keys/user_defined.c | 11 + security/selinux/hooks.c | 11 + security/smack/smack_lsm.c | 11 + 68 files changed, 2040 insertions(+), 22 deletions(-) create mode 100644 arch/x86/include/asm/haoc/iee-key.h create mode 100644 arch/x86/kernel/haoc/keyp/Makefile create mode 100644 arch/x86/kernel/haoc/keyp/keyp.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e0de378277fae..c745d0438f996 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1596,6 +1596,15 @@ config CREDP by IEE either. If unsure, say N. +config KEYP + bool "Struct Key protection(KEYP)" + depends on IEE + help + This option enables protection for the kernel's struct key. All modifications + of keys must be made and verified by IEE APIs to defend memory corruptions. + + If unsure, say N. + # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 7b573e304318c..3cb8f49283526 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -92,6 +92,7 @@ CONFIG_IEE=y CONFIG_IEE_PTRP=y CONFIG_IEE_SIP=y CONFIG_CREDP=y +CONFIG_KEYP=y CONFIG_PTP=y CONFIG_PTP_RESERVE_ORDER=12 CONFIG_NUMA=y diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index d317af44ef251..aab082ba5fe20 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -59,6 +59,35 @@ enum { IEE_OP_SET_CRED_RCU, IEE_OP_SET_CRED_UCOUNTS, #endif +#ifdef CONFIG_KEYP + IEE_OP_SET_KEY_UNION, + IEE_OP_SET_KEY_STRUCT, + IEE_OP_SET_KEY_PAYLOAD, + IEE_OP_SET_KEY_USAGE, + IEE_OP_SET_KEY_SERIAL, + IEE_OP_SET_KEY_WATCHERS, + IEE_OP_SET_KEY_USERS, + IEE_OP_SET_KEY_SECURITY, + IEE_OP_SET_KEY_EXPIRY, + IEE_OP_SET_KEY_REVOKED_AT, + IEE_OP_SET_KEY_LAST_USED_AT, + IEE_OP_SET_KEY_UID, + IEE_OP_SET_KEY_GID, + IEE_OP_SET_KEY_PERM, + IEE_OP_SET_KEY_QUOTALEN, + IEE_OP_SET_KEY_DATALEN, + IEE_OP_SET_KEY_STATE, + IEE_OP_SET_KEY_MAGIC, + IEE_OP_SET_KEY_FLAGS, + IEE_OP_SET_KEY_INDEX_KEY, + IEE_OP_SET_KEY_HASH, + IEE_OP_SET_KEY_LEN_DESC, + IEE_OP_SET_KEY_TYPE, + IEE_OP_SET_KEY_TAG, + IEE_OP_SET_KEY_DESCRIPTION, + IEE_OP_SET_KEY_RESTRICT_LINK, + IEE_OP_SET_KEY_FLAG_BIT, +#endif #ifdef CONFIG_PTP #ifdef CONFIG_IEE_PTRP IEE_OP_UNSET_TOKEN, @@ -90,4 +119,15 @@ enum { #define AT_INC_NOT_ZERO 2 #define AT_SUB_AND_TEST 3 #endif + +#ifdef CONFIG_KEYP +#define REFCOUNT_INC 1 +#define REFCOUNT_SET 2 +#define REFCOUNT_DEC_AND_TEST 3 +#define REFCOUNT_INC_NOT_ZERO 4 + +#define SET_BIT_OP 1 +#define TEST_AND_CLEAR_BIT 2 +#define TEST_AND_SET_BIT 3 +#endif #endif diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index f938382c8ad59..4fd1b32cbda4d 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -100,4 +100,39 @@ void _iee_set_cred_security(unsigned long __unused, struct cred *cred,void *secu void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu); void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); #endif + +#ifdef CONFIG_KEYP +#include +struct watch_list; + +void _iee_set_key_union(unsigned long __unused, struct key *key, struct key_union *key_union); +void _iee_set_key_struct(unsigned long __unused, struct key *key, struct key_struct *key_struct); +void _iee_set_key_payload(unsigned long __unused, struct key *key, union key_payload *key_payload); +unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, int n, int flag); +void _iee_set_key_serial(unsigned long __unused, struct key *key, key_serial_t serial); +void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers); +void _iee_set_key_user(unsigned long __unused, struct key *key, struct key_user *user); +void _iee_set_key_security(unsigned long __unused, struct key *key, void *security); +void _iee_set_key_expiry(unsigned long __unused, struct key *key, time64_t expiry); +void _iee_set_key_revoked_at(unsigned long __unused, struct key *key, time64_t revoked_at); +void _iee_set_key_last_used_at(unsigned long __unused, struct key *key, time64_t last_used_at); +void _iee_set_key_uid(unsigned long __unused, struct key *key, kuid_t uid); +void _iee_set_key_gid(unsigned long __unused, struct key *key, kgid_t gid); +void _iee_set_key_perm(unsigned long __unused, struct key *key, key_perm_t perm); +void _iee_set_key_quotalen(unsigned long __unused, struct key *key, unsigned short quotalen); +void _iee_set_key_datalen(unsigned long __unused, struct key *key, unsigned short datalen); +void _iee_set_key_state(unsigned long __unused, struct key *key, short state); +void _iee_set_key_magic(unsigned long __unused, struct key *key, unsigned int magic); +void _iee_set_key_flags(unsigned long __unused, struct key *key, unsigned long flags); +void _iee_set_key_index_key(unsigned long __unused, struct key *key, + struct keyring_index_key *index_key); +void _iee_set_key_hash(unsigned long __unused, struct key *key, unsigned long hash); +void _iee_set_key_len_desc(unsigned long __unused, struct key *key, unsigned long len_desc); +void _iee_set_key_type(unsigned long __unused, struct key *key, struct key_type *type); +void _iee_set_key_domain_tag(unsigned long __unused, struct key *key, struct key_tag *domain_tag); +void _iee_set_key_description(unsigned long __unused, struct key *key, char *description); +void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, + struct key_restriction *restrict_link); +unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); +#endif #endif diff --git a/arch/x86/include/asm/haoc/iee-key.h b/arch/x86/include/asm/haoc/iee-key.h new file mode 100644 index 0000000000000..86d499d9aff64 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-key.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_KEY_H +#define _LINUX_IEE_KEY_H + +#include +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_set_key_union(struct key *key, struct key_union *key_union) +{ + iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); +} + +static void __maybe_unused iee_set_key_struct(struct key *key, struct key_struct *key_struct) +{ + iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); +} + +static void __maybe_unused iee_set_key_payload(struct key *key, union key_payload *key_payload) +{ + iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); +} + +extern bool iee_set_key_usage(struct key *key, int n, int flag); + +static void __maybe_unused iee_set_key_serial(struct key *key, key_serial_t serial) +{ + iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); +} + +#ifdef CONFIG_KEY_NOTIFICATIONS +static void __maybe_unused iee_set_key_watchers(struct key *key, struct watch_list *watchers) +{ + iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); +} +#endif + +static void __maybe_unused iee_set_key_user(struct key *key, struct key_user *user) +{ + iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); +} + +static void __maybe_unused iee_set_key_security(struct key *key, void *security) +{ + iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); +} + +static void __maybe_unused iee_set_key_expiry(struct key *key, time64_t expiry) +{ + iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); +} + +static void __maybe_unused iee_set_key_revoked_at(struct key *key, time64_t revoked_at) +{ + iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); +} + +static void __maybe_unused iee_set_key_last_used_at(struct key *key, time64_t last_used_at) +{ + iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); +} + +static void __maybe_unused iee_set_key_uid(struct key *key, kuid_t uid) +{ + iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); +} + +static void __maybe_unused iee_set_key_gid(struct key *key, kgid_t gid) +{ + iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); +} + +static void __maybe_unused iee_set_key_perm(struct key *key, key_perm_t perm) +{ + iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); +} + +static void __maybe_unused iee_set_key_quotalen(struct key *key, unsigned short quotalen) +{ + iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); +} + +static void __maybe_unused iee_set_key_datalen(struct key *key, unsigned short datalen) +{ + iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); +} + +static void __maybe_unused iee_set_key_state(struct key *key, short state) +{ + iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); +} + +#ifdef KEY_DEBUGGING +static void __maybe_unused iee_set_key_magic(struct key *key, unsigned int magic) +{ + iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); +} +#endif + +static void __maybe_unused iee_set_key_flags(struct key *key, unsigned long flags) +{ + iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); +} + +static void __maybe_unused iee_set_key_index_key(struct key *key, + struct keyring_index_key *index_key) +{ + iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); +} + +static void __maybe_unused iee_set_key_hash(struct key *key, unsigned long hash) +{ + iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); +} + +static void __maybe_unused iee_set_key_len_desc(struct key *key, unsigned long len_desc) +{ + iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); +} + +static void __maybe_unused iee_set_key_type(struct key *key, struct key_type *type) +{ + iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); +} + +static void __maybe_unused iee_set_key_domain_tag(struct key *key, struct key_tag *domain_tag) +{ + iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); +} + +static void __maybe_unused iee_set_key_description(struct key *key, char *description) +{ + iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); +} + +static void __maybe_unused iee_set_key_restrict_link(struct key *key, + struct key_restriction *restrict_link) +{ + iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); +} + +static bool __maybe_unused iee_set_key_flag_bit(struct key *key, long nr, int flag) +{ + bool ret; + + ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); + return ret; +} + +#endif diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index 18294d6ab7448..0f13cdb92c787 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -2,3 +2,4 @@ obj-y += haoc.o bitmap.o obj-y += iee/ obj-$(CONFIG_PTP) += ptp/ obj-$(CONFIG_CREDP) += credp/ +obj-$(CONFIG_KEYP) += keyp/ diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 707ebb9454b05..6ab47453dd018 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -59,6 +59,35 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_cred_rcu, (iee_func)_iee_set_cred_ucounts, #endif +#ifdef CONFIG_KEYP + (iee_func)_iee_set_key_union, + (iee_func)_iee_set_key_struct, + (iee_func)_iee_set_key_payload, + (iee_func)_iee_set_key_usage, + (iee_func)_iee_set_key_serial, + (iee_func)_iee_set_key_watchers, + (iee_func)_iee_set_key_user, + (iee_func)_iee_set_key_security, + (iee_func)_iee_set_key_expiry, + (iee_func)_iee_set_key_revoked_at, + (iee_func)_iee_set_key_last_used_at, + (iee_func)_iee_set_key_uid, + (iee_func)_iee_set_key_gid, + (iee_func)_iee_set_key_perm, + (iee_func)_iee_set_key_quotalen, + (iee_func)_iee_set_key_datalen, + (iee_func)_iee_set_key_state, + (iee_func)_iee_set_key_magic, + (iee_func)_iee_set_key_flags, + (iee_func)_iee_set_key_index_key, + (iee_func)_iee_set_key_hash, + (iee_func)_iee_set_key_len_desc, + (iee_func)_iee_set_key_type, + (iee_func)_iee_set_key_domain_tag, + (iee_func)_iee_set_key_description, + (iee_func)_iee_set_key_restrict_link, + (iee_func)_iee_set_key_flag_bit, +#endif #ifdef CONFIG_PTP #ifdef CONFIG_IEE_PTRP (iee_func)_iee_unset_token, diff --git a/arch/x86/kernel/haoc/keyp/Makefile b/arch/x86/kernel/haoc/keyp/Makefile new file mode 100644 index 0000000000000..307cebd57ff71 --- /dev/null +++ b/arch/x86/kernel/haoc/keyp/Makefile @@ -0,0 +1 @@ +obj-y += keyp.o \ No newline at end of file diff --git a/arch/x86/kernel/haoc/keyp/keyp.c b/arch/x86/kernel/haoc/keyp/keyp.c new file mode 100644 index 0000000000000..b371548d08633 --- /dev/null +++ b/arch/x86/kernel/haoc/keyp/keyp.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +struct watch_list; + +static inline void iee_verify_key_type(const struct key *key) +{ + iee_verify_type(__pa(key), IEE_KEY, "key"); +} + +unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, + long nr, int flag) +{ + iee_verify_key_type(key); + switch (flag) { + case SET_BIT_OP: { + set_bit(nr, &key->flags); + break; + } + case TEST_AND_CLEAR_BIT: { + return test_and_clear_bit(nr, &key->flags); + } + case TEST_AND_SET_BIT: { + return test_and_set_bit(nr, &key->flags); + } + } + return 0; +} + +void _iee_set_key_restrict_link(unsigned long __unused, + struct key *key, + struct key_restriction *restrict_link) +{ + iee_verify_key_type(key); + key->restrict_link = restrict_link; +} + +void _iee_set_key_magic(unsigned long __unused, struct key *key, + unsigned int magic) +{ +#ifdef KEY_DEBUGGING + iee_verify_key_type(key); + key->magic = magic; +#endif +} + +void _iee_set_key_flags(unsigned long __unused, struct key *key, + unsigned long flags) +{ + iee_verify_key_type(key); + key->flags = flags; +} + +void _iee_set_key_index_key(unsigned long __unused, + struct key *key, + struct keyring_index_key *index_key) +{ + iee_verify_key_type(key); + key->index_key = *index_key; +} + +void _iee_set_key_hash(unsigned long __unused, struct key *key, + unsigned long hash) +{ + iee_verify_key_type(key); + key->hash = hash; +} + +void _iee_set_key_len_desc(unsigned long __unused, struct key *key, + unsigned long len_desc) +{ + iee_verify_key_type(key); + key->len_desc = len_desc; +} + +void _iee_set_key_type(unsigned long __unused, struct key *key, + struct key_type *type) +{ + iee_verify_key_type(key); + key->type = type; +} + +void _iee_set_key_domain_tag(unsigned long __unused, + struct key *key, + struct key_tag *domain_tag) +{ + iee_verify_key_type(key); + key->domain_tag = domain_tag; +} + +void _iee_set_key_description(unsigned long __unused, + struct key *key, char *description) +{ + iee_verify_key_type(key); + key->description = description; +} + +void _iee_set_key_uid(unsigned long __unused, struct key *key, + kuid_t uid) +{ + iee_verify_key_type(key); + key->uid = uid; +} + +void _iee_set_key_gid(unsigned long __unused, struct key *key, + kgid_t gid) +{ + iee_verify_key_type(key); + key->gid = gid; +} + +void _iee_set_key_perm(unsigned long __unused, struct key *key, + key_perm_t perm) +{ + iee_verify_key_type(key); + key->perm = perm; +} + +void _iee_set_key_quotalen(unsigned long __unused, struct key *key, + unsigned short quotalen) +{ + iee_verify_key_type(key); + key->quotalen = quotalen; +} + +void _iee_set_key_datalen(unsigned long __unused, struct key *key, + unsigned short datalen) +{ + iee_verify_key_type(key); + key->datalen = datalen; +} + +void _iee_set_key_state(unsigned long __unused, struct key *key, + short state) +{ + iee_verify_key_type(key); + smp_store_release(&key->state, state); +} + +void _iee_set_key_user(unsigned long __unused, struct key *key, + struct key_user *user) +{ + iee_verify_key_type(key); + key->user = user; +} + +void _iee_set_key_security(unsigned long __unused, struct key *key, + void *security) +{ + iee_verify_key_type(key); + key->security = security; +} + +void _iee_set_key_expiry(unsigned long __unused, struct key *key, + time64_t expiry) +{ + iee_verify_key_type(key); + key->expiry = expiry; +} + +void _iee_set_key_revoked_at(unsigned long __unused, + struct key *key, time64_t revoked_at) +{ + iee_verify_key_type(key); + key->revoked_at = revoked_at; +} + +void _iee_set_key_last_used_at(unsigned long __unused, + struct key *key, + time64_t last_used_at) +{ + iee_verify_key_type(key); + key->last_used_at = last_used_at; +} + +unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, + int n, int flag) +{ + iee_verify_key_type(key); + switch (flag) { + case REFCOUNT_INC: { + refcount_inc(&key->usage); + break; + } + case REFCOUNT_SET: { + refcount_set(&key->usage, n); + break; + } + case REFCOUNT_DEC_AND_TEST: { + return refcount_dec_and_test(&key->usage); + } + case REFCOUNT_INC_NOT_ZERO: { + return refcount_inc_not_zero(&key->usage); + } + } + return 0; +} + +void _iee_set_key_serial(unsigned long __unused, struct key *key, + key_serial_t serial) +{ + iee_verify_key_type(key); + key->serial = serial; +} + +void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers) +{ +#ifdef CONFIG_KEY_NOTIFICATIONS + iee_verify_key_type(key); + key->watchers = watchers; +#endif +} + +void _iee_set_key_union(unsigned long __unused, struct key *key, + struct key_union *key_union) +{ + iee_verify_key_type(key); + key->graveyard_link.next = (struct list_head *)key_union; +} + +void _iee_set_key_struct(unsigned long __unused, struct key *key, + struct key_struct *key_struct) +{ + iee_verify_key_type(key); + key->name_link.prev = (struct list_head *)key_struct; +} + +void _iee_set_key_payload(unsigned long __unused, struct key *key, + union key_payload *key_payload) +{ + iee_verify_key_type(key); + key->name_link.next = (struct list_head *)key_payload; +} diff --git a/block/sed-opal.c b/block/sed-opal.c index 1a1cb35bf4b79..442cdb7b801c9 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -316,7 +316,11 @@ static int read_sed_opal_key(const char *key_name, u_char *buffer, int buflen) return PTR_ERR(kref); key = key_ref_to_ptr(kref); +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = key_validate(key); if (ret == 0) { if (buflen > key->datalen) @@ -324,7 +328,11 @@ static int read_sed_opal_key(const char *key_name, u_char *buffer, int buflen) ret = key->type->read(key, (char *)buffer, buflen); } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_ref_put(kref); diff --git a/certs/blacklist.c b/certs/blacklist.c index 675dd7a8f07ab..1d139916a9f90 100644 --- a/certs/blacklist.c +++ b/certs/blacklist.c @@ -17,6 +17,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "blacklist.h" /* @@ -91,7 +94,11 @@ static int blacklist_key_instantiate(struct key *key, #endif /* Sets safe default permissions for keys loaded by user space. */ +#ifdef CONFIG_KEYP + iee_set_key_perm(key, BLACKLIST_KEY_PERM); +#else key->perm = BLACKLIST_KEY_PERM; +#endif /* * Skips the authentication step for builtin hashes, they are not diff --git a/certs/system_keyring.c b/certs/system_keyring.c index 9de610bf1f4b2..2d1e79b2de378 100644 --- a/certs/system_keyring.c +++ b/certs/system_keyring.c @@ -93,9 +93,15 @@ int restrict_link_by_builtin_and_secondary_trusted( /* If we have a secondary trusted keyring, then that contains a link * through to the builtin keyring and the search will follow that link. */ +#ifdef CONFIG_KEYP + if (type == &key_type_keyring && + dest_keyring == secondary_trusted_keys && + payload == (union key_payload *)(builtin_trusted_keys->name_link.next)) +#else if (type == &key_type_keyring && dest_keyring == secondary_trusted_keys && payload == &builtin_trusted_keys->payload) +#endif /* Allow the builtin keyring to be added to the secondary */ return 0; @@ -122,9 +128,15 @@ int restrict_link_by_digsig_builtin_and_secondary(struct key *dest_keyring, /* If we have a secondary trusted keyring, then that contains a link * through to the builtin keyring and the search will follow that link. */ +#ifdef CONFIG_KEYP + if (type == &key_type_keyring && + dest_keyring == secondary_trusted_keys && + payload == (union key_payload *)(builtin_trusted_keys->name_link.next)) +#else if (type == &key_type_keyring && dest_keyring == secondary_trusted_keys && payload == &builtin_trusted_keys->payload) +#endif /* Allow the builtin keyring to be added to the secondary */ return 0; @@ -209,9 +221,15 @@ int restrict_link_by_builtin_secondary_and_machine( const union key_payload *payload, struct key *restrict_key) { +#ifdef CONFIG_KEYP + if (machine_trusted_keys && type == &key_type_keyring && + dest_keyring == secondary_trusted_keys && + payload == (union key_payload *)(machine_trusted_keys->name_link.next)) +#else if (machine_trusted_keys && type == &key_type_keyring && dest_keyring == secondary_trusted_keys && payload == &machine_trusted_keys->payload) +#endif /* Allow the machine keyring to be added to the secondary */ return 0; diff --git a/crypto/af_alg.c b/crypto/af_alg.c index e7fea668473d9..84cec2f219b5e 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -304,7 +304,11 @@ static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, if (IS_ERR(key)) return PTR_ERR(key); +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = ERR_PTR(-ENOPROTOOPT); if (!strcmp(key->type->name, "user") || @@ -319,21 +323,33 @@ static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, } if (IS_ERR(ret)) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return PTR_ERR(ret); } key_data = sock_kmalloc(&ask->sk, key_datalen, GFP_KERNEL); if (!key_data) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return -ENOMEM; } memcpy(key_data, ret, key_datalen); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); err = type->setkey(ask->private, key_data, key_datalen); diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index 7859b0692b42b..fb92e47d6d7a4 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -16,6 +16,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "asymmetric_keys.h" @@ -473,6 +476,18 @@ static void asymmetric_key_free_preparse(struct key_preparsed_payload *prep) static void asymmetric_key_destroy(struct key *key) { struct asymmetric_key_subtype *subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + struct asymmetric_key_ids *kids = + ((union key_payload *)(key->name_link.next))->data[asym_key_ids]; + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + void *data = ((union key_payload *)(key->name_link.next))->data[asym_crypto]; + void *auth = ((union key_payload *)(key->name_link.next))->data[asym_auth]; + + key_payload->data[asym_crypto] = NULL; + key_payload->data[asym_subtype] = NULL; + key_payload->data[asym_key_ids] = NULL; + key_payload->data[asym_auth] = NULL; +#else struct asymmetric_key_ids *kids = key->payload.data[asym_key_ids]; void *data = key->payload.data[asym_crypto]; void *auth = key->payload.data[asym_auth]; @@ -481,6 +496,7 @@ static void asymmetric_key_destroy(struct key *key) key->payload.data[asym_subtype] = NULL; key->payload.data[asym_key_ids] = NULL; key->payload.data[asym_auth] = NULL; +#endif if (subtype) { subtype->destroy(data, auth); @@ -586,8 +602,13 @@ int asymmetric_key_eds_op(struct kernel_pkey_params *params, if (key->type != &key_type_asymmetric) return -EINVAL; subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + if (!subtype || + !((union key_payload *)(key->name_link.next))->data[0]) +#else if (!subtype || !key->payload.data[0]) +#endif return -EINVAL; if (!subtype->eds_op) return -ENOTSUPP; diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index 1dcab27986a6a..2f7db6236ee23 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -30,7 +30,12 @@ MODULE_LICENSE("GPL"); static void public_key_describe(const struct key *asymmetric_key, struct seq_file *m) { +#ifdef CONFIG_KEYP + struct public_key *key = + ((union key_payload *)(asymmetric_key->name_link.next))->data[asym_crypto]; +#else struct public_key *key = asymmetric_key->payload.data[asym_crypto]; +#endif if (key) seq_printf(m, "%s.%s", key->id_type, key->pkey_algo); @@ -158,7 +163,12 @@ static int software_key_query(const struct kernel_pkey_params *params, struct kernel_pkey_query *info) { struct crypto_akcipher *tfm; +#ifdef CONFIG_KEYP + struct public_key *pkey = + ((union key_payload *)(params->key->name_link.next))->data[asym_crypto]; +#else struct public_key *pkey = params->key->payload.data[asym_crypto]; +#endif char alg_name[CRYPTO_MAX_ALG_NAME]; struct crypto_sig *sig; u8 *key, *ptr; @@ -273,7 +283,12 @@ static int software_key_query(const struct kernel_pkey_params *params, static int software_key_eds_op(struct kernel_pkey_params *params, const void *in, void *out) { +#ifdef CONFIG_KEYP + const struct public_key *pkey = + ((union key_payload *)(params->key->name_link.next))->data[asym_crypto]; +#else const struct public_key *pkey = params->key->payload.data[asym_crypto]; +#endif char alg_name[CRYPTO_MAX_ALG_NAME]; struct crypto_akcipher *tfm; struct crypto_sig *sig; @@ -453,7 +468,12 @@ EXPORT_SYMBOL_GPL(public_key_verify_signature); static int public_key_verify_signature_2(const struct key *key, const struct public_key_signature *sig) { +#ifdef CONFIG_KEYP + const struct public_key *pk = + ((union key_payload *)(key->name_link.next))->data[asym_crypto]; +#else const struct public_key *pk = key->payload.data[asym_crypto]; +#endif return public_key_verify_signature(pk, sig); } diff --git a/crypto/asymmetric_keys/signature.c b/crypto/asymmetric_keys/signature.c index 2deff81f8af50..6d0708bf095bd 100644 --- a/crypto/asymmetric_keys/signature.c +++ b/crypto/asymmetric_keys/signature.c @@ -51,8 +51,13 @@ int query_asymmetric_key(const struct kernel_pkey_params *params, if (key->type != &key_type_asymmetric) return -EINVAL; subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + if (!subtype || + !((union key_payload *)(key->name_link.next))->data[0]) +#else if (!subtype || !key->payload.data[0]) +#endif return -EINVAL; if (!subtype->query) return -ENOTSUPP; @@ -145,8 +150,13 @@ int verify_signature(const struct key *key, if (key->type != &key_type_asymmetric) return -EINVAL; subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + if (!subtype || + !((union key_payload *)(key->name_link.next))->data[0]) +#else if (!subtype || !key->payload.data[0]) +#endif return -EINVAL; if (!subtype->verify_signature) return -ENOTSUPP; diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 7e2f2ffd3ceea..80948eebbdc83 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2505,7 +2505,11 @@ static int set_key_encrypted(struct crypt_config *cc, struct key *key) { const struct encrypted_key_payload *ekp; +#ifdef CONFIG_KEYP + ekp = ((union key_payload *)(key->name_link.next))->data[0]; +#else ekp = key->payload.data[0]; +#endif if (!ekp) return -EKEYREVOKED; @@ -2521,7 +2525,11 @@ static int set_key_trusted(struct crypt_config *cc, struct key *key) { const struct trusted_key_payload *tkp; +#ifdef CONFIG_KEYP + tkp = ((union key_payload *)(key->name_link.next))->data[0]; +#else tkp = key->payload.data[0]; +#endif if (!tkp) return -EKEYREVOKED; @@ -2583,17 +2591,29 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string return PTR_ERR(key); } +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = set_key(cc, key); if (ret < 0) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); kfree_sensitive(new_key_string); return ret; } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); /* clear the flag since following operations may invalidate previously valid key */ diff --git a/drivers/md/dm-verity-verify-sig.c b/drivers/md/dm-verity-verify-sig.c index 1ced2ef255f32..449432b80983e 100644 --- a/drivers/md/dm-verity-verify-sig.c +++ b/drivers/md/dm-verity-verify-sig.c @@ -40,7 +40,11 @@ static int verity_verify_get_sig_from_key(const char *key_desc, if (IS_ERR(key)) return PTR_ERR(key); +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ukp = user_key_payload_locked(key); if (!ukp) { @@ -58,7 +62,11 @@ static int verity_verify_get_sig_from_key(const char *key_desc, memcpy(sig_opts->sig, ukp->data, sig_opts->sig_size); end: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return ret; diff --git a/drivers/nvdimm/security.c b/drivers/nvdimm/security.c index a03e3c45f2973..a901824dfb7c7 100644 --- a/drivers/nvdimm/security.c +++ b/drivers/nvdimm/security.c @@ -28,7 +28,11 @@ static void *key_data(struct key *key) { struct encrypted_key_payload *epayload = dereference_key_locked(key); +#ifdef CONFIG_KEYP + lockdep_assert_held_read(&KEY_SEM(key)); +#else lockdep_assert_held_read(&key->sem); +#endif return epayload->decrypted_data; } @@ -38,7 +42,11 @@ static void nvdimm_put_key(struct key *key) if (!key) return; +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); } @@ -65,10 +73,18 @@ static struct key *nvdimm_request_key(struct nvdimm *nvdimm) } else { struct encrypted_key_payload *epayload; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif epayload = dereference_key_locked(key); if (epayload->decrypted_datalen != NVDIMM_PASSPHRASE_LEN) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); key = NULL; } @@ -107,10 +123,18 @@ static struct key *nvdimm_lookup_user_key(struct nvdimm *nvdimm, dev_dbg(dev, "%s: key found: %#x\n", __func__, key_serial(key)); +#ifdef CONFIG_KEYP + down_read_nested(&KEY_SEM(key), subclass); +#else down_read_nested(&key->sem, subclass); +#endif epayload = dereference_key_locked(key); if (epayload->decrypted_datalen != NVDIMM_PASSPHRASE_LEN) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); key = NULL; } diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index 7cbb1fd872acc..9108d5401bda8 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -590,8 +590,13 @@ static void fscrypt_provisioning_key_describe(const struct key *key, { seq_puts(m, key->description); if (key_is_positive(key)) { +#ifdef CONFIG_KEYP + const struct fscrypt_provisioning_key_payload *payload = + ((union key_payload *)(key->name_link.next))->data[0]; +#else const struct fscrypt_provisioning_key_payload *payload = key->payload.data[0]; +#endif seq_printf(m, ": %u [%u]", key->datalen, payload->type); } @@ -599,7 +604,11 @@ static void fscrypt_provisioning_key_describe(const struct key *key, static void fscrypt_provisioning_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree_sensitive(key->payload.data[0]); +#endif } static struct key_type key_type_fscrypt_provisioning = { @@ -641,7 +650,11 @@ static int get_keyring_key(u32 key_id, u32 type, if (key->type != &key_type_fscrypt_provisioning) goto bad_key; +#ifdef CONFIG_KEYP + payload = ((union key_payload *)(key->name_link.next))->data[0]; +#else payload = key->payload.data[0]; +#endif /* Don't allow fscrypt v1 keys to be used as v2 keys and vice versa. */ if (payload->type != type) @@ -1030,7 +1043,12 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) down_write(&mk->mk_sem); /* If relevant, remove current user's (or all users) claim to the key */ +#ifdef CONFIG_KEYP + if (mk->mk_users && + ((struct key_struct *)(mk->mk_users->name_link.prev))->keys.nr_leaves_on_tree != 0) { +#else if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) { +#endif if (all_users) err = keyring_clear(mk->mk_users); else @@ -1039,7 +1057,12 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) up_write(&mk->mk_sem); goto out_put_key; } +#ifdef CONFIG_KEYP + if (((struct key_struct *)(mk->mk_users->name_link.prev))-> + keys.nr_leaves_on_tree != 0) { +#else if (mk->mk_users->keys.nr_leaves_on_tree != 0) { +#endif /* * Other users have still added the key too. We removed * the current user's claim to the key, but we still @@ -1162,7 +1185,12 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) if (mk->mk_users) { struct key *mk_user; +#ifdef CONFIG_KEYP + arg.user_count = + ((struct key_struct *)(mk->mk_users->name_link.prev))->keys.nr_leaves_on_tree; +#else arg.user_count = mk->mk_users->keys.nr_leaves_on_tree; +#endif mk_user = find_master_key_user(mk); if (!IS_ERR(mk_user)) { arg.status_flags |= diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 7e0bb9a75c590..a5734069cebd6 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -110,7 +110,11 @@ find_and_lock_process_key(const char *prefix, if (IS_ERR(key)) return key; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ukp = user_key_payload_locked(key); if (!ukp) /* was the key revoked before we acquired its semaphore? */ @@ -137,7 +141,11 @@ find_and_lock_process_key(const char *prefix, return key; invalid: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return ERR_PTR(-ENOKEY); } @@ -322,7 +330,11 @@ int fscrypt_setup_v1_file_key_via_subscribed_keyrings(struct fscrypt_info *ci) return PTR_ERR(key); err = fscrypt_setup_v1_file_key(ci, payload->raw); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return err; } diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h index c586c5db18b5e..6bc056c38205a 100644 --- a/fs/ecryptfs/ecryptfs_kernel.h +++ b/fs/ecryptfs/ecryptfs_kernel.h @@ -82,7 +82,11 @@ ecryptfs_get_encrypted_key_payload_data(struct key *key) if (key->type != &key_type_encrypted) return NULL; +#ifdef CONFIG_KEYP + payload = ((union key_payload *)(key->name_link.next))->data[0]; +#else payload = key->payload.data[0]; +#endif if (!payload) return ERR_PTR(-EKEYREVOKED); diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c index 7f9f68c00ef63..3234e9ceff581 100644 --- a/fs/ecryptfs/keystore.c +++ b/fs/ecryptfs/keystore.c @@ -512,7 +512,11 @@ ecryptfs_find_global_auth_tok_for_sig( goto out_invalid_auth_tok; } +#ifdef CONFIG_KEYP + down_write(&(KEY_SEM(walker->global_auth_tok_key))); +#else down_write(&(walker->global_auth_tok_key->sem)); +#endif rc = ecryptfs_verify_auth_tok_from_key( walker->global_auth_tok_key, auth_tok); if (rc) @@ -525,7 +529,11 @@ ecryptfs_find_global_auth_tok_for_sig( rc = -ENOENT; goto out; out_invalid_auth_tok_unlock: +#ifdef CONFIG_KEYP + up_write(&(KEY_SEM(walker->global_auth_tok_key))); +#else up_write(&(walker->global_auth_tok_key->sem)); +#endif out_invalid_auth_tok: printk(KERN_WARNING "Invalidating auth tok with sig = [%s]\n", sig); walker->flags |= ECRYPTFS_AUTH_TOK_INVALID; @@ -846,7 +854,11 @@ ecryptfs_write_tag_70_packet(char *dest, size_t *remaining_bytes, mutex_unlock(s->tfm_mutex); out: if (auth_tok_key) { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); } skcipher_request_free(s->skcipher_req); @@ -1088,7 +1100,11 @@ ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, (*filename) = NULL; } if (auth_tok_key) { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); } skcipher_request_free(s->skcipher_req); @@ -1625,10 +1641,18 @@ int ecryptfs_keyring_auth_tok_for_sig(struct key **auth_tok_key, goto out; } } +#ifdef CONFIG_KEYP + down_write(&KEY_SEM((*auth_tok_key))); +#else down_write(&(*auth_tok_key)->sem); +#endif rc = ecryptfs_verify_auth_tok_from_key(*auth_tok_key, auth_tok); if (rc) { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM((*auth_tok_key))); +#else up_write(&(*auth_tok_key)->sem); +#endif key_put(*auth_tok_key); (*auth_tok_key) = NULL; goto out; @@ -1901,7 +1925,11 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, memcpy(&(candidate_auth_tok->token.private_key), &(matching_auth_tok->token.private_key), sizeof(struct ecryptfs_private_key)); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); rc = decrypt_pki_encrypted_session_key(candidate_auth_tok, crypt_stat); @@ -1909,12 +1937,20 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, memcpy(&(candidate_auth_tok->token.password), &(matching_auth_tok->token.password), sizeof(struct ecryptfs_password)); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); rc = decrypt_passphrase_encrypted_session_key( candidate_auth_tok, crypt_stat); } else { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); rc = -EINVAL; } @@ -1976,7 +2012,11 @@ pki_encrypt_session_key(struct key *auth_tok_key, crypt_stat->cipher, crypt_stat->key_size), crypt_stat, &payload, &payload_len); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); if (rc) { ecryptfs_printk(KERN_ERR, "Error generating tag 66 packet\n"); @@ -2040,7 +2080,11 @@ write_tag_1_packet(char *dest, size_t *remaining_bytes, memcpy(key_rec->enc_key, auth_tok->session_key.encrypted_key, auth_tok->session_key.encrypted_key_size); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); goto encrypted_session_key_set; } @@ -2438,7 +2482,11 @@ ecryptfs_generate_key_packet_set(char *dest_base, &max, auth_tok, crypt_stat, key_rec, &written); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); if (rc) { ecryptfs_printk(KERN_WARNING, "Error " @@ -2467,7 +2515,11 @@ ecryptfs_generate_key_packet_set(char *dest_base, } (*len) += written; } else { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); ecryptfs_printk(KERN_WARNING, "Unsupported " "authentication token type\n"); diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index 2dc927ba067fe..9c6c7e094b3a8 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -202,7 +202,11 @@ static int ecryptfs_init_global_auth_toks( goto out; } else { global_auth_tok->flags &= ~ECRYPTFS_AUTH_TOK_INVALID; +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(global_auth_tok->global_auth_tok_key)); +#else up_write(&(global_auth_tok->global_auth_tok_key)->sem); +#endif } } out: diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c index 4eb5574699324..0543d4061de1a 100644 --- a/fs/nfs/nfs4idmap.c +++ b/fs/nfs/nfs4idmap.c @@ -47,6 +47,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #ifdef CONFIG_CREDP @@ -230,14 +233,18 @@ int nfs_idmap_init(void) if (ret < 0) goto failed_reg_legacy; +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif id_resolver_cache = cred; return 0; @@ -306,7 +313,11 @@ static struct key *nfs_idmap_request_key(const char *name, size_t namelen, mutex_unlock(&idmap->idmap_mutex); } if (!IS_ERR(rkey)) +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(rkey, KEY_FLAG_ROOT_CAN_INVAL, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_INVAL, &rkey->flags); +#endif kfree(desc); return rkey; @@ -331,7 +342,11 @@ static ssize_t nfs_idmap_get_key(const char *name, size_t namelen, } rcu_read_lock(); +#ifdef CONFIG_KEYP + iee_set_key_perm(rkey, rkey->perm | KEY_USR_VIEW); +#else rkey->perm |= KEY_USR_VIEW; +#endif ret = key_validate(rkey); if (ret < 0) diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c index eea9ed2bef0b0..26dc520dff8eb 100644 --- a/fs/smb/client/cifs_spnego.c +++ b/fs/smb/client/cifs_spnego.c @@ -18,6 +18,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "cifsglob.h" #include "cifs_spnego.h" #include "cifs_debug.h" @@ -37,7 +40,11 @@ cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) goto error; /* attach the data */ +#ifdef CONFIG_KEYP + ((union key_payload *)(key->name_link.next))->data[0] = payload; +#else key->payload.data[0] = payload; +#endif ret = 0; error: @@ -47,7 +54,11 @@ cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) static void cifs_spnego_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree(key->payload.data[0]); +#endif } static int @@ -184,7 +195,12 @@ cifs_get_spnego_key(struct cifs_ses *sesInfo, #ifdef CONFIG_CIFS_DEBUG2 if (cifsFYI && !IS_ERR(spnego_key)) { +#ifdef CONFIG_KEYP + struct cifs_spnego_msg *msg = + ((union key_payload *)(spnego_key->name_link.next))->data[0]; +#else struct cifs_spnego_msg *msg = spnego_key->payload.data[0]; +#endif cifs_dump_mem("SPNEGO reply blob:", msg->data, min(1024U, msg->secblob_len + msg->sesskey_len)); } @@ -232,14 +248,18 @@ init_cifs_spnego(void) * instruct request_key() to use this special keyring as a cache for * the results it looks up */ +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif spnego_cred = cred; cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring)); diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index a57b263744c2a..f3cff2b47ce64 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -20,6 +20,10 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#include +#endif #include "cifspdu.h" #include "cifsglob.h" #include "cifsacl.h" @@ -81,16 +85,33 @@ cifs_idmap_key_instantiate(struct key *key, struct key_preparsed_payload *prep) * dereference payload.data! */ if (prep->datalen <= sizeof(key->payload)) { +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + key_payload->data[0] = NULL; + memcpy(key_payload, prep->data, prep->datalen); +#else key->payload.data[0] = NULL; memcpy(&key->payload, prep->data, prep->datalen); +#endif } else { payload = kmemdup(prep->data, prep->datalen, GFP_KERNEL); if (!payload) return -ENOMEM; +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + key_payload->data[0] = payload; +#else key->payload.data[0] = payload; +#endif } +#ifdef CONFIG_KEYP + iee_set_key_datalen(key, prep->datalen); +#else key->datalen = prep->datalen; +#endif return 0; } @@ -98,7 +119,11 @@ static inline void cifs_idmap_key_destroy(struct key *key) { if (key->datalen > sizeof(key->payload)) +#ifdef CONFIG_KEYP + kfree(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree(key->payload.data[0]); +#endif } static struct key_type cifs_idmap_key_type = { @@ -314,9 +339,15 @@ id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid) * there are no subauthorities and the host has 8-byte pointers, then * it could be. */ +#ifdef CONFIG_KEYP + ksid = sidkey->datalen <= sizeof(sidkey->payload) ? + (struct smb_sid *)(sidkey->name_link.next) : + (struct smb_sid *)((union key_payload *)(sidkey->name_link.next))->data[0]; +#else ksid = sidkey->datalen <= sizeof(sidkey->payload) ? (struct smb_sid *)&sidkey->payload : (struct smb_sid *)sidkey->payload.data[0]; +#endif ksid_size = CIFS_SID_BASE_SIZE + (ksid->num_subauth * sizeof(__le32)); if (ksid_size > sidkey->datalen) { @@ -425,14 +456,24 @@ sid_to_id(struct cifs_sb_info *cifs_sb, struct smb_sid *psid, if (sidtype == SIDOWNER) { kuid_t uid; uid_t id; +#ifdef CONFIG_KEYP + memcpy(&id, + &((union key_payload *)(sidkey->name_link.next))->data[0], sizeof(uid_t)); +#else memcpy(&id, &sidkey->payload.data[0], sizeof(uid_t)); +#endif uid = make_kuid(&init_user_ns, id); if (uid_valid(uid)) fuid = uid; } else { kgid_t gid; gid_t id; +#ifdef CONFIG_KEYP + memcpy(&id, &((union key_payload *)(sidkey->name_link.next))->data[0], + sizeof(gid_t)); +#else memcpy(&id, &sidkey->payload.data[0], sizeof(gid_t)); +#endif gid = make_kgid(&init_user_ns, id); if (gid_valid(gid)) fgid = gid; @@ -493,14 +534,18 @@ init_cifs_idmap(void) /* instruct request_key() to use this special keyring as a cache for * the results it looks up */ +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif root_cred = cred; cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring)); diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index fe5d21f174c47..5e165e87f3cfe 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -2208,7 +2208,11 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) is_domain = 1; } +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif upayload = user_key_payload_locked(key); if (IS_ERR_OR_NULL(upayload)) { rc = upayload ? PTR_ERR(upayload) : -EINVAL; @@ -2285,7 +2289,11 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) strscpy(ctx->workstation_name, ses->workstation_name, sizeof(ctx->workstation_name)); out_key_put: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); out_err: kfree(desc); diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c index bbde7180a90ac..4a8cff5a1810f 100644 --- a/fs/smb/client/sess.c +++ b/fs/smb/client/sess.c @@ -1625,7 +1625,11 @@ sess_auth_kerberos(struct sess_data *sess_data) goto out; } +#ifdef CONFIG_KEYP + msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; +#else msg = spnego_key->payload.data[0]; +#endif /* * check version field to make sure that cifs.upcall is * sending us a response in an expected form diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 7f26e6fdd6abf..2c4fb76afaea2 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -1644,7 +1644,11 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) goto out; } +#ifdef CONFIG_KEYP + msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; +#else msg = spnego_key->payload.data[0]; +#endif /* * check version field to make sure that cifs.upcall is * sending us a response in an expected form diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index e564d5ff87816..3503af364d878 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -284,7 +284,11 @@ int ubifs_init_authentication(struct ubifs_info *c) return PTR_ERR(keyring_key); } +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(keyring_key)); +#else down_read(&keyring_key->sem); +#endif if (keyring_key->type != &key_type_logon) { ubifs_err(c, "key type must be logon"); @@ -351,7 +355,11 @@ int ubifs_init_authentication(struct ubifs_info *c) if (err) crypto_free_shash(c->hash_tfm); out: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(keyring_key)); +#else up_read(&keyring_key->sem); +#endif key_put(keyring_key); return err; diff --git a/fs/verity/signature.c b/fs/verity/signature.c index 90c07573dd77b..3aafb8d467028 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -62,7 +62,12 @@ int fsverity_verify_signature(const struct fsverity_info *vi, return 0; } +#ifdef CONFIG_KEYP + if (((struct key_struct *)(fsverity_keyring->name_link.prev))-> + keys.nr_leaves_on_tree == 0) { +#else if (fsverity_keyring->keys.nr_leaves_on_tree == 0) { +#endif /* * The ".fs-verity" keyring is empty, due to builtin signatures * being supported by the kernel but not actually being used. diff --git a/include/keys/asymmetric-subtype.h b/include/keys/asymmetric-subtype.h index d55171f640a04..d996bd0476b50 100644 --- a/include/keys/asymmetric-subtype.h +++ b/include/keys/asymmetric-subtype.h @@ -54,7 +54,11 @@ struct asymmetric_key_subtype { static inline struct asymmetric_key_subtype *asymmetric_key_subtype(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[asym_subtype]; +#else return key->payload.data[asym_subtype]; +#endif } #endif /* _KEYS_ASYMMETRIC_SUBTYPE_H */ diff --git a/include/keys/asymmetric-type.h b/include/keys/asymmetric-type.h index 69a13e1e5b2e5..0d009989e3cee 100644 --- a/include/keys/asymmetric-type.h +++ b/include/keys/asymmetric-type.h @@ -69,13 +69,21 @@ extern struct asymmetric_key_id *asymmetric_key_generate_id(const void *val_1, static inline const struct asymmetric_key_ids *asymmetric_key_ids(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[asym_key_ids]; +#else return key->payload.data[asym_key_ids]; +#endif } static inline const struct public_key *asymmetric_key_public_key(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[asym_crypto]; +#else return key->payload.data[asym_crypto]; +#endif } extern struct key *find_asymmetric_key(struct key *keyring, diff --git a/include/keys/request_key_auth-type.h b/include/keys/request_key_auth-type.h index 01e42ee5f4099..7fc483cde2684 100644 --- a/include/keys/request_key_auth-type.h +++ b/include/keys/request_key_auth-type.h @@ -28,7 +28,11 @@ struct request_key_auth { static inline struct request_key_auth *get_request_key_auth(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[0]; +#else return key->payload.data[0]; +#endif } diff --git a/include/linux/key.h b/include/linux/key.h index ffd0c00e89c71..1881a5c77d37e 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -283,6 +283,36 @@ struct key { DEEPIN_KABI_RESERVE(2) }; +#ifdef CONFIG_KEYP +struct key_union { + union { + struct list_head graveyard_link; + struct rb_node serial_node; + }; + struct rw_semaphore sem; + struct key *key; +}; + +struct key_struct { + struct { + /* Keyring bits */ + struct list_head name_link; + struct assoc_array keys; + }; + struct key *key; +}; +#define KEY_SEM(KEY) (((struct key_union *)(KEY->graveyard_link.next))->sem) +#include +extern unsigned long long iee_rw_gate(int flag, ...); +static bool iee_set_key_usage(struct key *key, int n, int flag) +{ + bool ret; + + ret = iee_rw_gate(IEE_OP_SET_KEY_USAGE, key, n, flag); + return ret; +} +#endif + extern struct key *key_alloc(struct key_type *type, const char *desc, kuid_t uid, kgid_t gid, @@ -308,7 +338,11 @@ extern void key_remove_domain(struct key_tag *domain_tag); static inline struct key *__key_get(struct key *key) { +#ifdef CONFIG_KEYP + iee_set_key_usage(key, 0, REFCOUNT_INC); +#else refcount_inc(&key->usage); +#endif return key; } @@ -481,6 +515,47 @@ static inline bool key_is_negative(const struct key *key) return key_read_state(key) < 0; } +#ifdef CONFIG_KEYP +static inline void iee_write_key_payload_rcu_data0(struct key *key, void *rcu_data0) +{ + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + WRITE_ONCE(key_payload->rcu_data0, rcu_data0); +} + +#define dereference_key_rcu(KEY) \ + (rcu_dereference(((union key_payload *)(KEY->name_link.next))->rcu_data0)) + +#define dereference_key_locked(KEY) \ + (rcu_dereference_protected(((union key_payload *)(KEY->name_link.next))->rcu_data0, \ + rwsem_is_locked(&KEY_SEM(((struct key *)(KEY)))))) + +#define iee_smp_store_release(p, v, KEY) \ +do { \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + iee_write_key_payload_rcu_data0(KEY, v); \ +} while (0) + +#define iee_rcu_assign_pointer(p, v, KEY) \ +do { \ + uintptr_t _r_a_p__v = (uintptr_t)(v); \ + rcu_check_sparse(p, __rcu); \ + \ + if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \ + iee_write_key_payload_rcu_data0(KEY, (typeof(p))(_r_a_p__v)); \ + else \ + iee_smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v), KEY); \ +} while (0) + +#define rcu_assign_keypointer(KEY, PAYLOAD) \ +do { \ + union key_payload *tmp = (union key_payload *)(KEY->name_link.next); \ + \ + iee_rcu_assign_pointer(tmp->rcu_data0, (PAYLOAD), KEY); \ +} while (0) + +#else #define dereference_key_rcu(KEY) \ (rcu_dereference((KEY)->payload.rcu_data0)) @@ -492,6 +567,7 @@ static inline bool key_is_negative(const struct key *key) do { \ rcu_assign_pointer((KEY)->payload.rcu_data0, (PAYLOAD)); \ } while (0) +#endif /* * the userspace interface diff --git a/lib/digsig.c b/lib/digsig.c index 04b5e55ed95f5..33ebb03cd2aed 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -81,7 +81,11 @@ static int digsig_verify_rsa(struct key *key, const struct user_key_payload *ukp; struct pubkey_hdr *pkh; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ukp = user_key_payload_locked(key); if (!ukp) { @@ -176,7 +180,11 @@ static int digsig_verify_rsa(struct key *key, while (--i >= 0) mpi_free(pkey[i]); err1: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif return err; } diff --git a/mm/slab.h b/mm/slab.h index 164ee349cdb29..60e1abcfa8bc1 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -28,6 +28,12 @@ typedef u64 freelist_full_t; #ifdef CONFIG_CREDP extern struct kmem_cache *cred_jar; #endif +#ifdef CONFIG_KEYP +extern struct kmem_cache *key_jar; +extern struct kmem_cache *key_union_jar; +extern struct kmem_cache *key_struct_jar; +extern struct kmem_cache *key_payload_jar; +#endif /* * Freelist pointer and counter to cmpxchg together, avoids the typical ABA @@ -725,9 +731,9 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, return s; } -#ifdef CONFIG_CREDP -#include -#endif + #if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) + #include + #endif static inline void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, gfp_t flags, size_t size, void **p, bool init, @@ -776,8 +782,16 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, p[i] = kasan_slab_alloc(s, p[i], flags, kasan_init); if (p[i] && init && (!kasan_init || !kasan_has_integrated_init())) { + #if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) + bool use_iee_memset = false; + #ifdef CONFIG_CREDP - if (haoc_enabled && s == cred_jar) + use_iee_memset |= haoc_enabled && s == cred_jar; + #endif + #ifdef CONFIG_KEYP + use_iee_memset |= haoc_enabled && s == key_jar; + #endif + if (use_iee_memset) iee_memset(p[i], 0, zero_size); else memset(p[i], 0, zero_size); diff --git a/mm/slab_common.c b/mm/slab_common.c index 2e2b43fae2c3f..84eed4708894f 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -150,6 +150,12 @@ int slab_unmergeable(struct kmem_cache *s) return 1; #endif +#ifdef CONFIG_KEYP + if (strcmp(s->name, "key_jar") == 0 || strcmp(s->name, "key_union_jar") == 0 || + strcmp(s->name, "key_struct_jar") == 0 || strcmp(s->name, "key_payload_jar") == 0) + return 1; +#endif + /* * We may have set a slab to be unmergeable during bootstrap. */ @@ -170,6 +176,12 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, if (ctor) return NULL; +#ifdef CONFIG_KEYP + if (strcmp(name, "key_jar") == 0 || strcmp(name, "key_union_jar") == 0 || + strcmp(name, "key_struct_jar") == 0 || strcmp(name, "key_payload_jar") == 0) + return NULL; +#endif + size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); size = ALIGN(size, align); diff --git a/mm/slub.c b/mm/slub.c index a75373756a61f..88b2a9d473d66 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -490,7 +490,14 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) (void *)(freelist_ptr_encode(s, fp, freeptr_addr).v)); return; } - #endif +#endif +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar) { + iee_set_freeptr((void **)freeptr_addr, + (void *)freelist_ptr_encode(s, fp, freeptr_addr).v); + return; + } +#endif *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); } @@ -2131,6 +2138,11 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; +#ifdef CONFIG_KEYP + if (s == key_jar) + alloc_gfp |= __GFP_ZERO; +#endif + slab = alloc_slab_page(alloc_gfp, node, oo); #ifdef CONFIG_IEE_PTRP if(haoc_enabled) @@ -2143,6 +2155,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) * Allocation may have failed due to fragmentation. * Try a lower order alloc if possible */ +#ifdef CONFIG_KEYP + if (s == key_jar) + alloc_gfp |= __GFP_ZERO; +#endif slab = alloc_slab_page(alloc_gfp, node, oo); #ifdef CONFIG_IEE_PTRP if(haoc_enabled) @@ -2178,6 +2194,18 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) #endif } #endif +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar) { +#ifdef CONFIG_X86_64 + set_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << oo_order(oo), IEE_KEY); +#else + set_iee_page_type((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + oo_order(oo), IEE_KEY); +#endif + } +#endif + account_slab(slab, oo_order(oo), s, flags); slab->slab_cache = s; @@ -2248,7 +2276,17 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) pages, IEE_NORMAL); unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); } - #endif +#endif + } +#endif +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar) { +#ifdef CONFIG_X86_64 + unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), + 1 << order); +#else + unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); +#endif } #endif __free_pages(&folio->page, order); @@ -4632,9 +4670,13 @@ static int calculate_sizes(struct kmem_cache *s) s->size = size; s->reciprocal_size = reciprocal_value(size); order = calculate_order(size); - #ifdef CONFIG_IEE +#ifdef CONFIG_IEE order = iee_calculate_order(s, order); - #endif +#endif +#ifdef CONFIG_KEYP + if (strcmp(s->name, "key_jar") == 0) + order = IEE_DATA_ORDER; +#endif if ((int)order < 0) return 0; diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 15bba470960d8..e3f815c1bbda3 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -385,7 +385,11 @@ static int get_secret(struct ceph_crypto_key *dst, const char *name, goto out; } +#ifdef CONFIG_KEYP + ckey = ((union key_payload *)(ukey->name_link.next))->data[0]; +#else ckey = ukey->payload.data[0]; +#endif err = ceph_crypto_key_clone(dst, ckey); if (err) goto out_key; diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 3397d105f74f9..353d241439e06 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -338,7 +338,11 @@ static void ceph_key_free_preparse(struct key_preparsed_payload *prep) static void ceph_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + struct ceph_crypto_key *ckey = ((union key_payload *)(key->name_link.next))->data[0]; +#else struct ceph_crypto_key *ckey = key->payload.data[0]; +#endif ceph_crypto_key_destroy(ckey); kfree(ckey); diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c index a22c036b6050d..256c31fbc5abf 100644 --- a/net/dns_resolver/dns_key.c +++ b/net/dns_resolver/dns_key.c @@ -35,6 +35,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" MODULE_DESCRIPTION("DNS Resolver"); @@ -298,7 +301,12 @@ static void dns_resolver_describe(const struct key *key, struct seq_file *m) { seq_puts(m, key->description); if (key_is_positive(key)) { +#ifdef CONFIG_KEYP + int err = PTR_ERR(((union key_payload *)(key->name_link.next))-> + data[dns_key_error]); +#else int err = PTR_ERR(key->payload.data[dns_key_error]); +#endif if (err) seq_printf(m, ": %d", err); @@ -314,7 +322,11 @@ static void dns_resolver_describe(const struct key *key, struct seq_file *m) static long dns_resolver_read(const struct key *key, char *buffer, size_t buflen) { +#ifdef CONFIG_KEYP + int err = PTR_ERR(((union key_payload *)(key->name_link.next))->data[dns_key_error]); +#else int err = PTR_ERR(key->payload.data[dns_key_error]); +#endif if (err) return err; @@ -367,14 +379,18 @@ static int __init init_dns_resolver(void) /* instruct request_key() to use this special keyring as a cache for * the results it looks up */ +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif dns_resolver_cache = cred; kdebug("DNS resolver keyring: %d\n", key_serial(keyring)); diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 1a87cbd662c5a..676c0c14d45aa 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -307,7 +307,11 @@ struct rxrpc_call *rxrpc_kernel_begin_call(struct socket *sock, if (!key) key = rx->key; +#ifdef CONFIG_KEYP + if (key && !((union key_payload *)key->name_link.next)->data[0]) +#else if (key && !key->payload.data[0]) +#endif key = NULL; /* a no-security key */ memset(&p, 0, sizeof(p)); diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index 25fbef44ca9d6..749215fa60556 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -279,8 +279,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, if (ret < 0) return ret; +#ifdef CONFIG_KEYP + ret = conn->security->init_connection_security(conn, + ((union key_payload *)(conn->key->name_link.next))->data[0]); +#else ret = conn->security->init_connection_security( conn, conn->key->payload.data[0]); +#endif if (ret < 0) return ret; diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c index 73659197edffa..b5cf7d82a5b5e 100644 --- a/net/rxrpc/key.c +++ b/net/rxrpc/key.c @@ -418,7 +418,11 @@ static void rxrpc_free_preparse(struct key_preparsed_payload *prep) */ static void rxrpc_destroy(struct key *key) { +#ifdef CONFIG_KEYP + rxrpc_free_token_list(((union key_payload *)(key->name_link.next))->data[0]); +#else rxrpc_free_token_list(key->payload.data[0]); +#endif } /* @@ -431,7 +435,12 @@ static void rxrpc_describe(const struct key *key, struct seq_file *m) seq_puts(m, key->description); +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif seq_puts(m, sep); switch (token->security_index) { @@ -589,7 +598,12 @@ static long rxrpc_read(const struct key *key, size += 1 * 4; /* token count */ ntoks = 0; +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif toksize = 4; /* sec index */ switch (token->security_index) { @@ -659,7 +673,12 @@ static long rxrpc_read(const struct key *key, ENCODE(ntoks); tok = 0; +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif toksize = toksizes[tok++]; ENCODE(toksize); oldxdr = xdr; diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index 7b12fd90730d1..f0d0dca3fca0e 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -19,6 +19,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "ar-internal.h" #define RXKAD_VERSION 2 @@ -88,10 +91,19 @@ static void rxkad_free_preparse_server_key(struct key_preparsed_payload *prep) static void rxkad_destroy_server_key(struct key *key) { +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + if (key_payload->data[0]) { + crypto_free_skcipher(key_payload->data[0]); + key_payload->data[0] = NULL; + } +#else if (key->payload.data[0]) { crypto_free_skcipher(key->payload.data[0]); key->payload.data[0] = NULL; } +#endif } /* @@ -206,7 +218,11 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, return -ENOMEM; } +#ifdef CONFIG_KEYP + token = ((union key_payload *)(conn->key->name_link.next))->data[0]; +#else token = conn->key->payload.data[0]; +#endif memcpy(&iv, token->kad->session_key, sizeof(iv)); tmpbuf[0] = htonl(conn->proto.epoch); @@ -319,7 +335,11 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, } /* encrypt from the session key */ +#ifdef CONFIG_KEYP + token = ((union key_payload *)(call->conn->key->name_link.next))->data[0]; +#else token = call->conn->key->payload.data[0]; +#endif memcpy(&iv, token->kad->session_key, sizeof(iv)); sg_init_one(&sg, txb->data, txb->len); @@ -498,7 +518,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, sg_init_one(sg, data, len); /* decrypt from the session key */ +#ifdef CONFIG_KEYP + token = ((union key_payload *)(call->conn->key->name_link.next))->data[0]; +#else token = call->conn->key->payload.data[0]; +#endif memcpy(&iv, token->kad->session_key, sizeof(iv)); skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); @@ -820,7 +844,11 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, return rxrpc_abort_conn(conn, skb, RXKADLEVELFAIL, -EACCES, rxkad_abort_chall_level); +#ifdef CONFIG_KEYP + token = ((union key_payload *)(conn->key->name_link.next))->data[0]; +#else token = conn->key->payload.data[0]; +#endif /* build the response packet */ resp = kzalloc(sizeof(struct rxkad_response), GFP_NOFS); @@ -873,11 +901,25 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, *_expiry = 0; +#ifdef CONFIG_KEYP + ASSERT(((union key_payload *)(server_key->name_link.next))->data[0]); +#else ASSERT(server_key->payload.data[0] != NULL); +#endif + ASSERTCMP((unsigned long) ticket & 7UL, ==, 0); +#ifdef CONFIG_KEYP + memcpy(&iv, &((union key_payload *)(server_key->name_link.next))->data[2], sizeof(iv)); +#else memcpy(&iv, &server_key->payload.data[2], sizeof(iv)); +#endif +#ifdef CONFIG_KEYP + req = skcipher_request_alloc(((union key_payload *)(server_key->name_link.next))-> + data[0], GFP_NOFS); +#else req = skcipher_request_alloc(server_key->payload.data[0], GFP_NOFS); +#endif if (!req) return -ENOMEM; diff --git a/net/rxrpc/security.c b/net/rxrpc/security.c index 9784adc8f2759..e1be443dcf1e9 100644 --- a/net/rxrpc/security.c +++ b/net/rxrpc/security.c @@ -79,7 +79,12 @@ int rxrpc_init_client_call_security(struct rxrpc_call *call) if (ret < 0) return ret; +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif sec = rxrpc_security_lookup(token->security_index); if (sec) goto found; @@ -103,7 +108,12 @@ int rxrpc_init_client_conn_security(struct rxrpc_connection *conn) _enter("{%d},{%x}", conn->debug_id, key_serial(key)); +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif if (token->security_index == conn->security->security_index) goto found; } diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index 3dc54d6d4e316..c4f8886b15279 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -587,7 +587,11 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, } key = rx->key; +#ifdef CONFIG_KEYP + if (key && !((union key_payload *)rx->key->name_link.next)->data[0]) +#else if (key && !rx->key->payload.data[0]) +#endif key = NULL; memset(&cp, 0, sizeof(cp)); diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c index 9d95a6055b707..c1b0ccb921533 100644 --- a/net/rxrpc/server_key.c +++ b/net/rxrpc/server_key.c @@ -100,7 +100,11 @@ static void rxrpc_free_preparse_s(struct key_preparsed_payload *prep) static void rxrpc_destroy_s(struct key *key) { +#ifdef CONFIG_KEYP + const struct rxrpc_security *sec = ((union key_payload *)(key->name_link.next))->data[1]; +#else const struct rxrpc_security *sec = key->payload.data[1]; +#endif if (sec && sec->destroy_server_key) sec->destroy_server_key(key); @@ -108,7 +112,11 @@ static void rxrpc_destroy_s(struct key *key) static void rxrpc_describe_s(const struct key *key, struct seq_file *m) { +#ifdef CONFIG_KEYP + const struct rxrpc_security *sec = ((union key_payload *)(key->name_link.next))->data[1]; +#else const struct rxrpc_security *sec = key->payload.data[1]; +#endif seq_puts(m, key->description); if (sec && sec->describe_server_key) diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index b1ffd4cc0b448..4824249f21072 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -422,14 +422,23 @@ int evm_init_key(void) if (IS_ERR(evm_key)) return -ENOENT; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(evm_key)); + ekp = ((union key_payload *)(evm_key->name_link.next))->data[0]; +#else down_read(&evm_key->sem); ekp = evm_key->payload.data[0]; +#endif rc = evm_set_key(ekp->decrypted_data, ekp->decrypted_datalen); /* burn the original key contents */ memset(ekp->decrypted_data, 0, ekp->decrypted_datalen); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(evm_key)); +#else up_read(&evm_key->sem); +#endif key_put(evm_key); return rc; } diff --git a/security/keys/big_key.c b/security/keys/big_key.c index c3367622c683b..4a78d6171e012 100644 --- a/security/keys/big_key.c +++ b/security/keys/big_key.c @@ -164,7 +164,12 @@ void big_key_free_preparse(struct key_preparsed_payload *prep) */ void big_key_revoke(struct key *key) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif /* clear the quota */ key_payload_reserve(key, 0); @@ -177,7 +182,12 @@ void big_key_revoke(struct key *key) */ void big_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif if (payload->length > BIG_KEY_FILE_THRESHOLD) { path_put(&payload->path); @@ -210,7 +220,12 @@ int big_key_update(struct key *key, struct key_preparsed_payload *prep) */ void big_key_describe(const struct key *key, struct seq_file *m) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif seq_puts(m, key->description); @@ -226,7 +241,12 @@ void big_key_describe(const struct key *key, struct seq_file *m) */ long big_key_read(const struct key *key, char *buffer, size_t buflen) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif size_t datalen = payload->length; long ret; diff --git a/security/keys/dh.c b/security/keys/dh.c index da64c358474b2..8386a072903de 100644 --- a/security/keys/dh.c +++ b/security/keys/dh.c @@ -32,7 +32,11 @@ static ssize_t dh_data_from_key(key_serial_t keyid, const void **data) ret = -EOPNOTSUPP; if (key->type == &key_type_user) { +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif status = key_validate(key); if (status == 0) { const struct user_key_payload *payload; @@ -49,7 +53,11 @@ static ssize_t dh_data_from_key(key_serial_t keyid, const void **data) ret = -ENOMEM; } } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif } key_put(key); diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 1e313982af02a..354a9febbb6ea 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -314,11 +314,19 @@ static struct key *request_user_key(const char *master_desc, const u8 **master_k if (IS_ERR(ukey)) goto error; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(ukey)); +#else down_read(&ukey->sem); +#endif upayload = user_key_payload_locked(ukey); if (!upayload) { /* key was revoked before we acquired its semaphore */ +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(ukey)); +#else up_read(&ukey->sem); +#endif key_put(ukey); ukey = ERR_PTR(-EKEYREVOKED); goto error; @@ -729,7 +737,11 @@ static int encrypted_key_decrypt(struct encrypted_key_payload *epayload, if (ret < 0) pr_err("encrypted_key: failed to decrypt key (%d)\n", ret); out: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(mkey)); +#else up_read(&mkey->sem); +#endif key_put(mkey); memzero_explicit(derived_key, sizeof(derived_key)); return ret; @@ -874,7 +886,12 @@ static void encrypted_rcu_free(struct rcu_head *rcu) */ static int encrypted_update(struct key *key, struct key_preparsed_payload *prep) { +#ifdef CONFIG_KEYP + struct encrypted_key_payload *epayload = + ((union key_payload *)(key->name_link.next))->data[0]; +#else struct encrypted_key_payload *epayload = key->payload.data[0]; +#endif struct encrypted_key_payload *new_epayload; char *buf; char *new_master_desc = NULL; @@ -974,7 +991,11 @@ static long encrypted_read(const struct key *key, char *buffer, goto out; } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(mkey)); +#else up_read(&mkey->sem); +#endif key_put(mkey); memzero_explicit(derived_key, sizeof(derived_key)); @@ -983,7 +1004,11 @@ static long encrypted_read(const struct key *key, char *buffer, return asciiblob_len; out: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(mkey)); +#else up_read(&mkey->sem); +#endif key_put(mkey); memzero_explicit(derived_key, sizeof(derived_key)); return ret; @@ -994,7 +1019,11 @@ static long encrypted_read(const struct key *key, char *buffer, */ static void encrypted_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree_sensitive(key->payload.data[0]); +#endif } struct key_type key_type_encrypted = { diff --git a/security/keys/encrypted-keys/masterkey_trusted.c b/security/keys/encrypted-keys/masterkey_trusted.c index e6d22ce77e98f..d820a9f205c7e 100644 --- a/security/keys/encrypted-keys/masterkey_trusted.c +++ b/security/keys/encrypted-keys/masterkey_trusted.c @@ -34,8 +34,13 @@ struct key *request_trusted_key(const char *trusted_desc, if (IS_ERR(tkey)) goto error; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(tkey)); + tpayload = ((union key_payload *)(tkey->name_link.next))->data[0]; +#else down_read(&tkey->sem); tpayload = tkey->payload.data[0]; +#endif *master_key = tpayload->key; *master_keylen = tpayload->key_len; error: diff --git a/security/keys/gc.c b/security/keys/gc.c index eaddaceda14ea..a312e0c4b68ae 100644 --- a/security/keys/gc.c +++ b/security/keys/gc.c @@ -8,6 +8,10 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#include +#endif #include "internal.h" /* @@ -135,18 +139,31 @@ void key_gc_keytype(struct key_type *ktype) static noinline void key_gc_unused_keys(struct list_head *keys) { while (!list_empty(keys)) { +#ifdef CONFIG_KEYP + struct key *key = + list_entry(keys->next, struct key_union, graveyard_link)->key; +#else struct key *key = list_entry(keys->next, struct key, graveyard_link); +#endif short state = key->state; +#ifdef CONFIG_KEYP + list_del(&(((struct key_union *)(key->graveyard_link.next))->graveyard_link)); +#else list_del(&key->graveyard_link); +#endif kdebug("- %u", key->serial); key_check(key); #ifdef CONFIG_KEY_NOTIFICATIONS remove_watch_list(key->watchers, key->serial); +#ifdef CONFIG_KEYP + iee_set_key_watchers(key, NULL); +#else key->watchers = NULL; +#endif #endif /* Throw away the key data if the key is instantiated */ @@ -171,7 +188,15 @@ static noinline void key_gc_unused_keys(struct list_head *keys) key_put_tag(key->domain_tag); kfree(key->description); +#ifdef CONFIG_KEYP + kmem_cache_free(key_union_jar, (struct key_union *)(key->graveyard_link.next)); + kmem_cache_free(key_struct_jar, (struct key_struct *)(key->name_link.prev)); + kmem_cache_free(key_payload_jar, (union key_payload *)(key->name_link.next)); + iee_memset(key, 0, sizeof(*key)); + barrier_data(key); +#else memzero_explicit(key, sizeof(*key)); +#endif kmem_cache_free(key_jar, key); } } @@ -223,7 +248,11 @@ static void key_garbage_collector(struct work_struct *work) continue_scanning: while (cursor) { +#ifdef CONFIG_KEYP + key = rb_entry(cursor, struct key_union, serial_node)->key; +#else key = rb_entry(cursor, struct key, serial_node); +#endif cursor = rb_next(cursor); if (refcount_read(&key->usage) == 0) @@ -232,8 +261,13 @@ static void key_garbage_collector(struct work_struct *work) if (unlikely(gc_state & KEY_GC_REAPING_DEAD_1)) { if (key->type == key_gc_dead_keytype) { gc_state |= KEY_GC_FOUND_DEAD_KEY; +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(key, KEY_FLAG_DEAD, SET_BIT_OP); + iee_set_key_perm(key, 0); +#else set_bit(KEY_FLAG_DEAD, &key->flags); key->perm = 0; +#endif goto skip_dead_key; } else if (key->type == &key_type_keyring && key->restrict_link) { @@ -339,10 +373,20 @@ static void key_garbage_collector(struct work_struct *work) */ found_unreferenced_key: kdebug("unrefd key %d", key->serial); +#ifdef CONFIG_KEYP + rb_erase(&(((struct key_union *)(key->graveyard_link.next))->serial_node), + &key_serial_tree); +#else rb_erase(&key->serial_node, &key_serial_tree); +#endif spin_unlock(&key_serial_lock); +#ifdef CONFIG_KEYP + list_add_tail(&(((struct key_union *)(key->graveyard_link.next))->graveyard_link), + &graveyard); +#else list_add_tail(&key->graveyard_link, &graveyard); +#endif gc_state |= KEY_GC_REAP_AGAIN; goto maybe_resched; @@ -370,11 +414,21 @@ static void key_garbage_collector(struct work_struct *work) destroy_dead_key: spin_unlock(&key_serial_lock); kdebug("destroy key %d", key->serial); +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); + iee_set_key_type(key, &key_type_dead); +#else down_write(&key->sem); key->type = &key_type_dead; +#endif if (key_gc_dead_keytype->destroy) key_gc_dead_keytype->destroy(key); +#ifdef CONFIG_KEYP + iee_memset((key->name_link.next), KEY_DESTROY, sizeof(key->payload)); + up_write(&KEY_SEM(key)); +#else memset(&key->payload, KEY_DESTROY, sizeof(key->payload)); up_write(&key->sem); +#endif goto maybe_resched; } diff --git a/security/keys/internal.h b/security/keys/internal.h index b7b622bc36a13..fd3c583c3c39a 100644 --- a/security/keys/internal.h +++ b/security/keys/internal.h @@ -83,12 +83,20 @@ extern unsigned key_quota_maxbytes; extern struct kmem_cache *key_jar; +#ifdef CONFIG_KEYP +extern struct kmem_cache *key_union_jar; +extern struct kmem_cache *key_struct_jar; +extern struct kmem_cache *key_payload_jar; +#endif extern struct rb_root key_serial_tree; extern spinlock_t key_serial_lock; extern struct mutex key_construction_mutex; extern wait_queue_head_t request_key_conswq; extern void key_set_index_key(struct keyring_index_key *index_key); +#ifdef CONFIG_KEYP +extern void iee_key_set_index_key(struct keyring_index_key *index_key); +#endif extern struct key_type *key_type_lookup(const char *type); extern void key_type_put(struct key_type *ktype); diff --git a/security/keys/key.c b/security/keys/key.c index 35db23d05302e..0aab7c37636ea 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -15,9 +15,19 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#include +#include +#endif #include "internal.h" struct kmem_cache *key_jar; +#ifdef CONFIG_KEYP +struct kmem_cache *key_union_jar; +struct kmem_cache *key_struct_jar; +struct kmem_cache *key_payload_jar; +#endif struct rb_root key_serial_tree; /* tree of keys indexed by serial */ DEFINE_SPINLOCK(key_serial_lock); @@ -139,9 +149,17 @@ static inline void key_alloc_serial(struct key *key) /* propose a random serial number and look for a hole for it in the * serial number tree */ do { +#ifdef CONFIG_KEYP + key_serial_t tmp; + + get_random_bytes(&tmp, sizeof(key->serial)); + + iee_set_key_serial(key, tmp >> 1); +#else get_random_bytes(&key->serial, sizeof(key->serial)); key->serial >>= 1; /* negative numbers are not permitted */ +#endif } while (key->serial < 3); spin_lock(&key_serial_lock); @@ -152,7 +170,11 @@ static inline void key_alloc_serial(struct key *key) while (*p) { parent = *p; +#ifdef CONFIG_KEYP + xkey = rb_entry(parent, struct key_union, serial_node)->key; +#else xkey = rb_entry(parent, struct key, serial_node); +#endif if (key->serial < xkey->serial) p = &(*p)->rb_left; @@ -163,8 +185,14 @@ static inline void key_alloc_serial(struct key *key) } /* we've found a suitable hole - arrange for this key to occupy it */ +#ifdef CONFIG_KEYP + rb_link_node(&(((struct key_union *)(key->graveyard_link.next))->serial_node), parent, p); + rb_insert_color(&(((struct key_union *)(key->graveyard_link.next))->serial_node), + &key_serial_tree); +#else rb_link_node(&key->serial_node, parent, p); rb_insert_color(&key->serial_node, &key_serial_tree); +#endif spin_unlock(&key_serial_lock); return; @@ -173,9 +201,19 @@ static inline void key_alloc_serial(struct key *key) * that point looking for the next unused serial number */ serial_exists: for (;;) { +#ifdef CONFIG_KEYP + key_serial_t tmp = key->serial + 1; + + iee_set_key_serial(key, tmp); +#else key->serial++; +#endif if (key->serial < 3) { +#ifdef CONFIG_KEYP + iee_set_key_serial(key, 3); +#else key->serial = 3; +#endif goto attempt_insertion; } @@ -183,7 +221,11 @@ static inline void key_alloc_serial(struct key *key) if (!parent) goto attempt_insertion; +#ifdef CONFIG_KEYP + xkey = rb_entry(parent, struct key_union, serial_node)->key; +#else xkey = rb_entry(parent, struct key, serial_node); +#endif if (key->serial < xkey->serial) goto attempt_insertion; } @@ -231,6 +273,9 @@ struct key *key_alloc(struct key_type *type, const char *desc, struct key *key; size_t desclen, quotalen; int ret; +#ifdef CONFIG_KEYP + unsigned long kflags; +#endif key = ERR_PTR(-EINVAL); if (!desc || !*desc) @@ -274,17 +319,80 @@ struct key *key_alloc(struct key_type *type, const char *desc, } /* allocate and initialise the key and its description */ +#ifdef CONFIG_KEYP + key = kmem_cache_alloc(key_jar, GFP_KERNEL); +#else key = kmem_cache_zalloc(key_jar, GFP_KERNEL); +#endif if (!key) goto no_memory_2; +#ifdef CONFIG_KEYP + struct key_union *key_union = kmem_cache_zalloc(key_union_jar, GFP_KERNEL); + struct key_struct *key_struct = kmem_cache_zalloc(key_struct_jar, GFP_KERNEL); + union key_payload *key_payload = kmem_cache_alloc(key_payload_jar, GFP_KERNEL); + + if (!key_union || !key_struct || !key_payload) + goto no_memory_3; + + key_union->key = key; + key_struct->key = key; + iee_set_key_union(key, key_union); + iee_set_key_struct(key, key_struct); + iee_set_key_payload(key, key_payload); +#endif + +#ifdef CONFIG_KEYP + struct keyring_index_key tmp = key->index_key; + + tmp.desc_len = desclen; + tmp.description = kmemdup(desc, desclen + 1, GFP_KERNEL); + iee_set_key_index_key(key, &tmp); +#else key->index_key.desc_len = desclen; key->index_key.description = kmemdup(desc, desclen + 1, GFP_KERNEL); +#endif if (!key->index_key.description) goto no_memory_3; +#ifdef CONFIG_KEYP + tmp = key->index_key; + tmp.type = type; + iee_set_key_index_key(key, &tmp); + iee_key_set_index_key(&key->index_key); +#else key->index_key.type = type; key_set_index_key(&key->index_key); +#endif +#ifdef CONFIG_KEYP + iee_set_key_usage(key, 1, REFCOUNT_SET); + init_rwsem(&KEY_SEM(key)); + lockdep_set_class(&KEY_SEM(key), &type->lock_class); + iee_set_key_user(key, user); + iee_set_key_quotalen(key, quotalen); + iee_set_key_datalen(key, type->def_datalen); + iee_set_key_uid(key, uid); + iee_set_key_gid(key, gid); + iee_set_key_perm(key, perm); + iee_set_key_expiry(key, TIME64_MAX); + iee_set_key_restrict_link(key, restrict_link); + iee_set_key_last_used_at(key, ktime_get_real_seconds()); + + kflags = key->flags; + if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) + kflags |= 1 << KEY_FLAG_IN_QUOTA; + if (flags & KEY_ALLOC_BUILT_IN) + kflags |= 1 << KEY_FLAG_BUILTIN; + if (flags & KEY_ALLOC_UID_KEYRING) + kflags |= 1 << KEY_FLAG_UID_KEYRING; + if (flags & KEY_ALLOC_SET_KEEP) + kflags |= 1 << KEY_FLAG_KEEP; + iee_set_key_flags(key, kflags); + +#ifdef KEY_DEBUGGING + iee_set_key_magic(key, KEY_DEBUG_MAGIC); +#endif +#else refcount_set(&key->usage, 1); init_rwsem(&key->sem); lockdep_set_class(&key->sem, &type->lock_class); @@ -309,6 +417,7 @@ struct key *key_alloc(struct key_type *type, const char *desc, #ifdef KEY_DEBUGGING key->magic = KEY_DEBUG_MAGIC; +#endif #endif /* let the security module know about the key */ @@ -326,6 +435,15 @@ struct key *key_alloc(struct key_type *type, const char *desc, security_error: kfree(key->description); +#ifdef CONFIG_KEYP + if (key_union) + kmem_cache_free(key_union_jar, key_union); + if (key_struct) + kmem_cache_free(key_struct_jar, key_struct); + if (key_payload) + kmem_cache_free(key_payload_jar, key_payload); + iee_memset(key, 0, sizeof(struct key)); +#endif kmem_cache_free(key_jar, key); if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) { spin_lock(&user->lock); @@ -338,6 +456,15 @@ struct key *key_alloc(struct key_type *type, const char *desc, goto error; no_memory_3: +#ifdef CONFIG_KEYP + if (key_union) + kmem_cache_free(key_union_jar, key_union); + if (key_struct) + kmem_cache_free(key_struct_jar, key_struct); + if (key_payload) + kmem_cache_free(key_payload_jar, key_payload); + iee_memset(key, 0, sizeof(struct key)); +#endif kmem_cache_free(key_jar, key); no_memory_2: if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) { @@ -391,14 +518,22 @@ int key_payload_reserve(struct key *key, size_t datalen) } else { key->user->qnbytes += delta; +#ifdef CONFIG_KEYP + iee_set_key_quotalen(key, key->quotalen + delta); +#else key->quotalen += delta; +#endif } spin_unlock(&key->user->lock); } /* change the recorded data length if that didn't generate an error */ if (ret == 0) +#ifdef CONFIG_KEYP + iee_set_key_datalen(key, datalen); +#else key->datalen = datalen; +#endif return ret; } @@ -412,8 +547,14 @@ static void mark_key_instantiated(struct key *key, int reject_error) /* Commit the payload before setting the state; barrier versus * key_read_state(). */ +#ifdef CONFIG_KEYP + compiletime_assert_atomic_type(key->state); + barrier(); + iee_set_key_state(key, (reject_error < 0) ? reject_error : KEY_IS_POSITIVE); +#else smp_store_release(&key->state, (reject_error < 0) ? reject_error : KEY_IS_POSITIVE); +#endif } /* @@ -449,13 +590,22 @@ static int __key_instantiate_and_link(struct key *key, mark_key_instantiated(key, 0); notify_key(key, NOTIFY_KEY_INSTANTIATED, 0); +#ifdef CONFIG_KEYP + if (iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, TEST_AND_CLEAR_BIT)) + awaken = 1; +#else if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) awaken = 1; +#endif /* and link it into the destination keyring */ if (keyring) { if (test_bit(KEY_FLAG_KEEP, &keyring->flags)) +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(key, KEY_FLAG_KEEP, SET_BIT_OP); +#else set_bit(KEY_FLAG_KEEP, &key->flags); +#endif __key_link(keyring, key, _edit); } @@ -465,7 +615,11 @@ static int __key_instantiate_and_link(struct key *key, key_invalidate(authkey); if (prep->expiry != TIME64_MAX) +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, prep->expiry); +#else key_set_expiry(key, prep->expiry); +#endif } } @@ -605,10 +759,19 @@ int key_reject_and_link(struct key *key, atomic_inc(&key->user->nikeys); mark_key_instantiated(key, -error); notify_key(key, NOTIFY_KEY_INSTANTIATED, -error); +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, ktime_get_real_seconds() + timeout); +#else key_set_expiry(key, ktime_get_real_seconds() + timeout); +#endif +#ifdef CONFIG_KEYP + if (iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, TEST_AND_CLEAR_BIT)) + awaken = 1; +#else if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) awaken = 1; +#endif ret = 0; @@ -647,8 +810,13 @@ void key_put(struct key *key) if (key) { key_check(key); +#ifdef CONFIG_KEYP + if (iee_set_key_usage(key, 0, REFCOUNT_DEC_AND_TEST)) + schedule_work(&key_gc_work); +#else if (refcount_dec_and_test(&key->usage)) schedule_work(&key_gc_work); +#endif } } EXPORT_SYMBOL(key_put); @@ -666,7 +834,11 @@ struct key *key_lookup(key_serial_t id) /* search the tree for the specified key */ n = key_serial_tree.rb_node; while (n) { +#ifdef CONFIG_KEYP + key = rb_entry(n, struct key_union, serial_node)->key; +#else key = rb_entry(n, struct key, serial_node); +#endif if (id < key->serial) n = n->rb_left; @@ -684,8 +856,13 @@ struct key *key_lookup(key_serial_t id) /* A key is allowed to be looked up only if someone still owns a * reference to it - otherwise it's awaiting the gc. */ +#ifdef CONFIG_KEYP + if (!iee_set_key_usage(key, 0, REFCOUNT_INC_NOT_ZERO)) + goto not_found; +#else if (!refcount_inc_not_zero(&key->usage)) goto not_found; +#endif error: spin_unlock(&key_serial_lock); @@ -723,13 +900,25 @@ void key_set_timeout(struct key *key, unsigned timeout) time64_t expiry = TIME64_MAX; /* make the changes with the locks held to prevent races */ +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif if (timeout > 0) expiry = ktime_get_real_seconds() + timeout; +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, expiry); +#else key_set_expiry(key, expiry); +#endif +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif } EXPORT_SYMBOL_GPL(key_set_timeout); @@ -762,7 +951,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, if (!key->type->update) goto error; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif ret = key->type->update(key, prep); if (ret == 0) { @@ -771,7 +964,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, notify_key(key, NOTIFY_KEY_UPDATED, 0); } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif if (ret < 0) goto error; @@ -1087,7 +1284,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) goto error; } +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif ret = key->type->update(key, &prep); if (ret == 0) { @@ -1096,7 +1297,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) notify_key(key, NOTIFY_KEY_UPDATED, 0); } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif error: if (key->type->preparse) @@ -1125,6 +1330,23 @@ void key_revoke(struct key *key) * authorisation key whilst holding the sem on a key we've just * instantiated */ +#ifdef CONFIG_KEYP + down_write_nested(&KEY_SEM(key), 1); + if (!iee_set_key_flag_bit(key, KEY_FLAG_REVOKED, TEST_AND_SET_BIT)) { + notify_key(key, NOTIFY_KEY_REVOKED, 0); + if (key->type->revoke) + key->type->revoke(key); + + /* set the death time to no more than the expiry time */ + time = ktime_get_real_seconds(); + if (key->revoked_at == 0 || key->revoked_at > time) { + iee_set_key_revoked_at(key, time); + key_schedule_gc(key->revoked_at + key_gc_delay); + } + } + + up_write(&KEY_SEM(key)); +#else down_write_nested(&key->sem, 1); if (!test_and_set_bit(KEY_FLAG_REVOKED, &key->flags)) { notify_key(key, NOTIFY_KEY_REVOKED, 0); @@ -1140,6 +1362,7 @@ void key_revoke(struct key *key) } up_write(&key->sem); +#endif } EXPORT_SYMBOL(key_revoke); @@ -1157,12 +1380,21 @@ void key_invalidate(struct key *key) key_check(key); if (!test_bit(KEY_FLAG_INVALIDATED, &key->flags)) { +#ifdef CONFIG_KEYP + down_write_nested(&KEY_SEM(key), 1); + if (!iee_set_key_flag_bit(key, KEY_FLAG_INVALIDATED, TEST_AND_SET_BIT)) { + notify_key(key, NOTIFY_KEY_INVALIDATED, 0); + key_schedule_gc_links(); + } + up_write(&KEY_SEM(key)); +#else down_write_nested(&key->sem, 1); if (!test_and_set_bit(KEY_FLAG_INVALIDATED, &key->flags)) { notify_key(key, NOTIFY_KEY_INVALIDATED, 0); key_schedule_gc_links(); } up_write(&key->sem); +#endif } } EXPORT_SYMBOL(key_invalidate); @@ -1186,9 +1418,17 @@ int generic_key_instantiate(struct key *key, struct key_preparsed_payload *prep) ret = key_payload_reserve(key, prep->quotalen); if (ret == 0) { rcu_assign_keypointer(key, prep->payload.data[0]); +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + key_payload->data[1] = prep->payload.data[1]; + key_payload->data[2] = prep->payload.data[2]; + key_payload->data[3] = prep->payload.data[3]; +#else key->payload.data[1] = prep->payload.data[1]; key->payload.data[2] = prep->payload.data[2]; key->payload.data[3] = prep->payload.data[3]; +#endif prep->payload.data[0] = NULL; prep->payload.data[1] = NULL; prep->payload.data[2] = NULL; @@ -1262,6 +1502,16 @@ void __init key_init(void) /* allocate a slab in which we can store keys */ key_jar = kmem_cache_create("key_jar", sizeof(struct key), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); +#ifdef CONFIG_KEYP + key_union_jar = kmem_cache_create("key_union_jar", sizeof(struct key_union), + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + key_struct_jar = kmem_cache_create("key_struct_jar", sizeof(struct key_struct), + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + key_payload_jar = kmem_cache_create("key_payload_jar", sizeof(union key_payload)*4, + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + if (haoc_enabled) + pr_info("HAOC: CONFIG_KEYP enabled."); + #endif /* add the special key types */ list_add_tail(&key_type_keyring.link, &key_types_list); diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index 225129a881822..1c7dc16ec8a4f 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -25,6 +25,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" #define KEY_MAX_DESC_SIZE 4096 @@ -807,11 +810,19 @@ static long __keyctl_read_key(struct key *key, char *buffer, size_t buflen) { long ret; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = key_validate(key); if (ret == 0) ret = key->type->read(key, buffer, buflen); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif return ret; } @@ -981,7 +992,11 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) /* make the changes with the locks held to prevent chown/chown races */ ret = -EACCES; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif { bool is_privileged_op = false; @@ -1039,19 +1054,32 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) } zapowner = key->user; +#ifdef CONFIG_KEYP + iee_set_key_user(key, newowner); + iee_set_key_uid(key, uid); +#else key->user = newowner; key->uid = uid; +#endif } /* change the GID */ if (group != (gid_t) -1) +#ifdef CONFIG_KEYP + iee_set_key_gid(key, gid); +#else key->gid = gid; +#endif notify_key(key, NOTIFY_KEY_SETATTR, 0); ret = 0; error_put: +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif key_put(key); if (zapowner) key_user_put(zapowner); @@ -1093,16 +1121,28 @@ long keyctl_setperm_key(key_serial_t id, key_perm_t perm) /* make the changes with the locks held to prevent chown/chmod races */ ret = -EACCES; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif /* if we're not the sysadmin, we can only change a key that we own */ if (uid_eq(key->uid, current_fsuid()) || capable(CAP_SYS_ADMIN)) { +#ifdef CONFIG_KEYP + iee_set_key_perm(key, perm); +#else key->perm = perm; +#endif notify_key(key, NOTIFY_KEY_SETATTR, 0); ret = 0; } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif key_put(key); error: return ret; @@ -1856,25 +1896,45 @@ long keyctl_watch_key(key_serial_t id, int watch_queue_fd, int watch_id) if (ret < 0) goto err_watch; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif if (!key->watchers) { +#ifdef CONFIG_KEYP + iee_set_key_watchers(key, wlist); +#else key->watchers = wlist; +#endif wlist = NULL; } ret = add_watch_to_object(watch, key->watchers); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif if (ret == 0) watch = NULL; } else { ret = -EBADSLT; if (key->watchers) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif ret = remove_watch_from_object(key->watchers, wqueue, key_serial(key), false); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif } } diff --git a/security/keys/keyring.c b/security/keys/keyring.c index e105349794f23..0dbc1a81d6f83 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -19,6 +19,10 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#include +#endif #include "internal.h" /* @@ -112,7 +116,12 @@ static void keyring_publish_name(struct key *keyring) keyring->description[0] && keyring->description[0] != '.') { write_lock(&keyring_name_lock); +#ifdef CONFIG_KEYP + list_add_tail(&(((struct key_struct *)(keyring->name_link.prev))->name_link), + &ns->keyring_name_list); +#else list_add_tail(&keyring->name_link, &ns->keyring_name_list); +#endif write_unlock(&keyring_name_lock); } } @@ -140,7 +149,11 @@ static void keyring_free_preparse(struct key_preparsed_payload *prep) static int keyring_instantiate(struct key *keyring, struct key_preparsed_payload *prep) { +#ifdef CONFIG_KEYP + assoc_array_init(&((struct key_struct *)(keyring->name_link.prev))->keys); +#else assoc_array_init(&keyring->keys); +#endif /* make the keyring available by name if it has one */ keyring_publish_name(keyring); return 0; @@ -207,13 +220,91 @@ static void hash_key_type_and_desc(struct keyring_index_key *index_key) index_key->hash = hash; } +#ifdef CONFIG_KEYP +static void iee_hash_key_type_and_desc(struct keyring_index_key *index_key) +{ + const unsigned int level_shift = ASSOC_ARRAY_LEVEL_STEP; + const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK; + const char *description = index_key->description; + unsigned long hash, type; + u32 piece; + u64 acc; + int n, desc_len = index_key->desc_len; + + type = (unsigned long)index_key->type; + acc = mult_64x32_and_fold(type, desc_len + 13); + acc = mult_64x32_and_fold(acc, 9207); + piece = (unsigned long)index_key->domain_tag; + acc = mult_64x32_and_fold(acc, piece); + acc = mult_64x32_and_fold(acc, 9207); + + for (;;) { + n = desc_len; + if (n <= 0) + break; + if (n > 4) + n = 4; + piece = 0; + memcpy(&piece, description, n); + description += n; + desc_len -= n; + acc = mult_64x32_and_fold(acc, piece); + acc = mult_64x32_and_fold(acc, 9207); + } + + /* Fold the hash down to 32 bits if need be. */ + hash = acc; + if (ASSOC_ARRAY_KEY_CHUNK_SIZE == 32) + hash ^= acc >> 32; + + /* Squidge all the keyrings into a separate part of the tree to + * ordinary keys by making sure the lowest level segment in the hash is + * zero for keyrings and non-zero otherwise. + */ + if (index_key->type != &key_type_keyring && (hash & fan_mask) == 0) + hash |= (hash >> (ASSOC_ARRAY_KEY_CHUNK_SIZE - level_shift)) | 1; + else if (index_key->type == &key_type_keyring && (hash & fan_mask) != 0) + hash = (hash + (hash << level_shift)) & ~fan_mask; + struct keyring_index_key tmp = *index_key; + + tmp.hash = hash; + iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); +} + +static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), }; + +void iee_key_set_index_key(struct keyring_index_key *index_key) +{ + size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc)); + struct keyring_index_key tmp; + + iee_memcpy(index_key->desc, index_key->description, n); + + if (!index_key->domain_tag) { + if (index_key->type->flags & KEY_TYPE_NET_DOMAIN) { + tmp = *index_key; + tmp.domain_tag = current->nsproxy->net_ns->key_domain; + iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); + } else { + tmp = *index_key; + tmp.domain_tag = &default_domain_tag; + iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); + } + } + + iee_hash_key_type_and_desc(index_key); +} +#endif + /* * Finalise an index key to include a part of the description actually in the * index key, to set the domain tag and to calculate the hash. */ void key_set_index_key(struct keyring_index_key *index_key) { +#ifndef CONFIG_KEYP static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), }; +#endif size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc)); memcpy(index_key->desc, index_key->description, n); @@ -414,9 +505,15 @@ static void keyring_destroy(struct key *keyring) if (keyring->description) { write_lock(&keyring_name_lock); +#ifdef CONFIG_KEYP + if (((struct key_struct *)(keyring->name_link.prev))->name_link.next != NULL && + !list_empty(&(((struct key_struct *)(keyring->name_link.prev))->name_link))) + list_del(&(((struct key_struct *)(keyring->name_link.prev))->name_link)); +#else if (keyring->name_link.next != NULL && !list_empty(&keyring->name_link)) list_del(&keyring->name_link); +#endif write_unlock(&keyring_name_lock); } @@ -428,7 +525,12 @@ static void keyring_destroy(struct key *keyring) kfree(keyres); } +#ifdef CONFIG_KEYP + assoc_array_destroy(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops); +#else assoc_array_destroy(&keyring->keys, &keyring_assoc_array_ops); +#endif } /* @@ -442,8 +544,14 @@ static void keyring_describe(const struct key *keyring, struct seq_file *m) seq_puts(m, "[anon]"); if (key_is_positive(keyring)) { +#ifdef CONFIG_KEYP + if (((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree != 0) + seq_printf(m, ": %lu", ((struct key_struct *)(keyring->name_link.prev))-> + keys.nr_leaves_on_tree); +#else if (keyring->keys.nr_leaves_on_tree != 0) seq_printf(m, ": %lu", keyring->keys.nr_leaves_on_tree); +#endif else seq_puts(m, ": empty"); } @@ -494,8 +602,13 @@ static long keyring_read(const struct key *keyring, ctx.buffer = (key_serial_t *)buffer; ctx.buflen = buflen; ctx.count = 0; +#ifdef CONFIG_KEYP + ret = assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, + keyring_read_iterator, &ctx); +#else ret = assoc_array_iterate(&keyring->keys, keyring_read_iterator, &ctx); +#endif if (ret < 0) { kleave(" = %ld [iterate]", ret); return ret; @@ -503,7 +616,12 @@ static long keyring_read(const struct key *keyring, } /* Return the size of the buffer needed */ +#ifdef CONFIG_KEYP + ret = ((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree * + sizeof(key_serial_t); +#else ret = keyring->keys.nr_leaves_on_tree * sizeof(key_serial_t); +#endif if (ret <= buflen) kleave("= %ld [ok]", ret); else @@ -648,12 +766,23 @@ static int search_keyring(struct key *keyring, struct keyring_search_context *ct if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_DIRECT) { const void *object; +#ifdef CONFIG_KEYP + object = assoc_array_find(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, + &ctx->index_key); +#else object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops, &ctx->index_key); +#endif return object ? ctx->iterator(object, ctx) : 0; } +#ifdef CONFIG_KEYP + return assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, + ctx->iterator, ctx); +#else return assoc_array_iterate(&keyring->keys, ctx->iterator, ctx); +#endif } /* @@ -729,7 +858,11 @@ static bool search_nested_keyrings(struct key *keyring, if (!(ctx->flags & KEYRING_SEARCH_RECURSE)) goto not_this_keyring; +#ifdef CONFIG_KEYP + ptr = READ_ONCE(((struct key_struct *)(keyring->name_link.prev))->keys.root); +#else ptr = READ_ONCE(keyring->keys.root); +#endif if (!ptr) goto not_this_keyring; @@ -856,10 +989,17 @@ static bool search_nested_keyrings(struct key *keyring, key = key_ref_to_ptr(ctx->result); key_check(key); if (!(ctx->flags & KEYRING_SEARCH_NO_UPDATE_TIME)) { +#ifdef CONFIG_KEYP + iee_set_key_last_used_at(key, ctx->now); + iee_set_key_last_used_at(keyring, ctx->now); + while (sp > 0) + iee_set_key_last_used_at(stack[--sp].keyring, ctx->now); +#else key->last_used_at = ctx->now; keyring->last_used_at = ctx->now; while (sp > 0) stack[--sp].keyring->last_used_at = ctx->now; +#endif } kleave(" = true"); return true; @@ -1056,7 +1196,11 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, goto error; } +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif down_write(&keyring_serialise_restrict_sem); if (keyring->restrict_link) { @@ -1064,12 +1208,20 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, } else if (keyring_detect_restriction_cycle(keyring, restrict_link)) { ret = -EDEADLK; } else { +#ifdef CONFIG_KEYP + iee_set_key_restrict_link(keyring, restrict_link); +#else keyring->restrict_link = restrict_link; +#endif notify_key(keyring, NOTIFY_KEY_SETATTR, 0); } up_write(&keyring_serialise_restrict_sem); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif if (ret < 0) { key_put(restrict_link->key); @@ -1110,8 +1262,13 @@ key_ref_t find_key_to_update(key_ref_t keyring_ref, keyring->serial, index_key->type->name, index_key->description); guard(rcu)(); +#ifdef CONFIG_KEYP + object = assoc_array_find(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, index_key); +#else object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops, index_key); +#endif if (object) goto found; @@ -1155,7 +1312,14 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) /* Search this hash bucket for a keyring with a matching name that * grants Search permission and that hasn't been revoked */ +#ifdef CONFIG_KEYP + struct key_struct *key_struct; + + list_for_each_entry(key_struct, &ns->keyring_name_list, name_link) { + keyring = key_struct->key; +#else list_for_each_entry(keyring, &ns->keyring_name_list, name_link) { +#endif if (!kuid_has_mapping(ns, keyring->user->uid)) continue; @@ -1178,9 +1342,15 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) /* we've got a match but we might end up racing with * key_cleanup() if the keyring is currently 'dead' * (ie. it has a zero usage count) */ +#ifdef CONFIG_KEYP + if (!iee_set_key_usage(keyring, 0, REFCOUNT_INC_NOT_ZERO)) + continue; + iee_set_key_last_used_at(keyring, ktime_get_real_seconds()); +#else if (!refcount_inc_not_zero(&keyring->usage)) continue; keyring->last_used_at = ktime_get_real_seconds(); +#endif goto out; } @@ -1239,13 +1409,21 @@ static int keyring_detect_cycle(struct key *A, struct key *B) */ int __key_link_lock(struct key *keyring, const struct keyring_index_key *index_key) +#ifdef CONFIG_KEYP + __acquires(&KEY_SEM(keyring)) +#else __acquires(&keyring->sem) +#endif __acquires(&keyring_serialise_link_lock) { if (keyring->type != &key_type_keyring) return -ENOTDIR; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif /* Serialise link/link calls to prevent parallel calls causing a cycle * when linking two keyring in opposite orders. @@ -1261,8 +1439,13 @@ int __key_link_lock(struct key *keyring, */ int __key_move_lock(struct key *l_keyring, struct key *u_keyring, const struct keyring_index_key *index_key) +#ifdef CONFIG_KEYP + __acquires(&KEY_SEM(l_keyring)) + __acquires(&KEY_SEM(u_keyring)) +#else __acquires(&l_keyring->sem) __acquires(&u_keyring->sem) +#endif __acquires(&keyring_serialise_link_lock) { if (l_keyring->type != &key_type_keyring || @@ -1274,11 +1457,21 @@ int __key_move_lock(struct key *l_keyring, struct key *u_keyring, * move operation. */ if (l_keyring < u_keyring) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(l_keyring)); + down_write_nested(&KEY_SEM(u_keyring), 1); +#else down_write(&l_keyring->sem); down_write_nested(&u_keyring->sem, 1); +#endif } else { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(u_keyring)); + down_write_nested(&KEY_SEM(l_keyring), 1); +#else down_write(&u_keyring->sem); down_write_nested(&l_keyring->sem, 1); +#endif } /* Serialise link/link calls to prevent parallel calls causing a cycle @@ -1315,10 +1508,17 @@ int __key_link_begin(struct key *keyring, /* Create an edit script that will insert/replace the key in the * keyring tree. */ +#ifdef CONFIG_KEYP + edit = assoc_array_insert(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, + index_key, + NULL); +#else edit = assoc_array_insert(&keyring->keys, &keyring_assoc_array_ops, index_key, NULL); +#endif if (IS_ERR(edit)) { ret = PTR_ERR(edit); goto error; @@ -1386,7 +1586,11 @@ void __key_link(struct key *keyring, struct key *key, void __key_link_end(struct key *keyring, const struct keyring_index_key *index_key, struct assoc_array_edit *edit) +#ifdef CONFIG_KEYP + __releases(&KEY_SEM(keyring)) +#else __releases(&keyring->sem) +#endif __releases(&keyring_serialise_link_lock) { BUG_ON(index_key->type == NULL); @@ -1399,7 +1603,11 @@ void __key_link_end(struct key *keyring, } assoc_array_cancel_edit(edit); } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif if (index_key->type == &key_type_keyring) mutex_unlock(&keyring_serialise_link_lock); @@ -1412,8 +1620,13 @@ static int __key_link_check_restriction(struct key *keyring, struct key *key) { if (!keyring->restrict_link || !keyring->restrict_link->check) return 0; +#ifdef CONFIG_KEYP + return keyring->restrict_link->check(keyring, key->type, + ((union key_payload *)(key->name_link.next)), keyring->restrict_link->key); +#else return keyring->restrict_link->check(keyring, key->type, &key->payload, keyring->restrict_link->key); +#endif } /** @@ -1473,12 +1686,20 @@ EXPORT_SYMBOL(key_link); * Lock a keyring for unlink. */ static int __key_unlink_lock(struct key *keyring) +#ifdef CONFIG_KEYP + __acquires(&KEY_SEM(keyring)) +#else __acquires(&keyring->sem) +#endif { if (keyring->type != &key_type_keyring) return -ENOTDIR; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif return 0; } @@ -1492,8 +1713,13 @@ static int __key_unlink_begin(struct key *keyring, struct key *key, BUG_ON(*_edit != NULL); +#ifdef CONFIG_KEYP + edit = assoc_array_delete(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, &key->index_key); +#else edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops, &key->index_key); +#endif if (IS_ERR(edit)) return PTR_ERR(edit); @@ -1522,11 +1748,19 @@ static void __key_unlink(struct key *keyring, struct key *key, static void __key_unlink_end(struct key *keyring, struct key *key, struct assoc_array_edit *edit) +#ifdef CONFIG_KEYP + __releases(&KEY_SEM(keyring)) +#else __releases(&keyring->sem) +#endif { if (edit) assoc_array_cancel_edit(edit); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif } /** @@ -1656,9 +1890,16 @@ int keyring_clear(struct key *keyring) if (keyring->type != &key_type_keyring) return -ENOTDIR; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); + + edit = assoc_array_clear(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops); +#else down_write(&keyring->sem); edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops); +#endif if (IS_ERR(edit)) { ret = PTR_ERR(edit); } else { @@ -1669,7 +1910,11 @@ int keyring_clear(struct key *keyring) ret = 0; } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif return ret; } EXPORT_SYMBOL(keyring_clear); @@ -1683,7 +1928,12 @@ static void keyring_revoke(struct key *keyring) { struct assoc_array_edit *edit; +#ifdef CONFIG_KEYP + edit = assoc_array_clear(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops); +#else edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops); +#endif if (!IS_ERR(edit)) { if (edit) assoc_array_apply_edit(edit); @@ -1729,8 +1979,13 @@ void keyring_gc(struct key *keyring, time64_t limit) /* scan the keyring looking for dead keys */ rcu_read_lock(); +#ifdef CONFIG_KEYP + result = assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, + keyring_gc_check_iterator, &limit); +#else result = assoc_array_iterate(&keyring->keys, keyring_gc_check_iterator, &limit); +#endif rcu_read_unlock(); if (result == true) goto do_gc; @@ -1740,10 +1995,17 @@ void keyring_gc(struct key *keyring, time64_t limit) return; do_gc: +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); + assoc_array_gc(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, keyring_gc_select_iterator, &limit); + up_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); assoc_array_gc(&keyring->keys, &keyring_assoc_array_ops, keyring_gc_select_iterator, &limit); up_write(&keyring->sem); +#endif kleave(" [gc]"); } @@ -1782,7 +2044,11 @@ void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type) } /* Lock the keyring to ensure that a link is not in progress */ +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif keyres = keyring->restrict_link; @@ -1792,7 +2058,11 @@ void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type) keyres->key = NULL; keyres->keytype = NULL; +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif kleave(" [restriction gc]"); } diff --git a/security/keys/proc.c b/security/keys/proc.c index 4f4e2c1824f18..e7f08b7d877ea 100644 --- a/security/keys/proc.c +++ b/security/keys/proc.c @@ -67,7 +67,11 @@ static struct rb_node *key_serial_next(struct seq_file *p, struct rb_node *n) n = rb_next(n); while (n) { +#ifdef CONFIG_KEYP + struct key *key = rb_entry(n, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(n, struct key, serial_node); +#endif if (kuid_has_mapping(user_ns, key->user->uid)) break; n = rb_next(n); @@ -82,7 +86,11 @@ static struct key *find_ge_key(struct seq_file *p, key_serial_t id) struct key *minkey = NULL; while (n) { +#ifdef CONFIG_KEYP + struct key *key = rb_entry(n, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(n, struct key, serial_node); +#endif if (id < key->serial) { if (!minkey || minkey->serial > key->serial) minkey = key; @@ -102,10 +110,18 @@ static struct key *find_ge_key(struct seq_file *p, key_serial_t id) for (;;) { if (kuid_has_mapping(user_ns, minkey->user->uid)) return minkey; +#ifdef CONFIG_KEYP + n = rb_next(&(((struct key_union *)(minkey->graveyard_link.next))->serial_node)); +#else n = rb_next(&minkey->serial_node); +#endif if (!n) return NULL; +#ifdef CONFIG_KEYP + minkey = rb_entry(n, struct key_union, serial_node)->key; +#else minkey = rb_entry(n, struct key, serial_node); +#endif } } @@ -123,12 +139,20 @@ static void *proc_keys_start(struct seq_file *p, loff_t *_pos) if (!key) return NULL; *_pos = key->serial; +#ifdef CONFIG_KEYP + return &(((struct key_union *)(key->graveyard_link.next))->serial_node); +#else return &key->serial_node; +#endif } static inline key_serial_t key_node_serial(struct rb_node *n) { +#ifdef CONFIG_KEYP + struct key *key = rb_entry(n, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(n, struct key, serial_node); +#endif return key->serial; } @@ -153,7 +177,11 @@ static void proc_keys_stop(struct seq_file *p, void *v) static int proc_keys_show(struct seq_file *m, void *v) { struct rb_node *_p = v; +#ifdef CONFIG_KEYP + struct key *key = rb_entry(_p, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(_p, struct key, serial_node); +#endif unsigned long flags; key_ref_t key_ref, skey_ref; time64_t now, expiry; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index 7494e32c747b9..067c225a466a5 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -20,6 +20,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" /* Session keyring create vs join semaphore */ @@ -393,9 +396,15 @@ void key_fsuid_changed(struct cred *new_cred) { /* update the ownership of the thread keyring */ if (new_cred->thread_keyring) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(new_cred->thread_keyring)); + iee_set_key_uid(new_cred->thread_keyring, new_cred->fsuid); + up_write(&KEY_SEM(new_cred->thread_keyring)); +#else down_write(&new_cred->thread_keyring->sem); new_cred->thread_keyring->uid = new_cred->fsuid; up_write(&new_cred->thread_keyring->sem); +#endif } } @@ -406,9 +415,15 @@ void key_fsgid_changed(struct cred *new_cred) { /* update the ownership of the thread keyring */ if (new_cred->thread_keyring) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(new_cred->thread_keyring)); + iee_set_key_gid(new_cred->thread_keyring, new_cred->fsgid); + up_write(&KEY_SEM(new_cred->thread_keyring)); +#else down_write(&new_cred->thread_keyring->sem); new_cred->thread_keyring->gid = new_cred->fsgid; up_write(&new_cred->thread_keyring->sem); +#endif } } @@ -572,7 +587,14 @@ key_ref_t search_process_keyrings_rcu(struct keyring_search_context *ctx) const struct cred *cred = ctx->cred; if (key_validate(cred->request_key_auth) == 0) { +#ifdef CONFIG_KEYP + union key_payload *tmp = + (union key_payload *)(ctx->cred->request_key_auth->name_link.next); + + rka = tmp->data[0]; +#else rka = ctx->cred->request_key_auth->payload.data[0]; +#endif //// was search_process_keyrings() [ie. recursive] ctx->cred = rka->cred; @@ -740,17 +762,30 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, if (!ctx.cred->request_key_auth) goto error; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(ctx.cred->request_key_auth)); +#else down_read(&ctx.cred->request_key_auth->sem); +#endif if (test_bit(KEY_FLAG_REVOKED, &ctx.cred->request_key_auth->flags)) { key_ref = ERR_PTR(-EKEYREVOKED); key = NULL; } else { +#ifdef CONFIG_KEYP + rka = ((union key_payload *)(ctx.cred->request_key_auth->name_link.next)) + ->data[0]; +#else rka = ctx.cred->request_key_auth->payload.data[0]; +#endif key = rka->dest_keyring; __key_get(key); } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(ctx.cred->request_key_auth)); +#else up_read(&ctx.cred->request_key_auth->sem); +#endif if (!key) goto error; key_ref = make_key_ref(key, 1); @@ -819,7 +854,11 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, if (ret < 0) goto invalid_key; +#ifdef CONFIG_KEYP + iee_set_key_last_used_at(key, ktime_get_real_seconds()); +#else key->last_used_at = ktime_get_real_seconds(); +#endif error: put_cred(ctx.cred); diff --git a/security/keys/request_key.c b/security/keys/request_key.c index a7673ad86d18d..79fe763555e1d 100644 --- a/security/keys/request_key.c +++ b/security/keys/request_key.c @@ -14,6 +14,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" #include @@ -285,13 +288,21 @@ static int construct_get_dest_keyring(struct key **_dest_keyring) case KEY_REQKEY_DEFL_REQUESTOR_KEYRING: if (cred->request_key_auth) { authkey = cred->request_key_auth; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(authkey)); +#else down_read(&authkey->sem); +#endif rka = get_request_key_auth(authkey); if (!test_bit(KEY_FLAG_REVOKED, &authkey->flags)) dest_keyring = key_get(rka->dest_keyring); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(authkey)); +#else up_read(&authkey->sem); +#endif if (dest_keyring) { do_perm_check = false; break; @@ -398,7 +409,11 @@ static int construct_alloc_key(struct keyring_search_context *ctx, if (IS_ERR(key)) goto alloc_failed; +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, SET_BIT_OP); +#else set_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags); +#endif if (dest_keyring) { ret = __key_link_lock(dest_keyring, &key->index_key); diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c index bd34317b58cc8..2a53337729bb2 100644 --- a/security/keys/request_key_auth.c +++ b/security/keys/request_key_auth.c @@ -118,19 +118,27 @@ static void free_request_key_auth(struct request_key_auth *rka) /* * Take a reference to the request-key authorisation payload so callers can - * drop authkey->sem before doing operations that may sleep. + * drop the auth key semaphore before doing operations that may sleep. */ struct request_key_auth *request_key_auth_get(struct key *authkey) { struct request_key_auth *rka; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(authkey)); +#else down_read(&authkey->sem); +#endif rka = dereference_key_locked(authkey); if (rka && !test_bit(KEY_FLAG_REVOKED, &authkey->flags)) refcount_inc(&rka->usage); else rka = NULL; +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(authkey)); +#else up_read(&authkey->sem); +#endif return rka; } @@ -173,7 +181,12 @@ static void request_key_auth_revoke(struct key *key) */ static void request_key_auth_destroy(struct key *key) { +#ifdef CONFIG_KEYP + union key_payload *tmp = (union key_payload *)(key->name_link.next); + struct request_key_auth *rka = rcu_access_pointer(tmp->rcu_data0); +#else struct request_key_auth *rka = rcu_access_pointer(key->payload.rcu_data0); +#endif kenter("{%d}", key->serial); if (rka) { @@ -213,22 +226,43 @@ struct key *request_key_auth_new(struct key *target, const char *op, * another process */ if (cred->request_key_auth) { /* it is - use that instantiation context here too */ +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(cred->request_key_auth)); +#else down_read(&cred->request_key_auth->sem); +#endif /* if the auth key has been revoked, then the key we're * servicing is already instantiated */ if (test_bit(KEY_FLAG_REVOKED, &cred->request_key_auth->flags)) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(cred->request_key_auth)); +#else up_read(&cred->request_key_auth->sem); +#endif ret = -EKEYREVOKED; goto error_free_rka; } - irka = cred->request_key_auth->payload.data[0]; + irka = dereference_key_locked(cred->request_key_auth); + if (!irka) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(cred->request_key_auth)); +#else + up_read(&cred->request_key_auth->sem); +#endif + ret = -EKEYREVOKED; + goto error_free_rka; + } rka->cred = get_cred(irka->cred); rka->pid = irka->pid; +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(cred->request_key_auth)); +#else up_read(&cred->request_key_auth->sem); +#endif } else { /* it isn't - use this process as the context */ diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index fee1ab2c734d3..29037ece458b2 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -233,7 +233,11 @@ static int trusted_update(struct key *key, struct key_preparsed_payload *prep) if (key_is_negative(key)) return -ENOKEY; +#ifdef CONFIG_KEYP + p = ((union key_payload *)(key->name_link.next))->data[0]; +#else p = key->payload.data[0]; +#endif if (!p->migratable) return -EPERM; if (datalen <= 0 || datalen > 32767 || !prep->data) @@ -307,7 +311,11 @@ static long trusted_read(const struct key *key, char *buffer, */ static void trusted_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree_sensitive(key->payload.data[0]); +#endif } struct key_type key_type_trusted = { diff --git a/security/keys/user_defined.c b/security/keys/user_defined.c index 749e2a4dcb13e..23a3012927d9d 100644 --- a/security/keys/user_defined.c +++ b/security/keys/user_defined.c @@ -12,6 +12,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" static int logon_vet_description(const char *desc); @@ -109,7 +112,11 @@ int user_update(struct key *key, struct key_preparsed_payload *prep) return ret; /* attach the new data, displacing the old */ +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, prep->expiry); +#else key->expiry = prep->expiry; +#endif if (key_is_positive(key)) zap = dereference_key_locked(key); rcu_assign_keypointer(key, prep->payload.data[0]); @@ -145,7 +152,11 @@ EXPORT_SYMBOL(user_revoke); */ void user_destroy(struct key *key) { +#ifdef CONFIG_KEYP + struct user_key_payload *upayload = ((union key_payload *)(key->name_link.next))->data[0]; +#else struct user_key_payload *upayload = key->payload.data[0]; +#endif kfree_sensitive(upayload); } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 8560e2fc9dc07..15ba4af48541b 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -93,6 +93,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "avc.h" #include "objsec.h" @@ -6715,7 +6718,11 @@ static int selinux_key_alloc(struct key *k, const struct cred *cred, else ksec->sid = tsec->sid; +#ifdef CONFIG_KEYP + iee_set_key_security(k, ksec); +#else k->security = ksec; +#endif return 0; } @@ -6723,7 +6730,11 @@ static void selinux_key_free(struct key *k) { struct key_security_struct *ksec = k->security; +#ifdef CONFIG_KEYP + iee_set_key_security(k, NULL); +#else k->security = NULL; +#endif kfree(ksec); } diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index a0b5c658fe4e5..a365679e1d699 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -44,6 +44,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "smack.h" #define TRANS_TRUE "TRUE" @@ -4426,7 +4429,11 @@ static int smack_key_alloc(struct key *key, const struct cred *cred, { struct smack_known *skp = smk_of_task(smack_cred(cred)); +#ifdef CONFIG_KEYP + iee_set_key_security(key, skp); +#else key->security = skp; +#endif return 0; } @@ -4438,7 +4445,11 @@ static int smack_key_alloc(struct key *key, const struct cred *cred, */ static void smack_key_free(struct key *key) { +#ifdef CONFIG_KEYP + iee_set_key_security(key, NULL); +#else key->security = NULL; +#endif } /** From 5fbf223baa3b88244f7e16a5a9309a35b0bd58a2 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 9 Jul 2026 17:57:15 +0800 Subject: [PATCH 09/26] HAOC3.0/arm64/KEYP: Add support for arm64 Key Protection Support key protection for arm64. Co-authored-by: zhangshiyangB7 Signed-off-by: Liu Zhehui --- .../configs/deepin_arm64_desktop_defconfig | 1 + arch/arm64/include/asm/haoc/haoc-def.h | 40 +++ arch/arm64/include/asm/haoc/haoc.h | 35 +++ arch/arm64/include/asm/haoc/iee-key.h | 289 ++++++++++++++++++ arch/arm64/kernel/haoc/Kconfig | 9 + arch/arm64/kernel/haoc/Makefile | 1 + arch/arm64/kernel/haoc/haoc.c | 35 ++- arch/arm64/kernel/haoc/keyp/Makefile | 1 + arch/arm64/kernel/haoc/keyp/keyp.c | 243 +++++++++++++++ include/linux/key.h | 22 +- mm/slub.c | 5 + 11 files changed, 674 insertions(+), 7 deletions(-) create mode 100644 arch/arm64/include/asm/haoc/iee-key.h create mode 100644 arch/arm64/kernel/haoc/keyp/Makefile create mode 100644 arch/arm64/kernel/haoc/keyp/keyp.c diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index 5c9dc3868a0f4..1e83be50259ec 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -165,6 +165,7 @@ CONFIG_IEE=y CONFIG_IEE_PTRP=y CONFIG_IEE_SIP=y CONFIG_CREDP=y +CONFIG_KEYP=y CONFIG_PTP=y CONFIG_PTP_RESERVE_ORDER=12 CONFIG_KPROBES=y diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index 15ccc77282dd6..73d00df63aa01 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -74,6 +74,35 @@ enum { IEE_OP_SET_PMD_CMPXCHG, IEE_OP_SET_SENSITIVE_PTE, IEE_OP_UNSET_SENSITIVE_PTE, +#endif +#ifdef CONFIG_KEYP + IEE_OP_SET_KEY_UNION, + IEE_OP_SET_KEY_STRUCT, + IEE_OP_SET_KEY_PAYLOAD, + IEE_OP_SET_KEY_USAGE, + IEE_OP_SET_KEY_SERIAL, + IEE_OP_SET_KEY_WATCHERS, + IEE_OP_SET_KEY_USERS, + IEE_OP_SET_KEY_SECURITY, + IEE_OP_SET_KEY_EXPIRY, + IEE_OP_SET_KEY_REVOKED_AT, + IEE_OP_SET_KEY_LAST_USED_AT, + IEE_OP_SET_KEY_UID, + IEE_OP_SET_KEY_GID, + IEE_OP_SET_KEY_PERM, + IEE_OP_SET_KEY_QUOTALEN, + IEE_OP_SET_KEY_DATALEN, + IEE_OP_SET_KEY_STATE, + IEE_OP_SET_KEY_MAGIC, + IEE_OP_SET_KEY_FLAGS, + IEE_OP_SET_KEY_INDEX_KEY, + IEE_OP_SET_KEY_HASH, + IEE_OP_SET_KEY_LEN_DESC, + IEE_OP_SET_KEY_TYPE, + IEE_OP_SET_KEY_TAG, + IEE_OP_SET_KEY_DESCRIPTION, + IEE_OP_SET_KEY_RESTRICT_LINK, + IEE_OP_SET_KEY_FLAG_BIT, #endif IEE_FLAG_END }; @@ -84,6 +113,17 @@ enum { #define AT_SUB_AND_TEST 3 #endif +#ifdef CONFIG_KEYP +#define REFCOUNT_INC 1 +#define REFCOUNT_SET 2 +#define REFCOUNT_DEC_AND_TEST 3 +#define REFCOUNT_INC_NOT_ZERO 4 + +#define SET_BIT_OP 1 +#define TEST_AND_CLEAR_BIT 2 +#define TEST_AND_SET_BIT 3 +#endif + /* The entry gate of all IEE APIs. The first parameter must be a valid * IEE function index. */ diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index 70d173918638e..684557d65ddcf 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -115,4 +115,39 @@ void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd); #endif + +#ifdef CONFIG_KEYP +#include +struct watch_list; + +void _iee_set_key_union(unsigned long __unused, struct key *key, struct key_union *key_union); +void _iee_set_key_struct(unsigned long __unused, struct key *key, struct key_struct *key_struct); +void _iee_set_key_payload(unsigned long __unused, struct key *key, union key_payload *key_payload); +unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, int n, int flag); +void _iee_set_key_serial(unsigned long __unused, struct key *key, key_serial_t serial); +void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers); +void _iee_set_key_user(unsigned long __unused, struct key *key, struct key_user *user); +void _iee_set_key_security(unsigned long __unused, struct key *key, void *security); +void _iee_set_key_expiry(unsigned long __unused, struct key *key, time64_t expiry); +void _iee_set_key_revoked_at(unsigned long __unused, struct key *key, time64_t revoked_at); +void _iee_set_key_last_used_at(unsigned long __unused, struct key *key, time64_t last_used_at); +void _iee_set_key_uid(unsigned long __unused, struct key *key, kuid_t uid); +void _iee_set_key_gid(unsigned long __unused, struct key *key, kgid_t gid); +void _iee_set_key_perm(unsigned long __unused, struct key *key, key_perm_t perm); +void _iee_set_key_quotalen(unsigned long __unused, struct key *key, unsigned short quotalen); +void _iee_set_key_datalen(unsigned long __unused, struct key *key, unsigned short datalen); +void _iee_set_key_state(unsigned long __unused, struct key *key, short state); +void _iee_set_key_magic(unsigned long __unused, struct key *key, unsigned int magic); +void _iee_set_key_flags(unsigned long __unused, struct key *key, unsigned long flags); +void _iee_set_key_index_key(unsigned long __unused, struct key *key, + struct keyring_index_key *index_key); +void _iee_set_key_hash(unsigned long __unused, struct key *key, unsigned long hash); +void _iee_set_key_len_desc(unsigned long __unused, struct key *key, unsigned long len_desc); +void _iee_set_key_type(unsigned long __unused, struct key *key, struct key_type *type); +void _iee_set_key_domain_tag(unsigned long __unused, struct key *key, struct key_tag *domain_tag); +void _iee_set_key_description(unsigned long __unused, struct key *key, char *description); +void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, + struct key_restriction *restrict_link); +unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); +#endif #endif diff --git a/arch/arm64/include/asm/haoc/iee-key.h b/arch/arm64/include/asm/haoc/iee-key.h new file mode 100644 index 0000000000000..ef30ed898e3fd --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-key.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_KEY_H +#define _LINUX_IEE_KEY_H + +#include +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_set_key_union(struct key *key, struct key_union *key_union) +{ + if (!haoc_enabled) { + key->graveyard_link.next = (struct list_head *)key_union; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); +} + +static void __maybe_unused iee_set_key_struct(struct key *key, struct key_struct *key_struct) +{ + if (!haoc_enabled) { + key->name_link.prev = (struct list_head *)key_struct; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); +} + +static void __maybe_unused iee_set_key_payload(struct key *key, union key_payload *key_payload) +{ + if (!haoc_enabled) { + key->name_link.next = (struct list_head *)key_payload; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); +} + +extern bool iee_set_key_usage(struct key *key, int n, int flag); + +static void __maybe_unused iee_set_key_serial(struct key *key, key_serial_t serial) +{ + if (!haoc_enabled) { + key->serial = serial; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); +} + +#ifdef CONFIG_KEY_NOTIFICATIONS +static void __maybe_unused iee_set_key_watchers(struct key *key, struct watch_list *watchers) +{ + if (!haoc_enabled) { + key->watchers = watchers; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); +} +#endif + +static void __maybe_unused iee_set_key_user(struct key *key, struct key_user *user) +{ + if (!haoc_enabled) { + key->user = user; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); +} + +static void __maybe_unused iee_set_key_security(struct key *key, void *security) +{ + if (!haoc_enabled) { + key->security = security; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); +} + +static void __maybe_unused iee_set_key_expiry(struct key *key, time64_t expiry) +{ + if (!haoc_enabled) { + key->expiry = expiry; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); +} + +static void __maybe_unused iee_set_key_revoked_at(struct key *key, time64_t revoked_at) +{ + if (!haoc_enabled) { + key->revoked_at = revoked_at; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); +} + +static void __maybe_unused iee_set_key_last_used_at(struct key *key, time64_t last_used_at) +{ + if (!haoc_enabled) { + key->last_used_at = last_used_at; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); +} + +static void __maybe_unused iee_set_key_uid(struct key *key, kuid_t uid) +{ + if (!haoc_enabled) { + key->uid = uid; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); +} + +static void __maybe_unused iee_set_key_gid(struct key *key, kgid_t gid) +{ + if (!haoc_enabled) { + key->gid = gid; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); +} + +static void __maybe_unused iee_set_key_perm(struct key *key, key_perm_t perm) +{ + if (!haoc_enabled) { + key->perm = perm; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); +} + +static void __maybe_unused iee_set_key_quotalen(struct key *key, unsigned short quotalen) +{ + if (!haoc_enabled) { + key->quotalen = quotalen; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); +} + +static void __maybe_unused iee_set_key_datalen(struct key *key, unsigned short datalen) +{ + if (!haoc_enabled) { + key->datalen = datalen; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); +} + +static void __maybe_unused iee_set_key_state(struct key *key, short state) +{ + if (!haoc_enabled) { + smp_store_release(&key->state, state); + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); +} + +#ifdef KEY_DEBUGGING +static void __maybe_unused iee_set_key_magic(struct key *key, unsigned int magic) +{ + if (!haoc_enabled) { + key->magic = magic; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); +} +#endif + +static void __maybe_unused iee_set_key_flags(struct key *key, unsigned long flags) +{ + if (!haoc_enabled) { + key->flags = flags; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); +} + +static void __maybe_unused iee_set_key_index_key(struct key *key, + struct keyring_index_key *index_key) +{ + if (!haoc_enabled) { + key->index_key = *index_key; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); +} + +static void __maybe_unused iee_set_key_hash(struct key *key, unsigned long hash) +{ + if (!haoc_enabled) { + key->hash = hash; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); +} + +static void __maybe_unused iee_set_key_len_desc(struct key *key, unsigned long len_desc) +{ + if (!haoc_enabled) { + key->len_desc = len_desc; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); +} + +static void __maybe_unused iee_set_key_type(struct key *key, struct key_type *type) +{ + if (!haoc_enabled) { + key->type = type; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); +} + +static void __maybe_unused iee_set_key_domain_tag(struct key *key, struct key_tag *domain_tag) +{ + if (!haoc_enabled) { + key->domain_tag = domain_tag; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); +} + +static void __maybe_unused iee_set_key_description(struct key *key, char *description) +{ + if (!haoc_enabled) { + key->description = description; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); +} + +static void __maybe_unused iee_set_key_restrict_link(struct key *key, + struct key_restriction *restrict_link) +{ + if (!haoc_enabled) { + key->restrict_link = restrict_link; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); +} + +static bool __maybe_unused iee_set_key_flag_bit(struct key *key, long nr, int flag) +{ + bool ret; + + if (!haoc_enabled) { + switch (flag) { + case SET_BIT_OP: + set_bit(nr, &key->flags); + break; + case TEST_AND_CLEAR_BIT: + return test_and_clear_bit(nr, &key->flags); + case TEST_AND_SET_BIT: + return test_and_set_bit(nr, &key->flags); + } + return false; + } + + ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); + return ret; +} + +#endif diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index e34960c41acde..14b3a9021162c 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -74,4 +74,13 @@ config PTP_RESERVE_ORDER depends on PTP range 9 15 default 12 + +config KEYP + bool "Struct Key protection(KEYP)" + depends on IEE + help + This option enables protection for the kernel's struct key. All modifications + of keys must be made and verified by IEE APIs to defend memory corruptions. + + If unsure, say N. endmenu # HAOC diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile index f874c584bb86f..3860197b26f8e 100644 --- a/arch/arm64/kernel/haoc/Makefile +++ b/arch/arm64/kernel/haoc/Makefile @@ -3,3 +3,4 @@ obj-y += haoc.o haoc-bitmap.o obj-y += iee/ obj-$(CONFIG_CREDP) += credp/ obj-$(CONFIG_PTP) += ptp/ +obj-$(CONFIG_KEYP) += keyp/ diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index 3d2046e452398..f257cb2ecbbee 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -79,8 +79,37 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_pmd_xchg_relaxed, (iee_func)_iee_set_cmpxchg_relaxed, (iee_func)_iee_set_pmd_cmpxchg_relaxed, - (iee_func)_iee_set_sensitive_pte, - (iee_func)_iee_unset_sensitive_pte, + (iee_func)_iee_set_sensitive_pte, + (iee_func)_iee_unset_sensitive_pte, #endif - NULL +#ifdef CONFIG_KEYP + (iee_func)_iee_set_key_union, + (iee_func)_iee_set_key_struct, + (iee_func)_iee_set_key_payload, + (iee_func)_iee_set_key_usage, + (iee_func)_iee_set_key_serial, + (iee_func)_iee_set_key_watchers, + (iee_func)_iee_set_key_user, + (iee_func)_iee_set_key_security, + (iee_func)_iee_set_key_expiry, + (iee_func)_iee_set_key_revoked_at, + (iee_func)_iee_set_key_last_used_at, + (iee_func)_iee_set_key_uid, + (iee_func)_iee_set_key_gid, + (iee_func)_iee_set_key_perm, + (iee_func)_iee_set_key_quotalen, + (iee_func)_iee_set_key_datalen, + (iee_func)_iee_set_key_state, + (iee_func)_iee_set_key_magic, + (iee_func)_iee_set_key_flags, + (iee_func)_iee_set_key_index_key, + (iee_func)_iee_set_key_hash, + (iee_func)_iee_set_key_len_desc, + (iee_func)_iee_set_key_type, + (iee_func)_iee_set_key_domain_tag, + (iee_func)_iee_set_key_description, + (iee_func)_iee_set_key_restrict_link, + (iee_func)_iee_set_key_flag_bit, +#endif + NULL }; diff --git a/arch/arm64/kernel/haoc/keyp/Makefile b/arch/arm64/kernel/haoc/keyp/Makefile new file mode 100644 index 0000000000000..307cebd57ff71 --- /dev/null +++ b/arch/arm64/kernel/haoc/keyp/Makefile @@ -0,0 +1 @@ +obj-y += keyp.o \ No newline at end of file diff --git a/arch/arm64/kernel/haoc/keyp/keyp.c b/arch/arm64/kernel/haoc/keyp/keyp.c new file mode 100644 index 0000000000000..a6d4681e97b59 --- /dev/null +++ b/arch/arm64/kernel/haoc/keyp/keyp.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +struct watch_list; + +static inline void iee_verify_key_type(const struct key *key) +{ + iee_verify_type((unsigned long)key, IEE_KEY, "key"); +} + +static inline struct key *key_write_ptr(struct key *key) +{ + iee_verify_key_type(key); + return haoc_enabled ? __ptr_to_iee(key) : key; +} + +unsigned long __iee_code _iee_set_key_flag_bit(unsigned long __unused, struct key *key, + long nr, int flag) +{ + key = key_write_ptr(key); + switch (flag) { + case SET_BIT_OP: { + set_bit(nr, &key->flags); + break; + } + case TEST_AND_CLEAR_BIT: { + return test_and_clear_bit(nr, &key->flags); + } + case TEST_AND_SET_BIT: { + return test_and_set_bit(nr, &key->flags); + } + } + return 0; +} + +void __iee_code _iee_set_key_restrict_link(unsigned long __unused, + struct key *key, + struct key_restriction *restrict_link) +{ + key = key_write_ptr(key); + key->restrict_link = restrict_link; +} + +void __iee_code _iee_set_key_magic(unsigned long __unused, struct key *key, + unsigned int magic) +{ + key = key_write_ptr(key); +#ifdef KEY_DEBUGGING + key->magic = magic; +#endif +} + +void __iee_code _iee_set_key_flags(unsigned long __unused, struct key *key, + unsigned long flags) +{ + key = key_write_ptr(key); + key->flags = flags; +} + +void __iee_code _iee_set_key_index_key(unsigned long __unused, + struct key *key, + struct keyring_index_key *index_key) +{ + key = key_write_ptr(key); + key->index_key = *index_key; +} + +void __iee_code _iee_set_key_hash(unsigned long __unused, struct key *key, + unsigned long hash) +{ + key = key_write_ptr(key); + key->hash = hash; +} + +void __iee_code _iee_set_key_len_desc(unsigned long __unused, struct key *key, + unsigned long len_desc) +{ + key = key_write_ptr(key); + key->len_desc = len_desc; +} + +void __iee_code _iee_set_key_type(unsigned long __unused, struct key *key, + struct key_type *type) +{ + key = key_write_ptr(key); + key->type = type; +} + +void __iee_code _iee_set_key_domain_tag(unsigned long __unused, + struct key *key, + struct key_tag *domain_tag) +{ + key = key_write_ptr(key); + key->domain_tag = domain_tag; +} + +void __iee_code _iee_set_key_description(unsigned long __unused, + struct key *key, char *description) +{ + key = key_write_ptr(key); + key->description = description; +} + +void __iee_code _iee_set_key_uid(unsigned long __unused, struct key *key, + kuid_t uid) +{ + key = key_write_ptr(key); + key->uid = uid; +} + +void __iee_code _iee_set_key_gid(unsigned long __unused, struct key *key, + kgid_t gid) +{ + key = key_write_ptr(key); + key->gid = gid; +} + +void __iee_code _iee_set_key_perm(unsigned long __unused, struct key *key, + key_perm_t perm) +{ + key = key_write_ptr(key); + key->perm = perm; +} + +void __iee_code _iee_set_key_quotalen(unsigned long __unused, struct key *key, + unsigned short quotalen) +{ + key = key_write_ptr(key); + key->quotalen = quotalen; +} + +void __iee_code _iee_set_key_datalen(unsigned long __unused, struct key *key, + unsigned short datalen) +{ + key = key_write_ptr(key); + key->datalen = datalen; +} + +void __iee_code _iee_set_key_state(unsigned long __unused, struct key *key, + short state) +{ + key = key_write_ptr(key); + smp_store_release(&key->state, state); +} + +void __iee_code _iee_set_key_user(unsigned long __unused, struct key *key, + struct key_user *user) +{ + key = key_write_ptr(key); + key->user = user; +} + +void __iee_code _iee_set_key_security(unsigned long __unused, struct key *key, + void *security) +{ + key = key_write_ptr(key); + key->security = security; +} + +void __iee_code _iee_set_key_expiry(unsigned long __unused, struct key *key, + time64_t expiry) +{ + key = key_write_ptr(key); + key->expiry = expiry; +} + +void __iee_code _iee_set_key_revoked_at(unsigned long __unused, + struct key *key, time64_t revoked_at) +{ + key = key_write_ptr(key); + key->revoked_at = revoked_at; +} + +void __iee_code _iee_set_key_last_used_at(unsigned long __unused, + struct key *key, + time64_t last_used_at) +{ + key = key_write_ptr(key); + key->last_used_at = last_used_at; +} + +unsigned long __iee_code _iee_set_key_usage(unsigned long __unused, struct key *key, + int n, int flag) +{ + key = key_write_ptr(key); + switch (flag) { + case REFCOUNT_INC: { + refcount_inc(&key->usage); + break; + } + case REFCOUNT_SET: { + refcount_set(&key->usage, n); + break; + } + case REFCOUNT_DEC_AND_TEST: { + return refcount_dec_and_test(&key->usage); + } + case REFCOUNT_INC_NOT_ZERO: { + return refcount_inc_not_zero(&key->usage); + } + } + return 0; +} + +void __iee_code _iee_set_key_serial(unsigned long __unused, struct key *key, + key_serial_t serial) +{ + key = key_write_ptr(key); + key->serial = serial; +} + +void __iee_code _iee_set_key_watchers(unsigned long __unused, struct key *key, + struct watch_list *watchers) +{ + key = key_write_ptr(key); +#ifdef CONFIG_KEY_NOTIFICATIONS + key->watchers = watchers; +#endif +} + +void __iee_code _iee_set_key_union(unsigned long __unused, struct key *key, + struct key_union *key_union) +{ + key = key_write_ptr(key); + key->graveyard_link.next = (struct list_head *)key_union; +} + +void __iee_code _iee_set_key_struct(unsigned long __unused, struct key *key, + struct key_struct *key_struct) +{ + key = key_write_ptr(key); + key->name_link.prev = (struct list_head *)key_struct; +} + +void __iee_code _iee_set_key_payload(unsigned long __unused, struct key *key, + union key_payload *key_payload) +{ + key = key_write_ptr(key); + key->name_link.next = (struct list_head *)key_payload; +} diff --git a/include/linux/key.h b/include/linux/key.h index 1881a5c77d37e..5c38bc6aedc03 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -303,13 +303,27 @@ struct key_struct { }; #define KEY_SEM(KEY) (((struct key_union *)(KEY->graveyard_link.next))->sem) #include +extern bool haoc_enabled; extern unsigned long long iee_rw_gate(int flag, ...); static bool iee_set_key_usage(struct key *key, int n, int flag) { - bool ret; - - ret = iee_rw_gate(IEE_OP_SET_KEY_USAGE, key, n, flag); - return ret; + if (!haoc_enabled) { + switch (flag) { + case REFCOUNT_INC: + refcount_inc(&key->usage); + break; + case REFCOUNT_SET: + refcount_set(&key->usage, n); + break; + case REFCOUNT_DEC_AND_TEST: + return refcount_dec_and_test(&key->usage); + case REFCOUNT_INC_NOT_ZERO: + return refcount_inc_not_zero(&key->usage); + } + return false; + } + + return iee_rw_gate(IEE_OP_SET_KEY_USAGE, key, n, flag); } #endif diff --git a/mm/slub.c b/mm/slub.c index 88b2a9d473d66..60519835da977 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2149,6 +2149,11 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab = iee_alloc_task_token_slab(s, slab, oo_order(oo)); #endif if (unlikely(!slab)) { +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar && + !IS_ENABLED(CONFIG_IEE_ALLOW_SPLIT_LM)) + return NULL; +#endif oo = s->min; alloc_gfp = flags; /* From 9535302daafd3f9ef52c8112d8bbf563d2e63628 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 9 Jul 2026 21:59:38 +0800 Subject: [PATCH 10/26] HAOC3.0/x86_64/SELINUXP: Add support for x86 SELinux Protection Support selinux protection for x86. Co-authored-by: Signed-off-by: sh Signed-off-by: Liu Zhehui --- arch/x86/Kconfig | 16 +++++- arch/x86/configs/deepin_x86_desktop_defconfig | 1 + arch/x86/include/asm/haoc/haoc-def.h | 7 +++ arch/x86/include/asm/haoc/haoc.h | 10 ++++ arch/x86/include/asm/haoc/iee-selinux.h | 56 ++++++++++++++++++ arch/x86/include/asm/haoc/iee-si.h | 8 ++- arch/x86/kernel/haoc/Makefile | 2 + arch/x86/kernel/haoc/haoc.c | 7 +++ arch/x86/kernel/haoc/iee/iee-init.c | 7 +++ arch/x86/kernel/haoc/iee/iee-si.c | 6 ++ arch/x86/kernel/haoc/ptp/ptp.c | 18 ++++++ arch/x86/kernel/haoc/selinuxp/Makefile | 5 ++ arch/x86/kernel/haoc/selinuxp/selinuxp.c | 57 +++++++++++++++++++ arch/x86/kernel/vmlinux.lds.S | 12 ++++ mm/slab.h | 4 ++ mm/slab_common.c | 10 +++- mm/slub.c | 45 ++++++++++++++- security/selinux/hooks.c | 44 ++++++++++++++ security/selinux/ima.c | 15 +++++ security/selinux/include/security.h | 5 ++ security/selinux/selinuxfs.c | 51 +++++++++++++++++ security/selinux/ss/services.c | 55 ++++++++++++++++++ security/selinux/status.c | 31 ++++++++++ 23 files changed, 466 insertions(+), 6 deletions(-) create mode 100644 arch/x86/include/asm/haoc/iee-selinux.h create mode 100644 arch/x86/kernel/haoc/selinuxp/Makefile create mode 100644 arch/x86/kernel/haoc/selinuxp/selinuxp.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index c745d0438f996..ad824af795cbd 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1586,7 +1586,6 @@ config IEE_SIP Protects critical instructions that may break the isolation of IEE, such as writing system control registers. These instructions would be executated inside IEE. - config CREDP bool "Struct cred protection(CREDP)" depends on IEE @@ -1605,6 +1604,21 @@ config KEYP If unsure, say N. +config IEE_SELINUX_P + bool "SELinux Protection(IEE_SELINUX_P)" + help + Simple enhancement of the famous linux security module SELinux by putting + several critical data, like selinux policy, into the IEE region. + + If unsure, say N. + depends on SECURITY_SELINUX + depends on IEE + +config PTP_RESERVE_ORDER + depends on PTP + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + range 9 15 + # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 3cb8f49283526..f968005937b9a 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -93,6 +93,7 @@ CONFIG_IEE_PTRP=y CONFIG_IEE_SIP=y CONFIG_CREDP=y CONFIG_KEYP=y +CONFIG_IEE_SELINUX_P=y CONFIG_PTP=y CONFIG_PTP_RESERVE_ORDER=12 CONFIG_NUMA=y diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index aab082ba5fe20..7dfc31638c40a 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -88,6 +88,13 @@ enum { IEE_OP_SET_KEY_RESTRICT_LINK, IEE_OP_SET_KEY_FLAG_BIT, #endif +#ifdef CONFIG_IEE_SELINUX_P + IEE_SEL_SET_STATUS_PG, + IEE_SEL_SET_ENFORCING, + IEE_SEL_SET_INITIALIZED, + IEE_SEL_SET_POLICY_CAP, + IEE_SEL_RCU_ASSIGN_POLICY, +#endif #ifdef CONFIG_PTP #ifdef CONFIG_IEE_PTRP IEE_OP_UNSET_TOKEN, diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index 4fd1b32cbda4d..0ee4c19513e56 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -28,6 +28,16 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); void _iee_set_bitmap_type(unsigned long __unused, uint8_t *bitmap_ptr, enum HAOC_BITMAP_TYPE type, int num_pages); +#ifdef CONFIG_IEE_SELINUX_P +#include +void _iee_set_selinux_status_pg(unsigned long iee_offset, struct page *new_page); +void _iee_set_selinux_enforcing(unsigned long iee_offset, bool value); +void _iee_mark_selinux_initialized(unsigned long iee_offset); +void _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap); +void _iee_sel_rcu_assign_policy(unsigned long iee_offset, + struct selinux_policy *new_policy, struct selinux_policy *iee_new_policy); +#endif + #ifdef CONFIG_PTP #ifdef CONFIG_IEE_PTRP void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, diff --git a/arch/x86/include/asm/haoc/iee-selinux.h b/arch/x86/include/asm/haoc/iee-selinux.h new file mode 100644 index 0000000000000..413802b7c1273 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-selinux.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_SELINUX_P_H +#define _LINUX_IEE_SELINUX_P_H + +#include +#include +#include "security.h" +#include "ss/services.h" + +extern struct kmem_cache *policy_jar; +extern unsigned long long iee_rw_gate(int flag, ...); +extern unsigned long __iee_selinux_data_start[]; +extern unsigned long __iee_selinux_data_end[]; + +static inline struct mutex *iee_get_selinux_policy_lock(void) +{ + return (struct mutex *)(selinux_state.policy_mutex.owner.counter); +} + +static inline struct mutex *iee_get_selinux_status_lock(void) +{ + return (struct mutex *)(selinux_state.status_lock.owner.counter); +} + +static inline void iee_set_selinux_status_pg(struct page *new_page) +{ + iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); +} + +static inline void enforcing_set(bool value) +{ + iee_rw_gate(IEE_SEL_SET_ENFORCING, value); +} + +static inline void selinux_mark_initialized(void) +{ + iee_rw_gate(IEE_SEL_SET_INITIALIZED); +} + +static inline void iee_set_sel_policy_cap(unsigned int idx, int cap) +{ + iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); +} + +/* + * Please make sure param iee_new_policy is from policy_jar memcache. + * Need to free new_policy after calling this func as it's only used to + * trans data from kernel. + */ +static inline void iee_sel_rcu_assign_policy(struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); +} + +#endif diff --git a/arch/x86/include/asm/haoc/iee-si.h b/arch/x86/include/asm/haoc/iee-si.h index 3baf1813f19d3..3f60275696b6e 100644 --- a/arch/x86/include/asm/haoc/iee-si.h +++ b/arch/x86/include/asm/haoc/iee-si.h @@ -39,5 +39,11 @@ static inline void iee_load_idt(void *ptr) { iee_rwx_gate(IEE_LOAD_IDT, ptr); } +extern bool iee_init_done; -#endif \ No newline at end of file +#ifdef CONFIG_IEE_SELINUX_P +extern unsigned long __iee_selinux_data_start[]; +extern unsigned long __iee_selinux_data_end[]; +#endif + +#endif diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index 0f13cdb92c787..7de5a80764f05 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -3,3 +3,5 @@ obj-y += iee/ obj-$(CONFIG_PTP) += ptp/ obj-$(CONFIG_CREDP) += credp/ obj-$(CONFIG_KEYP) += keyp/ +ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 6ab47453dd018..4365c1239d15a 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -88,6 +88,13 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_key_restrict_link, (iee_func)_iee_set_key_flag_bit, #endif +#ifdef CONFIG_IEE_SELINUX_P + (iee_func)_iee_set_selinux_status_pg, + (iee_func)_iee_set_selinux_enforcing, + (iee_func)_iee_mark_selinux_initialized, + (iee_func)_iee_set_sel_policy_cap, + (iee_func)_iee_sel_rcu_assign_policy, +#endif #ifdef CONFIG_PTP #ifdef CONFIG_IEE_PTRP (iee_func)_iee_unset_token, diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index f80cde9113e72..bab99a040865c 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -28,6 +28,10 @@ #ifdef CONFIG_CREDP extern void credp_init(void); #endif + +#ifdef CONFIG_IEE_SELINUX_P +extern void iee_selinuxp_init(void); +#endif #ifdef CONFIG_PTP extern void haoc_ptp_init(void); #endif @@ -276,6 +280,9 @@ void __init iee_post_init(void) #endif #ifdef CONFIG_CREDP credp_init(); +#endif +#ifdef CONFIG_IEE_SELINUX_P + iee_selinuxp_init(); #endif haoc_init_done = true; } diff --git a/arch/x86/kernel/haoc/iee/iee-si.c b/arch/x86/kernel/haoc/iee/iee-si.c index 8feb9d8312246..1a92b76ff4d30 100644 --- a/arch/x86/kernel/haoc/iee/iee-si.c +++ b/arch/x86/kernel/haoc/iee/iee-si.c @@ -122,6 +122,12 @@ void __init iee_sip_init(void) end = (unsigned long)__iee_si_data_end; num_pages = (end - start) / PAGE_SIZE; set_iee_pages(start, num_pages, IEE_SIP_DATA); +#ifdef CONFIG_IEE_SELINUX_P + start = (unsigned long)__iee_selinux_data_start; + end = (unsigned long)__iee_selinux_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_pages(start, num_pages, IEE_SIP_DATA); +#endif /* All initialization is done. Do some simple tests. */ pr_info("IEE: testing iee_exec_entry si_test..."); iee_sip_test(); diff --git a/arch/x86/kernel/haoc/ptp/ptp.c b/arch/x86/kernel/haoc/ptp/ptp.c index 2ead8feb18308..7769b1b77ace5 100644 --- a/arch/x86/kernel/haoc/ptp/ptp.c +++ b/arch/x86/kernel/haoc/ptp/ptp.c @@ -186,6 +186,16 @@ static inline void check_dep_and_ro(unsigned long image_addr, unsigned long size (unsigned long)_stext, (unsigned long)_etext, pte, "text"); check_addr_range_ro(image_addr, image_addr + size, (unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); +#ifdef CONFIG_IEE_SIP + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_si_text_start, (unsigned long)__iee_si_text_end, pte, "iee_si_text"); + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_si_data_start, (unsigned long)__iee_si_data_end, pte, "iee_si_data"); +#endif +#ifdef CONFIG_IEE_SELINUX_P + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_selinux_data_start, (unsigned long)__iee_selinux_data_end, pte, "iee_si_data"); +#endif } static inline bool preserves_existing_writable_mapping(pte_t *ptep, pte_t pte) @@ -376,6 +386,14 @@ static inline void check_text_poke_ro(unsigned long image_addr, unsigned long si check_addr_range_ro(image_addr, image_addr + size, (unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); +#ifdef CONFIG_IEE_SIP + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_si_data_start, (unsigned long)__iee_si_data_end, pte, "iee_si_data"); +#endif +#ifdef CONFIG_IEE_SELINUX_P + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_selinux_data_start, (unsigned long)__iee_selinux_data_end, pte, "iee_si_data"); +#endif } void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte) diff --git a/arch/x86/kernel/haoc/selinuxp/Makefile b/arch/x86/kernel/haoc/selinuxp/Makefile new file mode 100644 index 0000000000000..8e5f128f5a430 --- /dev/null +++ b/arch/x86/kernel/haoc/selinuxp/Makefile @@ -0,0 +1,5 @@ +ccflags-y += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +obj-y += selinuxp.o + +FLASK_HDR := $(srctree)/security/selinux/flask.h +$(obj)/selinuxp.o: $(FLASK_HDR) diff --git a/arch/x86/kernel/haoc/selinuxp/selinuxp.c b/arch/x86/kernel/haoc/selinuxp/selinuxp.c new file mode 100644 index 0000000000000..6b4a18dc316e5 --- /dev/null +++ b/arch/x86/kernel/haoc/selinuxp/selinuxp.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "security.h" +#include "ss/services.h" +#include +#include +#include + +void _iee_set_selinux_status_pg(unsigned long iee_offset, struct page *new_page) +{ + selinux_state.status_page = new_page; +} + +void _iee_set_selinux_enforcing(unsigned long iee_offset, bool value) +{ + selinux_state.enforcing = value; +} + +void _iee_mark_selinux_initialized(unsigned long iee_offset) +{ + /* do a synchronized write to avoid race conditions */ + smp_store_release(((bool *)&(selinux_state.initialized)), true); + pr_info("IEE: Mark selinux initialized."); +} + +void _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap) +{ + selinux_state.policycap[idx] = cap; +} + +/* + * Please make sure param iee_new_policy is from policy_jar memcache. + * Need to free new_policy after calling this func as it's only used to + * trans data from kernel. + */ +void _iee_sel_rcu_assign_policy(unsigned long iee_offset, struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + iee_verify_type(__pa(iee_new_policy), IEE_POLICY, "policy"); + /* TODO: Verify information from incoming policy. */ + + /* Copy data from kernel to new allocated policy struct inside iee. */ + memcpy(iee_new_policy, new_policy, sizeof(struct selinux_policy)); + rcu_assign_pointer(selinux_state.policy, iee_new_policy); + pr_info("IEE: assigned rcu pointer selinux_state.policy."); +} + +void __init iee_selinuxp_init(void) +{ + unsigned long start, end; + int num_pages; + + start = (unsigned long)__iee_selinux_data_start; + end = (unsigned long)__iee_selinux_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_pages(start, num_pages, IEE_SIP_DATA); +} diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index a4cb219a8a92c..0385cf9ce8c88 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -126,6 +126,15 @@ __iee_si_data_end = .; #define CRED_DATA #endif +#ifdef CONFIG_IEE_SELINUX_P +#define IEE_SELINUX_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_start = .; \ + *(.iee.selinux) \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_end = .; +#endif + PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(6); /* RW_ */ @@ -216,6 +225,9 @@ SECTIONS #endif #ifdef CONFIG_CREDP CRED_DATA +#endif +#ifdef CONFIG_IEE_SELINUX_P + IEE_SELINUX_DATA #endif CONSTRUCTORS diff --git a/mm/slab.h b/mm/slab.h index 60e1abcfa8bc1..81ac1cfc0d07e 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -35,6 +35,10 @@ extern struct kmem_cache *key_struct_jar; extern struct kmem_cache *key_payload_jar; #endif +#ifdef CONFIG_IEE_SELINUX_P +extern struct kmem_cache *policy_jar; +#endif + /* * Freelist pointer and counter to cmpxchg together, avoids the typical ABA * problems with cmpxchg of just a pointer. diff --git a/mm/slab_common.c b/mm/slab_common.c index 84eed4708894f..91ed99c6a2057 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -145,6 +145,11 @@ int slab_unmergeable(struct kmem_cache *s) if (s->ctor) return 1; +#ifdef CONFIG_IEE_SELINUX_P + if (strcmp(s->name, "policy_jar") == 0) + return 1; +#endif + #ifdef CONFIG_HARDENED_USERCOPY if (s->usersize) return 1; @@ -175,12 +180,15 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, if (ctor) return NULL; - #ifdef CONFIG_KEYP if (strcmp(name, "key_jar") == 0 || strcmp(name, "key_union_jar") == 0 || strcmp(name, "key_struct_jar") == 0 || strcmp(name, "key_payload_jar") == 0) return NULL; #endif +#ifdef CONFIG_IEE_SELINUX_P + if (strcmp(name, "policy_jar") == 0) + return NULL; +#endif size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); diff --git a/mm/slub.c b/mm/slub.c index 60519835da977..de3d3a427a598 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -484,7 +484,7 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) #endif freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); - #ifdef CONFIG_CREDP +#ifdef CONFIG_CREDP if (haoc_enabled && s == cred_jar) { iee_set_freeptr((void **)freeptr_addr, (void *)(freelist_ptr_encode(s, fp, freeptr_addr).v)); @@ -497,6 +497,13 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) (void *)freelist_ptr_encode(s, fp, freeptr_addr).v); return; } +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && s == policy_jar) { + iee_set_freeptr((void **)freeptr_addr, + (void *)freelist_ptr_encode(s, fp, freeptr_addr).v); + return; + } #endif *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); } @@ -2137,11 +2144,14 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; - #ifdef CONFIG_KEYP if (s == key_jar) alloc_gfp |= __GFP_ZERO; #endif +#ifdef CONFIG_IEE_SELINUX_P + if (s == policy_jar) + alloc_gfp |= __GFP_ZERO; +#endif slab = alloc_slab_page(alloc_gfp, node, oo); #ifdef CONFIG_IEE_PTRP @@ -2156,6 +2166,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) #endif oo = s->min; alloc_gfp = flags; +#ifdef CONFIG_IEE_SELINUX_P + if (s == policy_jar) + alloc_gfp |= __GFP_ZERO; +#endif /* * Allocation may have failed due to fragmentation. * Try a lower order alloc if possible @@ -2177,7 +2191,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->objects = oo_objects(oo); slab->inuse = 0; slab->frozen = 0; - #ifdef CONFIG_IEE if(haoc_enabled) iee_allocate_slab_data(s, slab, oo_order(oo)); @@ -2209,6 +2222,16 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) oo_order(oo), IEE_KEY); #endif } +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && s == policy_jar) +#ifdef CONFIG_X86_64 + set_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << oo_order(oo), IEE_POLICY); +#else + set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + oo_order(oo), IEE_SELINUX); +#endif #endif account_slab(slab, oo_order(oo), s, flags); @@ -2294,6 +2317,18 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) #endif } #endif + +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && s == policy_jar) { +#ifdef CONFIG_X86_64 + unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << order); +#else + unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); +#endif + } +#endif + __free_pages(&folio->page, order); } @@ -4681,6 +4716,10 @@ static int calculate_sizes(struct kmem_cache *s) #ifdef CONFIG_KEYP if (strcmp(s->name, "key_jar") == 0) order = IEE_DATA_ORDER; +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && strcmp(s->name, "policy_jar") == 0) + order = IEE_DATA_ORDER; #endif if ((int)order < 0) return 0; diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 15ba4af48541b..dce4479cfe307 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -96,6 +96,10 @@ #ifdef CONFIG_KEYP #include #endif +#ifdef CONFIG_IEE_SELINUX_P +#include +#include +#endif #include "avc.h" #include "objsec.h" @@ -110,7 +114,11 @@ #define SELINUX_INODE_INIT_XATTRS 1 +#ifdef CONFIG_IEE_SELINUX_P +struct selinux_state selinux_state __section(".iee.selinux"); +#else struct selinux_state selinux_state; +#endif /* SECMARK reference count */ static atomic_t selinux_secmark_refcount = ATOMIC_INIT(0); @@ -7429,15 +7437,51 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { #endif }; +#ifdef CONFIG_IEE_SELINUX_P +struct kmem_cache *policy_jar; + +static void policy_cache_init(void) +{ + struct selinux_policy *unused; + + policy_jar = kmem_cache_create("policy_jar", sizeof(struct selinux_policy), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + /* Test this cache */ + unused = kmem_cache_alloc(policy_jar, GFP_KERNEL); + kmem_cache_free(policy_jar, unused); + + if (haoc_enabled) + pr_info("IEE SELINUXP: policy cache created."); +} +#endif + static __init int selinux_init(void) { pr_info("SELinux: Initializing.\n"); memset(&selinux_state, 0, sizeof(selinux_state)); +#ifdef CONFIG_IEE_SELINUX_P + WRITE_ONCE(selinux_state.enforcing, selinux_enforcing_boot); +#else enforcing_set(selinux_enforcing_boot); +#endif selinux_avc_init(); +#ifdef CONFIG_IEE_SELINUX_P + /* Put selinux_status inside IEE. */ + /* Prepare mutex lock and write the ptr to mutex->owner. */ + struct mutex *status_lock = kzalloc(GFP_KERNEL, sizeof(struct mutex)); + struct mutex *policy_mutex = kzalloc(GFP_KERNEL, sizeof(struct mutex)); + + mutex_init(status_lock); + mutex_init(policy_mutex); + selinux_state.status_lock.owner.counter = (s64)status_lock; + selinux_state.policy_mutex.owner.counter = (s64)policy_mutex; + + policy_cache_init(); +#else mutex_init(&selinux_state.status_lock); mutex_init(&selinux_state.policy_mutex); +#endif /* Set the security state for the initial task. */ cred_init_security(); diff --git a/security/selinux/ima.c b/security/selinux/ima.c index aa34da9b0aebd..498a1f9c8fd18 100644 --- a/security/selinux/ima.c +++ b/security/selinux/ima.c @@ -9,6 +9,9 @@ */ #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif #include "security.h" #include "ima.h" @@ -74,7 +77,11 @@ void selinux_ima_measure_state_locked(void) size_t policy_len; int rc = 0; +#ifdef CONFIG_IEE_SELINUX_P + lockdep_assert_held(iee_get_selinux_policy_lock()); +#else lockdep_assert_held(&selinux_state.policy_mutex); +#endif state_str = selinux_ima_collect_state(); if (!state_str) { @@ -112,9 +119,17 @@ void selinux_ima_measure_state_locked(void) */ void selinux_ima_measure_state(void) { +#ifdef CONFIG_IEE_SELINUX_P + lockdep_assert_not_held(iee_get_selinux_policy_lock()); + + mutex_lock(iee_get_selinux_policy_lock()); + selinux_ima_measure_state_locked(); + mutex_unlock(iee_get_selinux_policy_lock()); +#else lockdep_assert_not_held(&selinux_state.policy_mutex); mutex_lock(&selinux_state.policy_mutex); selinux_ima_measure_state_locked(); mutex_unlock(&selinux_state.policy_mutex); +#endif } diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index a9de89af8fdc5..8ba4c3c95a50e 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -113,11 +113,13 @@ static inline bool selinux_initialized(void) return smp_load_acquire(&selinux_state.initialized); } +#ifndef CONFIG_IEE_SELINUX_P static inline void selinux_mark_initialized(void) { /* do a synchronized write to avoid race conditions */ smp_store_release(&selinux_state.initialized, true); } +#endif #ifdef CONFIG_SECURITY_SELINUX_DEVELOP static inline bool enforcing_enabled(void) @@ -125,10 +127,13 @@ static inline bool enforcing_enabled(void) return READ_ONCE(selinux_state.enforcing); } +#ifndef CONFIG_IEE_SELINUX_P static inline void enforcing_set(bool value) { WRITE_ONCE(selinux_state.enforcing, value); } +#endif + #else static inline bool enforcing_enabled(void) { diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c index 3e28fa2444fbd..00621fb1f979e 100644 --- a/security/selinux/selinuxfs.c +++ b/security/selinux/selinuxfs.c @@ -31,6 +31,9 @@ #include #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif /* selinuxfs pseudo filesystem for exporting the security policy API. Based on the proc code and the fs/nfsd/nfsctl.c code. */ @@ -349,7 +352,11 @@ static int sel_open_policy(struct inode *inode, struct file *filp) BUG_ON(filp->private_data); +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif rc = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__READ_POLICY, NULL); @@ -379,11 +386,19 @@ static int sel_open_policy(struct inode *inode, struct file *filp) filp->private_data = plm; +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif return 0; err: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif if (plm) vfree(plm->data); @@ -585,7 +600,11 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, goto out; } +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif length = security_load_policy(data, count, &load_state); if (length) { pr_warn_ratelimited("SELinux: failed to load policy\n"); @@ -607,7 +626,11 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, audit_get_sessionid(current)); out_unlock: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif out: vfree(data); return length; @@ -1192,7 +1215,11 @@ static ssize_t sel_read_bool(struct file *filep, char __user *buf, unsigned index = file_inode(filep)->i_ino & SEL_INO_MASK; const char *name = filep->f_path.dentry->d_name.name; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif ret = -EINVAL; if (index >= fsi->bool_num || strcmp(name, @@ -1211,14 +1238,22 @@ static ssize_t sel_read_bool(struct file *filep, char __user *buf, } length = scnprintf(page, PAGE_SIZE, "%d %d", cur_enforcing, fsi->bool_pending_values[index]); +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif ret = simple_read_from_buffer(buf, count, ppos, page, length); out_free: free_page((unsigned long)page); return ret; out_unlock: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif goto out_free; } @@ -1243,7 +1278,11 @@ static ssize_t sel_write_bool(struct file *filep, const char __user *buf, if (IS_ERR(page)) return PTR_ERR(page); +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif length = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__SETBOOL, @@ -1267,7 +1306,11 @@ static ssize_t sel_write_bool(struct file *filep, const char __user *buf, length = count; out: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif kfree(page); return length; } @@ -1298,7 +1341,11 @@ static ssize_t sel_commit_bools_write(struct file *filep, if (IS_ERR(page)) return PTR_ERR(page); +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif length = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__SETBOOL, @@ -1319,7 +1366,11 @@ static ssize_t sel_commit_bools_write(struct file *filep, length = count; out: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif kfree(page); return length; } diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 03427046c8242..bf1068f0f7645 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -49,6 +49,9 @@ #include #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif #include "flask.h" #include "avc.h" @@ -2114,9 +2117,14 @@ static void security_load_policycaps(struct selinux_policy *policy) p = &policy->policydb; +#ifdef CONFIG_IEE_SELINUX_P + for (i = 0; i < ARRAY_SIZE(selinux_state.policycap); i++) + iee_set_sel_policy_cap(i, ebitmap_get_bit(&p->policycaps, i)); +#else for (i = 0; i < ARRAY_SIZE(selinux_state.policycap); i++) WRITE_ONCE(selinux_state.policycap[i], ebitmap_get_bit(&p->policycaps, i)); +#endif for (i = 0; i < ARRAY_SIZE(selinux_policycap_names); i++) pr_info("SELinux: policy capability %s=%d\n", @@ -2179,6 +2187,9 @@ void selinux_policy_commit(struct selinux_load_state *load_state) { struct selinux_state *state = &selinux_state; struct selinux_policy *oldpolicy, *newpolicy = load_state->policy; +#ifdef CONFIG_IEE_SELINUX_P + struct selinux_policy *temppolicy; +#endif unsigned long flags; u32 seqno; @@ -2203,10 +2214,20 @@ void selinux_policy_commit(struct selinux_load_state *load_state) /* Install the new policy. */ if (oldpolicy) { sidtab_freeze_begin(oldpolicy->sidtab, &flags); +#ifdef CONFIG_IEE_SELINUX_P + iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); + kfree(newpolicy); +#else rcu_assign_pointer(state->policy, newpolicy); +#endif sidtab_freeze_end(oldpolicy->sidtab, &flags); } else { +#ifdef CONFIG_IEE_SELINUX_P + iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); + kfree(newpolicy); +#else rcu_assign_pointer(state->policy, newpolicy); +#endif } /* Load the policycaps from the new policy */ @@ -2224,7 +2245,20 @@ void selinux_policy_commit(struct selinux_load_state *load_state) /* Free the old policy */ synchronize_rcu(); +#ifdef CONFIG_IEE_SELINUX_P + /* + * Normal free process includes setting freed objects pointers to be NULL, however it + * would be hard as old policy is already inside IEE. So Make a kernel copy of the old + * policy to free objects it points to. + */ + if (oldpolicy) { + temppolicy = kmemdup(oldpolicy, sizeof(*temppolicy), GFP_KERNEL); + selinux_policy_free(temppolicy); + kfree(oldpolicy); + } +#else selinux_policy_free(oldpolicy); +#endif kfree(load_state->convert_data); /* Notify others of the policy change */ @@ -3022,6 +3056,9 @@ int security_set_bools(u32 len, int *values) { struct selinux_state *state = &selinux_state; struct selinux_policy *newpolicy, *oldpolicy; +#ifdef CONFIG_IEE_SELINUX_P + struct selinux_policy *temppolicy; +#endif int rc; u32 i, seqno = 0; @@ -3075,7 +3112,12 @@ int security_set_bools(u32 len, int *values) seqno = newpolicy->latest_granting; /* Install the new policy */ +#ifdef CONFIG_IEE_SELINUX_P + iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); + kfree(newpolicy); +#else rcu_assign_pointer(state->policy, newpolicy); +#endif /* * Free the conditional portions of the old policydb @@ -3083,7 +3125,20 @@ int security_set_bools(u32 len, int *values) * structure itself but not what it references. */ synchronize_rcu(); +#ifdef CONFIG_IEE_SELINUX_P + /* + * Normal free process includes setting freed objects pointers to be NULL, however it + * would be hard as old policy is already inside IEE. So Make a kernel copy of the old + * policy to free objects it points to. + */ + temppolicy = kmemdup(oldpolicy, sizeof(*temppolicy), GFP_KERNEL); + if (!temppolicy) + return -ENOMEM; + selinux_policy_cond_free(temppolicy); + kfree(oldpolicy); +#else selinux_policy_cond_free(oldpolicy); +#endif /* Notify others of the policy change */ selinux_notify_policy_change(seqno); diff --git a/security/selinux/status.c b/security/selinux/status.c index dffca22ce6f74..25a4589c261b7 100644 --- a/security/selinux/status.c +++ b/security/selinux/status.c @@ -10,6 +10,9 @@ #include #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif #include "avc.h" #include "security.h" @@ -44,9 +47,17 @@ struct page *selinux_kernel_status_page(void) struct selinux_kernel_status *status; struct page *result = NULL; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_status_lock()); +#else mutex_lock(&selinux_state.status_lock); +#endif if (!selinux_state.status_page) { +#ifdef CONFIG_IEE_SELINUX_P + iee_set_selinux_status_pg(alloc_page(GFP_KERNEL|__GFP_ZERO)); +#else selinux_state.status_page = alloc_page(GFP_KERNEL|__GFP_ZERO); +#endif if (selinux_state.status_page) { status = page_address(selinux_state.status_page); @@ -66,7 +77,11 @@ struct page *selinux_kernel_status_page(void) } } result = selinux_state.status_page; +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_status_lock()); +#else mutex_unlock(&selinux_state.status_lock); +#endif return result; } @@ -80,7 +95,11 @@ void selinux_status_update_setenforce(bool enforcing) { struct selinux_kernel_status *status; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_status_lock()); +#else mutex_lock(&selinux_state.status_lock); +#endif if (selinux_state.status_page) { status = page_address(selinux_state.status_page); @@ -92,7 +111,11 @@ void selinux_status_update_setenforce(bool enforcing) smp_wmb(); status->sequence++; } +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_status_lock()); +#else mutex_unlock(&selinux_state.status_lock); +#endif } /* @@ -105,7 +128,11 @@ void selinux_status_update_policyload(u32 seqno) { struct selinux_kernel_status *status; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_status_lock()); +#else mutex_lock(&selinux_state.status_lock); +#endif if (selinux_state.status_page) { status = page_address(selinux_state.status_page); @@ -118,5 +145,9 @@ void selinux_status_update_policyload(u32 seqno) smp_wmb(); status->sequence++; } +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_status_lock()); +#else mutex_unlock(&selinux_state.status_lock); +#endif } From 9f1619c69f8e5d29a095f5364d26b92a4ca3c522 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Thu, 9 Jul 2026 22:03:03 +0800 Subject: [PATCH 11/26] HAOC3.0/arm64/SELINUXP: Add support for arm64 SELinux Protection Support selinux protection for arm64. Co-authored-by: zhangshiyangB7 Signed-off-by: Liu Zhehui --- .../configs/deepin_arm64_desktop_defconfig | 1 + arch/arm64/include/asm/haoc/haoc-def.h | 7 +++ arch/arm64/include/asm/haoc/haoc.h | 12 +++++ arch/arm64/include/asm/haoc/iee-init.h | 4 ++ arch/arm64/include/asm/haoc/iee-selinux.h | 49 +++++++++++++++++ arch/arm64/kernel/haoc/Kconfig | 10 ++++ arch/arm64/kernel/haoc/Makefile | 3 ++ arch/arm64/kernel/haoc/haoc.c | 7 +++ arch/arm64/kernel/haoc/selinuxp/Makefile | 17 ++++++ arch/arm64/kernel/haoc/selinuxp/selinuxp.c | 53 +++++++++++++++++++ arch/arm64/kernel/vmlinux.lds.S | 12 +++++ 11 files changed, 175 insertions(+) create mode 100644 arch/arm64/include/asm/haoc/iee-selinux.h create mode 100644 arch/arm64/kernel/haoc/selinuxp/Makefile create mode 100644 arch/arm64/kernel/haoc/selinuxp/selinuxp.c diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index 1e83be50259ec..1808b06e60674 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -166,6 +166,7 @@ CONFIG_IEE_PTRP=y CONFIG_IEE_SIP=y CONFIG_CREDP=y CONFIG_KEYP=y +CONFIG_IEE_SELINUX_P=y CONFIG_PTP=y CONFIG_PTP_RESERVE_ORDER=12 CONFIG_KPROBES=y diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index 73d00df63aa01..e1e20940cced5 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -103,6 +103,13 @@ enum { IEE_OP_SET_KEY_DESCRIPTION, IEE_OP_SET_KEY_RESTRICT_LINK, IEE_OP_SET_KEY_FLAG_BIT, +#endif +#ifdef CONFIG_IEE_SELINUX_P + IEE_SEL_SET_STATUS_PG, + IEE_SEL_SET_ENFORCING, + IEE_SEL_SET_INITIALIZED, + IEE_SEL_SET_POLICY_CAP, + IEE_SEL_RCU_ASSIGN_POLICY, #endif IEE_FLAG_END }; diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index 684557d65ddcf..ea81e638ab64b 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -150,4 +150,16 @@ void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, struct key_restriction *restrict_link); unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); #endif + +#ifdef CONFIG_IEE_SELINUX_P +struct page; +struct selinux_policy; + +void _iee_set_selinux_status_pg(unsigned long __unused, struct page *new_page); +void _iee_set_selinux_enforcing(unsigned long __unused, bool value); +void _iee_mark_selinux_initialized(unsigned long __unused); +void _iee_set_sel_policy_cap(unsigned long __unused, unsigned int idx, int cap); +void _iee_sel_rcu_assign_policy(unsigned long __unused, + struct selinux_policy *new_policy, struct selinux_policy *iee_new_policy); +#endif #endif diff --git a/arch/arm64/include/asm/haoc/iee-init.h b/arch/arm64/include/asm/haoc/iee-init.h index 2a724f05b763b..cc27b237ef112 100644 --- a/arch/arm64/include/asm/haoc/iee-init.h +++ b/arch/arm64/include/asm/haoc/iee-init.h @@ -27,6 +27,10 @@ extern char __iee_ptp_data_end[]; extern char __iee_cred_data_start[]; extern char __iee_cred_data_end[]; #endif +#ifdef CONFIG_IEE_SELINUX_P +extern char __iee_selinux_data_start[]; +extern char __iee_selinux_data_end[]; +#endif extern struct mutex fixmap_lock; #endif diff --git a/arch/arm64/include/asm/haoc/iee-selinux.h b/arch/arm64/include/asm/haoc/iee-selinux.h new file mode 100644 index 0000000000000..d169d23790e45 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-selinux.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_SELINUX_P_H +#define _LINUX_IEE_SELINUX_P_H + +#include +#include +#include "security.h" +#include "ss/services.h" + +extern struct kmem_cache *policy_jar; +extern unsigned long long iee_rw_gate(int flag, ...); + +static inline struct mutex *iee_get_selinux_policy_lock(void) +{ + return (struct mutex *)(selinux_state.policy_mutex.owner.counter); +} + +static inline struct mutex *iee_get_selinux_status_lock(void) +{ + return (struct mutex *)(selinux_state.status_lock.owner.counter); +} + +static inline void iee_set_selinux_status_pg(struct page *new_page) +{ + iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); +} + +static inline void enforcing_set(bool value) +{ + iee_rw_gate(IEE_SEL_SET_ENFORCING, value); +} + +static inline void selinux_mark_initialized(void) +{ + iee_rw_gate(IEE_SEL_SET_INITIALIZED); +} + +static inline void iee_set_sel_policy_cap(unsigned int idx, int cap) +{ + iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); +} + +static inline void iee_sel_rcu_assign_policy(struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); +} + +#endif diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index 14b3a9021162c..a1fe16d804e28 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -83,4 +83,14 @@ config KEYP of keys must be made and verified by IEE APIs to defend memory corruptions. If unsure, say N. + +config IEE_SELINUX_P + bool "SELinux Protection(IEE_SELINUX_P)" + help + Simple enhancement of the famous linux security module SELinux by putting + several critical data, like selinux policy, into the IEE region. + + If unsure, say N. + depends on SECURITY_SELINUX + depends on IEE endmenu # HAOC diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile index 3860197b26f8e..723dc21ef15ef 100644 --- a/arch/arm64/kernel/haoc/Makefile +++ b/arch/arm64/kernel/haoc/Makefile @@ -4,3 +4,6 @@ obj-y += iee/ obj-$(CONFIG_CREDP) += credp/ obj-$(CONFIG_PTP) += ptp/ obj-$(CONFIG_KEYP) += keyp/ + +ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index f257cb2ecbbee..249bf792aab4c 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -110,6 +110,13 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_key_description, (iee_func)_iee_set_key_restrict_link, (iee_func)_iee_set_key_flag_bit, +#endif +#ifdef CONFIG_IEE_SELINUX_P + (iee_func)_iee_set_selinux_status_pg, + (iee_func)_iee_set_selinux_enforcing, + (iee_func)_iee_mark_selinux_initialized, + (iee_func)_iee_set_sel_policy_cap, + (iee_func)_iee_sel_rcu_assign_policy, #endif NULL }; diff --git a/arch/arm64/kernel/haoc/selinuxp/Makefile b/arch/arm64/kernel/haoc/selinuxp/Makefile new file mode 100644 index 0000000000000..0e5b0be4376c8 --- /dev/null +++ b/arch/arm64/kernel/haoc/selinuxp/Makefile @@ -0,0 +1,17 @@ +ccflags-y += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +ccflags-y += -I$(objtree)/security/selinux +obj-y += selinuxp.o + +ifdef CONFIG_IEE_SELINUX_P +SELINUX_GENHDRS := $(objtree)/security/selinux/flask.h \ + $(objtree)/security/selinux/av_permissions.h + +quiet_cmd_haoc_genhdrs = GEN $(SELINUX_GENHDRS) + cmd_haoc_genhdrs = mkdir -p $(objtree)/security/selinux; \ + $< $(SELINUX_GENHDRS) + +$(SELINUX_GENHDRS): scripts/selinux/genheaders/genheaders FORCE + $(call if_changed,haoc_genhdrs) + +$(obj)/selinuxp.o: $(SELINUX_GENHDRS) +endif diff --git a/arch/arm64/kernel/haoc/selinuxp/selinuxp.c b/arch/arm64/kernel/haoc/selinuxp/selinuxp.c new file mode 100644 index 0000000000000..ce4a89f8f7adf --- /dev/null +++ b/arch/arm64/kernel/haoc/selinuxp/selinuxp.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "security.h" +#include "ss/services.h" +#include + +static inline void iee_verify_selinux_type(unsigned long va) +{ + iee_verify_type(va, IEE_SELINUX, "selinux"); +} + +void __iee_code _iee_set_selinux_status_pg(unsigned long iee_offset, + struct page *new_page) +{ + *__ptr_to_iee(&selinux_state.status_page) = new_page; +} + +void __iee_code _iee_set_selinux_enforcing(unsigned long iee_offset, bool value) +{ + *__ptr_to_iee(&selinux_state.enforcing) = value; +} + +void __iee_code _iee_mark_selinux_initialized(unsigned long iee_offset) +{ + /* do a synchronized write to avoid race conditions */ + smp_store_release(__ptr_to_iee(&(selinux_state.initialized)), true); + if (haoc_enabled) + pr_info("HAOC: Mark selinux initialized."); +} + +void __iee_code _iee_set_sel_policy_cap(unsigned long iee_offset, + unsigned int idx, int cap) +{ + *__ptr_to_iee(&(selinux_state.policycap[idx])) = cap; +} + +/* + * Please make sure param iee_new_policy is from policy_jar memcache. + * Need to free new_policy after calling this func as it's only used to + * trans data from kernel. + */ +void __iee_code _iee_sel_rcu_assign_policy(unsigned long iee_offset, + struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + /* TODO: Verify information from incoming policy. */ + iee_verify_selinux_type((unsigned long)iee_new_policy); + + /* Copy data from kernel to new allocated policy struct inside iee. */ + memcpy(__ptr_to_iee(iee_new_policy), new_policy, sizeof(struct selinux_policy)); + rcu_assign_pointer(*__ptr_to_iee(&selinux_state.policy), iee_new_policy); +} diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 40b75a1c486da..15541e91742df 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -171,6 +171,17 @@ jiffies = jiffies_64; #define CRED_DATA #endif +#ifdef CONFIG_IEE_SELINUX_P +#define IEE_SELINUX_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_start = .; \ + *(.iee.selinux) \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_end = .; +#else +#define IEE_SELINUX_DATA +#endif + #ifdef CONFIG_IEE #define IEE_INIT_DATA \ .iee.data : { \ @@ -178,6 +189,7 @@ jiffies = jiffies_64; iee_init_data_begin = .; \ PTP_DATA \ CRED_DATA \ + IEE_SELINUX_DATA \ . = ALIGN(PAGE_SIZE); \ iee_init_data_end = .; \ } From 30d1947f28c8d52e2bf6e8f8f17305d488dd3045 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Fri, 10 Jul 2026 00:34:56 +0800 Subject: [PATCH 12/26] HAOC4.0/x86_64/PTRP_W: Add support for x86 Pointer Write-Only Protection Clarify IEE_PTRP and IEE_PTRP_W Kconfig help on x86. Drop separate IEE_IO_CHECK and IEE_CS_CHECK gates in generic paths so read-side iee_verify_token() runs whenever PTRP is enabled and PTRP_W is off. Co-authored-by: szh Signed-off-by: Liu Zhehui --- arch/x86/Kconfig | 16 ++- arch/x86/configs/deepin_x86_desktop_defconfig | 1 + arch/x86/include/asm/haoc/iee-cred.h | 6 +- arch/x86/include/asm/haoc/iee-token.h | 25 ++--- arch/x86/kernel/haoc/credp/credp.c | 13 +++ arch/x86/kernel/haoc/iee/iee-token.c | 102 ++++++++++++++++++ fs/exec.c | 10 ++ fs/ioctl.c | 14 +++ fs/read_write.c | 33 ++++++ kernel/fork.c | 15 ++- kernel/sched/core.c | 10 +- 11 files changed, 221 insertions(+), 24 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index ad824af795cbd..63343721f5dad 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1574,10 +1574,18 @@ config IEE_PTRP bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE help - Provide IEE matadata for each process called task_token to allow - verifing pointers inside task_struct, like struct cred* that determines - the capabilities of a process. - Could be a enhancement of other sub-module of HAOC. + Enable task_token pointer protection for IEE. When the write-only + sub-option IEE_PTRP_W is disabled, both read and write checks are + active. When IEE_PTRP_W is enabled, only write protection is active. + +config IEE_PTRP_W + bool "Pointer write-only protection (IEE_PTRP_W)" + depends on IEE_PTRP + help + When enabled, only write protection is performed, without read-side + verification. Disable this option to enable full read and write checks + under IEE_PTRP. This mode keeps pointer writes mediated by IEE while + avoiding read-side verification overhead. config IEE_SIP bool "Sensitive Instruction Protection for IEE(IEE_SIP)" diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index f968005937b9a..4e50022875b12 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -90,6 +90,7 @@ CONFIG_X86_CPUID=m CONFIG_AMD_MEM_ENCRYPT=y CONFIG_IEE=y CONFIG_IEE_PTRP=y +CONFIG_IEE_PTRP_W=y CONFIG_IEE_SIP=y CONFIG_CREDP=y CONFIG_KEYP=y diff --git a/arch/x86/include/asm/haoc/iee-cred.h b/arch/x86/include/asm/haoc/iee-cred.h index fa6a3a1d81a24..1bb1b84de4c33 100644 --- a/arch/x86/include/asm/haoc/iee-cred.h +++ b/arch/x86/include/asm/haoc/iee-cred.h @@ -22,7 +22,7 @@ static inline void iee_detect_cred_uaf(struct cred *cred, const char *caller) atomic_long_read(&cred->usage)); } -#ifdef CONFIG_IEE_PTRP +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) static inline void __iee_verify_cred(void) { struct task_struct *curr_task; @@ -64,10 +64,10 @@ static inline void iee_verify_update_cred(struct cred *new) if (token->new_cred != new) panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); } -#else +#else /* !CONFIG_IEE_PTRP || CONFIG_IEE_PTRP_W */ static inline void iee_verify_cred(void) { } static inline void iee_verify_update_cred(struct cred *new) { } -#endif +#endif /* CONFIG_IEE_PTRP && !CONFIG_IEE_PTRP_W */ static void __maybe_unused iee_copy_cred(struct cred *new) { diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h index 7ddd13acb97fd..2045a9ae7874c 100644 --- a/arch/x86/include/asm/haoc/iee-token.h +++ b/arch/x86/include/asm/haoc/iee-token.h @@ -6,6 +6,7 @@ #include #include #include +#include extern unsigned long long iee_rw_gate(int flag, ...); @@ -42,23 +43,13 @@ struct task_token { const struct cred *new_cred; /* The valid target for commit_creds. */ const struct cred *curr_cred; /* The current subjective credentials. */ #endif +#ifdef CONFIG_IEE_PTRP + seqcount_t seq; +#endif }; #endif /* CONFIG_IEE */ #ifdef CONFIG_IEE_PTRP -static inline void iee_verify_pgd(struct task_struct *tsk) -{ - struct task_token *token; - - if (tsk == &init_task) - return; - - token = (struct task_token *)__addr_to_token(tsk); - if (token->pgd != tsk->mm->pgd) - panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", - (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); -} - static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) { iee_rw_gate(IEE_OP_SET_TOKEN_PGD, tsk, pgd); @@ -73,6 +64,12 @@ static inline void iee_validate_token(struct task_struct *tsk) { iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); } -#endif /* CONFIG_IEE_PTRP */ +#if !defined(CONFIG_IEE_PTRP_W) +void iee_verify_pgd(struct task_struct *next); +void iee_verify_token(struct task_struct *tsk); #endif + +#endif /* CONFIG_IEE_PTRP */ + +#endif /* _LINUX_IEE_TOKEN_H */ diff --git a/arch/x86/kernel/haoc/credp/credp.c b/arch/x86/kernel/haoc/credp/credp.c index f131df77ec255..cfc8cf115ae7b 100644 --- a/arch/x86/kernel/haoc/credp/credp.c +++ b/arch/x86/kernel/haoc/credp/credp.c @@ -310,7 +310,12 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); /* task->cred shall be updated once. */ token->new_cred = NULL; + write_seqcount_begin(&token->seq); token->curr_cred = new; + rcu_assign_pointer(task->real_cred, new); + rcu_assign_pointer(task->cred, new); + write_seqcount_end(&token->seq); + return; } #endif @@ -393,7 +398,11 @@ void _iee_override_creds(unsigned long __unused, const struct cred *new) if (haoc_init_done) { struct task_token *token = (struct task_token *)__addr_to_token(current); + write_seqcount_begin(&token->seq); token->curr_cred = new; + rcu_assign_pointer(current->cred, new); + write_seqcount_end(&token->seq); + return; } #endif @@ -409,7 +418,11 @@ void _iee_revert_creds(unsigned long __unused, const struct cred *old) if (haoc_init_done) { struct task_token *token = (struct task_token *)__addr_to_token(current); + write_seqcount_begin(&token->seq); token->curr_cred = old; + rcu_assign_pointer(current->cred, old); + write_seqcount_end(&token->seq); + return; } #endif diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c index a25aeac727cc5..d1eceb58c102d 100644 --- a/arch/x86/kernel/haoc/iee/iee-token.c +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -1,6 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include +#include +#include +#include +#include #include "slab.h" void iee_set_token_page_valid(unsigned long token, unsigned long token_page, @@ -141,6 +146,7 @@ void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) token->new_cred = NULL; token->curr_cred = NULL; #endif + seqcount_init(&token->seq); token->valid = false; } @@ -154,6 +160,7 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) token->new_cred = NULL; token->curr_cred = tsk->cred; #endif + seqcount_init(&token->seq); token->valid = true; } @@ -198,3 +205,98 @@ void _iee_set_token(unsigned long __unused, pte_t *token_ptep, } } #endif + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +void iee_verify_token(struct task_struct *tsk) +{ + struct task_token *token; +#ifdef CONFIG_CREDP + const struct cred *old_cred; + const struct cred *token_cred; +#endif + bool valid; + unsigned int seq; + + if (unlikely(!haoc_init_done)) + return; + + if (!haoc_enabled) + return; + + if (tsk == &init_task) + return; + + token = (struct task_token *)__addr_to_token(tsk); + do { + seq = read_seqcount_begin(&token->seq); + valid = token->valid; +#ifdef CONFIG_CREDP + old_cred = tsk->cred; + token_cred = token->curr_cred; +#endif + } while (read_seqcount_retry(&token->seq, seq)); + + if (!valid) + panic("IEE: (%s) Invalid Token.", __func__); +#ifdef CONFIG_CREDP + if (token_cred != old_cred) + panic("IEE: (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx", + __func__, (u64)token_cred, (u64)old_cred); +#endif +} + +static void check_all_threads(void) +{ + struct task_struct *task; + + rcu_read_lock(); + for_each_process(task) + iee_verify_token(task); + rcu_read_unlock(); +} + +static int checker_thread(void *data) +{ + int check_interval_ms = 500; + + while (!kthread_should_stop()) { + check_all_threads(); + msleep_interruptible(check_interval_ms); + } + pr_info("[IEE] Kernel thread exiting\n"); + return 0; +} + +static int __init thread_checker_init(void) +{ + struct task_struct *checker_task; + + if (!haoc_enabled) + return 0; + + pr_info("IEE: Initializing thread checker\n"); + checker_task = kthread_run(checker_thread, NULL, "thread_credp_cycle"); + if (IS_ERR(checker_task)) { + pr_err("IEE: Failed to create thread checker task: %ld\n", + PTR_ERR(checker_task)); + return PTR_ERR(checker_task); + } + + pr_info("IEE: Thread checker started successfully\n"); + return 0; +} + +late_initcall(thread_checker_init); + +void iee_verify_pgd(struct task_struct *next) +{ + if (haoc_enabled && next != &init_task) { + struct task_token *token; + + token = (struct task_token *)__addr_to_token(next); + if (token->pgd != next->mm->pgd) + panic("IEE Pgd Error: next_pgd: 0x%lx, token_pgd: 0x%lx", + (unsigned long)next->mm->pgd, (unsigned long)token->pgd); + } +} +#endif /* CONFIG_IEE_PTRP && !CONFIG_IEE_PTRP_W */ diff --git a/fs/exec.c b/fs/exec.c index 9466a12eb527c..1d1212cb56a4c 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -84,6 +84,10 @@ #ifdef CONFIG_CREDP #include #endif + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif #ifdef CONFIG_PTP #include #endif @@ -1975,6 +1979,12 @@ static int do_execveat_common(int fd, struct filename *filename, ptp_disable_wp(&cr0); #endif +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif /* * We move the actual failure in case of RLIMIT_NPROC excess from * set*uid() to execve() because too many poorly written programs diff --git a/fs/ioctl.c b/fs/ioctl.c index 85260a3598f89..60a277f3c3044 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -27,6 +27,14 @@ #include +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif +#ifdef CONFIG_IEE_PTRP +#include +#include +#endif + /* So that the fiemap access checks can't overflow on 32 bit machines. */ #define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) @@ -43,6 +51,12 @@ */ long vfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif int error = -ENOTTY; if (!filp->f_op->unlocked_ioctl) diff --git a/fs/read_write.c b/fs/read_write.c index 4771701c896ba..4881922a6602b 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -22,9 +22,18 @@ #include #include "internal.h" +#ifdef CONFIG_IEE_PTRP +#include +#include +#endif + #include #include +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif + const struct file_operations generic_ro_fops = { .llseek = generic_file_llseek, .read_iter = generic_file_read_iter, @@ -451,6 +460,12 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) { ssize_t ret; +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif if (!(file->f_mode & FMODE_READ)) return -EBADF; if (!(file->f_mode & FMODE_CAN_READ)) @@ -565,6 +580,12 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ { ssize_t ret; +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif if (!(file->f_mode & FMODE_WRITE)) return -EBADF; if (!(file->f_mode & FMODE_CAN_WRITE)) @@ -905,6 +926,12 @@ EXPORT_SYMBOL(vfs_iter_write); static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; struct iov_iter iter; @@ -922,6 +949,12 @@ static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; struct iov_iter iter; diff --git a/kernel/fork.c b/kernel/fork.c index ec554539b3d05..302bd9615ebe1 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -117,10 +117,17 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_IEE +#include +#endif #ifdef CONFIG_PTP #include #endif +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif + /* * Minimum number of threads to boot the kernel */ @@ -2392,8 +2399,14 @@ __latent_entropy struct task_struct *copy_process( if (!p) goto fork_out; #ifdef CONFIG_IEE_PTRP - if(haoc_enabled) + if (haoc_enabled) iee_validate_token(p); +#endif +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif #endif p->flags &= ~PF_KTHREAD; if (args->kthread) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f358980bff455..408c5d3decdd1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5179,6 +5179,12 @@ static __always_inline struct rq * context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(next); +# endif +#endif prepare_task_switch(rq, prev, next); /* @@ -5216,8 +5222,8 @@ context_switch(struct rq *rq, struct task_struct *prev, * case 'prev->active_mm == next->mm' through * finish_task_switch()'s mmdrop(). */ -#ifdef CONFIG_IEE_PTRP - if(haoc_enabled) +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) + if (haoc_enabled) iee_verify_pgd(next); #endif switch_mm_irqs_off(prev->active_mm, next->mm, next); From 6d04ea62b171f8a03c0eff0ea2684f1a267791a5 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Fri, 10 Jul 2026 00:40:10 +0800 Subject: [PATCH 13/26] HAOC4.0/arm64/PTRP_W: Add support for arm64 Pointer Write-Only Protection - HAOC4.0/arm64/CYCLE_CHECK: support CONFIG_IEE_PTRP_W & CONFIG_IEE_CYCLE_CHECK - HAOC4.0/arm64/CS_CHECK: support CONFIG_IEE_PTRP_W & CONFIG_IEE_CS_CHECK - HAOC4.0/arm64/IO_CHECK: support CONFIG_IEE_PTRP_W & CONFIG_IEE_IO_CHECK Signed-off-by: Liu Zhehui --- .../configs/deepin_arm64_desktop_defconfig | 1 + arch/arm64/include/asm/haoc/iee-cred.h | 2 +- arch/arm64/include/asm/haoc/iee-token.h | 4 + arch/arm64/kernel/haoc/Kconfig | 46 ++++++- arch/arm64/kernel/haoc/credp/credp.c | 120 ++++++++++++++++++ arch/arm64/kernel/haoc/iee/iee-token.c | 4 +- arch/arm64/mm/fault.c | 8 +- fs/exec.c | 9 +- fs/ioctl.c | 7 +- fs/read_write.c | 28 +++- kernel/fork.c | 7 +- kernel/sched/core.c | 13 +- 12 files changed, 234 insertions(+), 15 deletions(-) diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index 1808b06e60674..fd5288df06deb 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -163,6 +163,7 @@ CONFIG_VIRTUALIZATION=y CONFIG_KVM=y CONFIG_IEE=y CONFIG_IEE_PTRP=y +CONFIG_IEE_PTRP_W=y CONFIG_IEE_SIP=y CONFIG_CREDP=y CONFIG_KEYP=y diff --git a/arch/arm64/include/asm/haoc/iee-cred.h b/arch/arm64/include/asm/haoc/iee-cred.h index 9c14fed8da9a3..7384515c4512d 100644 --- a/arch/arm64/include/asm/haoc/iee-cred.h +++ b/arch/arm64/include/asm/haoc/iee-cred.h @@ -20,7 +20,7 @@ static inline void iee_detect_cred_uaf(struct cred *cred, const char *caller) atomic_long_read(&cred->usage)); } -#ifdef CONFIG_IEE_PTRP +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) static inline void iee_verify_cred(void) { struct task_struct *curr_task; diff --git a/arch/arm64/include/asm/haoc/iee-token.h b/arch/arm64/include/asm/haoc/iee-token.h index 4bde51ae4926a..df1c4f947d294 100644 --- a/arch/arm64/include/asm/haoc/iee-token.h +++ b/arch/arm64/include/asm/haoc/iee-token.h @@ -4,6 +4,7 @@ #include #include +#include #ifdef CONFIG_IEE_PTRP extern struct kmem_cache *task_struct_cachep; @@ -41,6 +42,9 @@ struct task_token { const struct cred *new_cred; /* The valid target for commit_creds. */ const struct cred *curr_cred; /* The current subjective credentials. */ #endif +#ifdef CONFIG_IEE_PTRP + seqcount_t seq; +#endif }; #endif /* CONFIG_IEE */ diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index a1fe16d804e28..4babc31a82282 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -38,10 +38,48 @@ config IEE_PTRP bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE help - Provide IEE matadata for each process called task_token to allow - verifing pointers inside task_struct, like struct cred* that determines - the capabilities of a process. - Could be a enhancement of other sub-module of HAOC. + Enable task_token pointer protection for IEE. When the write-only + sub-option IEE_PTRP_W is disabled, both read and write checks are + active. When IEE_PTRP_W is enabled, only write protection is active. + +config IEE_PTRP_W + bool "Pointer Write-Only Protection for IEE to improve performance.(IEE_PTRP_W)" + depends on IEE_PTRP + help + Compared to IEE_PTRP, this option only performs write protection on pointers. + Enabling this option improves performance and reduces the protection surface. + Disable this option when read-side pointer verification is also required. + Use the full IEE_PTRP mode when both read and write checks are needed. + +config IEE_CYCLE_CHECK + bool "Cycle check all process cred struct.(IEE_CYCLE_CHECK)" + depends on IEE_PTRP && !IEE_PTRP_W && CREDP + def_bool y + help + Perform periodic read protection checks on all processes. This mode + scans task credentials while pointer write-only protection is disabled. + It is enabled by default when credential protection can use read checks. + Disable pointer write-only protection to keep this periodic scan active. + +config IEE_CS_CHECK + bool "Context-Switch check cred struct.(CONFIG_IEE_CS_CHECK)" + depends on IEE_PTRP && !IEE_PTRP_W && CREDP + def_bool y + help + Perform read protection checks on processes during context switching. + This mode validates task credentials at context-switch boundaries. + It is enabled by default when credential protection can use read checks. + Disable pointer write-only protection to keep context-switch checks active. + +config IEE_IO_CHECK + bool "IO check cred struct.(CONFIG_IEE_IO_CHECK)" + depends on IEE_PTRP && !IEE_PTRP_W && CREDP + def_bool n + help + Perform read protection checks on the process during I/O operations. + This mode validates task credentials when selected I/O paths run. + It is disabled by default and can be enabled for additional coverage. + Disable pointer write-only protection before enabling this read check. config IEE_SIP bool "Sensitive Instruction Protection for IEE(IEE_SIP)" diff --git a/arch/arm64/kernel/haoc/credp/credp.c b/arch/arm64/kernel/haoc/credp/credp.c index 5ca7fc7968de5..f54c3e05e0cbf 100644 --- a/arch/arm64/kernel/haoc/credp/credp.c +++ b/arch/arm64/kernel/haoc/credp/credp.c @@ -6,8 +6,115 @@ #include #include #include +#ifdef CONFIG_IEE_PTRP +#include +#include +#include +#include +#include +#endif extern struct cred init_cred; +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +void iee_cycle_verify_cred(struct task_struct *current_task) +{ + struct task_struct *task = current_task; + struct task_token *token = (struct task_token *)iee_get_task_token(task); + const struct cred *old_cred; + const struct cred *token_cred; + bool valid; + unsigned int seq; + + do { + seq = read_seqcount_begin(&token->seq); + valid = token->valid; + old_cred = task->cred; + token_cred = token->curr_cred; + } while (read_seqcount_retry(&token->seq, seq)); + + if (!valid) + panic("IEE: check_all_threads (%s) Invalid Token.", __func__); + if (task != &init_task && token_cred != old_cred) + panic("IEE: check_all_threads (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx", + __func__, (u64)token_cred, (u64)old_cred); +} + +static int iee_cycle_verify_cred_in_thread(struct task_struct *current_task) +{ + struct task_struct *task = current_task; + const struct cred *old_cred = task->cred; + struct task_token *token = (struct task_token *)iee_get_task_token(task); + + if (!token->valid) + return 1; + if (task != &init_task && token->curr_cred != old_cred) + return 2; + return 0; +} + +#ifdef CONFIG_IEE_CYCLE_CHECK +static void check_all_threads(void) +{ + struct task_struct *task; + + rcu_read_lock(); + for_each_process(task) { + unsigned int seq; + int ret; + struct task_token *token = iee_get_task_token(task); + + if (!token) + continue; + + do { + seq = read_seqcount_begin(&token->seq); + ret = iee_cycle_verify_cred_in_thread(task); + } while (read_seqcount_retry(&token->seq, seq)); + + if (ret == 1) + panic("IEE: check_all_threads (%s) Invalid Token.", __func__); + if (ret == 2) + panic("IEE: check_all_threads (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx", + __func__, (u64)token->curr_cred, (u64)task->cred); + } + rcu_read_unlock(); +} + +static int checker_thread(void *data) +{ + int check_interval_ms = 500; + + while (!kthread_should_stop()) { + check_all_threads(); + msleep_interruptible(check_interval_ms); + } + pr_info("[IEE] Kernel thread exiting\n"); + return 0; +} + +static int __init thread_checker_init(void) +{ + struct task_struct *checker_task; + + if (!haoc_enabled) + return 0; + + pr_info("IEE: Initializing thread checker\n"); + checker_task = kthread_run(checker_thread, NULL, "thread_credp_cycle"); + if (IS_ERR(checker_task)) { + pr_err("IEE: Failed to create thread checker task: %ld\n", + PTR_ERR(checker_task)); + return PTR_ERR(checker_task); + } + + pr_info("IEE: Thread checker started successfully\n"); + return 0; +} + +late_initcall(thread_checker_init); +#endif +#endif + static inline void iee_verify_cred_type(const struct cred *cred) { if (!iee_init_done || !haoc_bitmap_ready) @@ -325,7 +432,12 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) __func__, (u64)new, (u64)token->new_cred); /* task->cred shall be updated once. */ token->new_cred = NULL; + write_seqcount_begin(&token->seq); token->curr_cred = new; + rcu_assign_pointer(task->real_cred, new); + rcu_assign_pointer(task->cred, new); + write_seqcount_end(&token->seq); + return; } #endif @@ -439,7 +551,11 @@ void __iee_code _iee_override_creds(unsigned long __unused, const struct cred *n if (haoc_enabled) { struct task_token *token = (struct task_token *)iee_get_task_token(current); + write_seqcount_begin(&token->seq); token->curr_cred = new; + rcu_assign_pointer(current->cred, new); + write_seqcount_end(&token->seq); + return; } #endif @@ -455,7 +571,11 @@ void __iee_code _iee_revert_creds(unsigned long __unused, const struct cred *old if (haoc_enabled) { struct task_token *token = (struct task_token *)iee_get_task_token(current); + write_seqcount_begin(&token->seq); token->curr_cred = old; + rcu_assign_pointer(current->cred, old); + write_seqcount_end(&token->seq); + return; } #endif diff --git a/arch/arm64/kernel/haoc/iee/iee-token.c b/arch/arm64/kernel/haoc/iee/iee-token.c index 28e9506d1ff83..52f03df20e698 100644 --- a/arch/arm64/kernel/haoc/iee/iee-token.c +++ b/arch/arm64/kernel/haoc/iee/iee-token.c @@ -222,7 +222,8 @@ void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct * token->new_cred = NULL; token->curr_cred = tsk->cred; #endif - token->valid = true; + seqcount_init(&token->seq); + smp_store_release(&token->valid, true); } void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) @@ -234,5 +235,6 @@ void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct token->new_cred = NULL; token->curr_cred = NULL; #endif + seqcount_init(&token->seq); token->valid = false; } diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 5ea9f63d559eb..9c0f08e2bb827 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -42,6 +42,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #ifdef CONFIG_IEE_SIP #include #endif @@ -318,7 +321,10 @@ static void die_kernel_fault(const char *msg, unsigned long addr, pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg, addr); - +#ifdef CONFIG_IEE + if (__is_lm_address((u64)addr)) + pr_alert("iee_get_bitmap_type(addr) : %d\n", iee_get_bitmap_type(addr)); +#endif kasan_non_canonical_hook(addr); mem_abort_decode(esr); diff --git a/fs/exec.c b/fs/exec.c index 1d1212cb56a4c..3d8af4a18f9e5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1980,7 +1980,14 @@ static int do_execveat_common(int fd, struct filename *filename, #endif #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) { + pr_info_once("HAOC: CONFIG_IEE_IO_CHECK enabled."); + iee_cycle_verify_cred(current); + } +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif diff --git a/fs/ioctl.c b/fs/ioctl.c index 60a277f3c3044..11065563c7ea6 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -52,7 +52,12 @@ extern void iee_cycle_verify_cred(struct task_struct *current_task); long vfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif diff --git a/fs/read_write.c b/fs/read_write.c index 4881922a6602b..debe6e8f4a814 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -461,7 +461,12 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) ssize_t ret; #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif @@ -581,7 +586,12 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ ssize_t ret; #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif @@ -927,7 +937,12 @@ static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif @@ -950,7 +965,12 @@ static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif diff --git a/kernel/fork.c b/kernel/fork.c index 302bd9615ebe1..ff6843d749bf3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2403,7 +2403,12 @@ __latent_entropy struct task_struct *copy_process( iee_validate_token(p); #endif #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(current); # endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 408c5d3decdd1..e38438537bd9e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5172,6 +5172,10 @@ void __weak ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0) { } void __weak ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0) { } #endif +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif + /* * context_switch - switch to the new MM and the new thread's register state. */ @@ -5180,7 +5184,14 @@ context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf) { #if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) -# if defined(CONFIG_X86_64) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_CS_CHECK + if (haoc_enabled) { + pr_info_once("HAOC: CONFIG_IEE_CS_CHECK enabled."); + iee_cycle_verify_cred(next); + } +# endif +# elif defined(CONFIG_X86_64) if (haoc_enabled) iee_verify_token(next); # endif From 19e90e1e6fef033a689d18f4a457838994ea0591 Mon Sep 17 00:00:00 2001 From: chengwu12 Date: Wed, 20 May 2026 09:23:25 +0800 Subject: [PATCH 14/26] HAOC4.0/x86_64/VARP: Add support for x86 variable Protection Co-authored-by: chengwu12 Signed-off-by: Liu Zhehui --- arch/x86/Kconfig | 10 +++++ arch/x86/configs/deepin_x86_desktop_defconfig | 1 + arch/x86/include/asm/haoc/haoc-def.h | 3 ++ arch/x86/include/asm/haoc/haoc.h | 10 +++++ arch/x86/include/asm/haoc/iee-varp.h | 23 +++++++++++ arch/x86/kernel/haoc/Makefile | 1 + arch/x86/kernel/haoc/haoc.c | 3 ++ arch/x86/kernel/haoc/iee/iee-init.c | 7 ++++ arch/x86/kernel/haoc/varp/Makefile | 1 + arch/x86/kernel/haoc/varp/varp.c | 41 +++++++++++++++++++ arch/x86/kernel/vmlinux.lds.S | 12 ++++++ kernel/module/kmod.c | 8 ++++ kernel/sysctl.c | 40 ++++++++++++++++++ 13 files changed, 160 insertions(+) create mode 100644 arch/x86/include/asm/haoc/iee-varp.h create mode 100644 arch/x86/kernel/haoc/varp/Makefile create mode 100644 arch/x86/kernel/haoc/varp/varp.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 63343721f5dad..d221a72c3987b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1622,6 +1622,16 @@ config IEE_SELINUX_P depends on SECURITY_SELINUX depends on IEE +config VARP + bool "variable protection for IEE" + depends on IEE + help + Protect variables with IEE. At present, only modprobe_path is + covered. Writes to protected variables are routed through IEE + interfaces instead of direct kernel stores. + Enable this option when modprobe_path must be protected by the + isolated execution environment. + config PTP_RESERVE_ORDER depends on PTP int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 4e50022875b12..8eafd558f19ff 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -97,6 +97,7 @@ CONFIG_KEYP=y CONFIG_IEE_SELINUX_P=y CONFIG_PTP=y CONFIG_PTP_RESERVE_ORDER=12 +CONFIG_VARP=y CONFIG_NUMA=y CONFIG_ARCH_MEMORY_PROBE=y CONFIG_X86_PMEM_LEGACY=m diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index 7dfc31638c40a..45fee9457444e 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -109,6 +109,9 @@ enum { IEE_OP_SET_PMD_IDENT, IEE_OP_SET_PUD_IDENT, IEE_OP_SET_SX_PTE, +#endif +#ifdef CONFIG_VARP + IEE_OP_SET_VARP_MODPROBE_PATH, #endif IEE_FLAG_END }; diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index 0ee4c19513e56..82565551065f8 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -111,6 +111,16 @@ void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_hea void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); #endif +#ifdef CONFIG_VARP +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, int maxlen, size_t len, + char *buffer, size_t *lenp); +#endif + +#ifdef CONFIG_VARP +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, int maxlen, size_t len, + char *buffer, size_t *lenp); +#endif + #ifdef CONFIG_KEYP #include struct watch_list; diff --git a/arch/x86/include/asm/haoc/iee-varp.h b/arch/x86/include/asm/haoc/iee-varp.h new file mode 100644 index 0000000000000..2c5566d2c3755 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-varp.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_VARP_H +#define _LINUX_IEE_VARP_H + +#include + +#ifdef CONFIG_VARP +#define __iee_varp_data __section(".iee.varp") +#endif + +extern unsigned long long iee_rw_gate(int flag, ...); + +extern unsigned long __iee_varp_data_start[]; +extern unsigned long __iee_varp_data_end[]; + +static void __maybe_unused iee_set_varp_modprobe_path(char *data, int maxlen, + size_t len, char *buffer, + size_t *lenp) +{ + iee_rw_gate(IEE_OP_SET_VARP_MODPROBE_PATH, data, maxlen, len, buffer, + lenp); +} +#endif diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index 7de5a80764f05..826c316116f99 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_CREDP) += credp/ obj-$(CONFIG_KEYP) += keyp/ ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ +obj-$(CONFIG_VARP) += varp/ diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 4365c1239d15a..280c83a31cf0d 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -109,6 +109,9 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_pmd_ident, (iee_func)_iee_set_pud_ident, (iee_func)_iee_set_sx_pte, +#endif +#ifdef CONFIG_VARP + (iee_func)_iee_set_varp_modprobe_path, #endif NULL }; diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index bab99a040865c..d909d4abbc4fc 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -35,6 +35,9 @@ extern void iee_selinuxp_init(void); #ifdef CONFIG_PTP extern void haoc_ptp_init(void); #endif +#ifdef CONFIG_VARP +extern void varp_init(void); +#endif /* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ unsigned long IEE_OFFSET = 0x200000000000; @@ -284,5 +287,9 @@ void __init iee_post_init(void) #ifdef CONFIG_IEE_SELINUX_P iee_selinuxp_init(); #endif + +#ifdef CONFIG_VARP + varp_init(); +#endif haoc_init_done = true; } diff --git a/arch/x86/kernel/haoc/varp/Makefile b/arch/x86/kernel/haoc/varp/Makefile new file mode 100644 index 0000000000000..775b2da3f64b1 --- /dev/null +++ b/arch/x86/kernel/haoc/varp/Makefile @@ -0,0 +1 @@ +obj-y += varp.o \ No newline at end of file diff --git a/arch/x86/kernel/haoc/varp/varp.c b/arch/x86/kernel/haoc/varp/varp.c new file mode 100644 index 0000000000000..271a16428e94f --- /dev/null +++ b/arch/x86/kernel/haoc/varp/varp.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, + int maxlen, size_t len, char *buffer, + size_t *lenp) +{ + char c, *p; + + if (!data || !buffer || !lenp) + return; + p = buffer; + while ((p - buffer) < *lenp && len < maxlen - 1) { + c = *(p++); + if (c == 0 || c == '\n') + break; + data[len++] = c; + } + data[len] = 0; +} + +void __init varp_init(void) +{ + unsigned long start, end, varp_logical_addr; + int num_pages; + + /* Map .iee.varp as RO pages, the variable need to be protected can be added */ + start = (unsigned long)__iee_varp_data_start; + end = (unsigned long)__iee_varp_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_page(start, num_pages, IEE_VARP); + + /* protect linear mapping */ + varp_logical_addr = (unsigned long)__va(__pa_symbol(start)); + set_iee_page(varp_logical_addr, num_pages, IEE_VARP); + + pr_info("HAOC: CONFIG_VARP enabled."); +} diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 0385cf9ce8c88..9e1b283e03a3f 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -135,6 +135,15 @@ __iee_si_data_end = .; __iee_selinux_data_end = .; #endif +#ifdef CONFIG_VARP +#define VARP_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_start = .; \ + *(.iee.varp) \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_end = .; +#endif + PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(6); /* RW_ */ @@ -228,6 +237,9 @@ SECTIONS #endif #ifdef CONFIG_IEE_SELINUX_P IEE_SELINUX_DATA +#endif +#ifdef CONFIG_VARP + VARP_DATA #endif CONSTRUCTORS diff --git a/kernel/module/kmod.c b/kernel/module/kmod.c index 0800d98916921..f656d6575ab54 100644 --- a/kernel/module/kmod.c +++ b/kernel/module/kmod.c @@ -32,6 +32,10 @@ #include #include "internal.h" +#ifdef CONFIG_VARP +#include +#endif + /* * Assuming: * @@ -61,7 +65,11 @@ static DEFINE_SEMAPHORE(kmod_concurrent_max, MAX_KMOD_CONCURRENT); /* modprobe_path is set via /proc/sys. */ +#ifdef CONFIG_VARP +char modprobe_path[KMOD_PATH_LEN] __iee_varp_data = CONFIG_MODPROBE_PATH; +#else char modprobe_path[KMOD_PATH_LEN] = CONFIG_MODPROBE_PATH; +#endif static void free_modprobe_argv(struct subprocess_info *info) { diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 4eee1add47412..3b72b6d7082a1 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -80,6 +80,9 @@ #ifdef CONFIG_RT_MUTEXES #include #endif +#ifdef CONFIG_VARP +#include +#endif /* shared constants to be used in various sysctls */ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; @@ -205,6 +208,37 @@ static int _proc_do_string(char *data, int maxlen, int write, return 0; } +#ifdef CONFIG_VARP +static int _iee_proc_do_string(char *data, int maxlen, int write, + char *buffer, size_t *lenp, loff_t *ppos) +{ + size_t len; + + if (!data || !maxlen || !*lenp || !buffer) { + *lenp = 0; + return 0; + } + + if (sysctl_writes_strict == SYSCTL_WRITES_STRICT) { + /* Only continue writes not past the end of buffer. */ + len = strlen(data); + if (len > maxlen - 1) + len = maxlen - 1; + + if (*ppos > len) + return 0; + len = *ppos; + } else { + /* Start writing from beginning of buffer. */ + len = 0; + } + + *ppos += *lenp; + iee_set_varp_modprobe_path(data, maxlen, len, buffer, lenp); + return 0; +} +#endif + static void warn_sysctl_write(struct ctl_table *table) { pr_warn_once("%s wrote to %s when file position was not 0!\n" @@ -262,6 +296,12 @@ int proc_dostring(struct ctl_table *table, int write, if (write) proc_first_pos_non_zero_ignore(ppos, table); +#ifdef CONFIG_VARP + if (table && table->data == modprobe_path && write) + return _iee_proc_do_string(table->data, table->maxlen, write, buffer, lenp, + ppos); +#endif + return _proc_do_string(table->data, table->maxlen, write, buffer, lenp, ppos); } From a8de09bdc45607fbe013a671739b9c1b2b9028fe Mon Sep 17 00:00:00 2001 From: chengwu12 Date: Wed, 20 May 2026 17:03:45 +0800 Subject: [PATCH 15/26] HAOC4.0/arm64/VARP: Add support for arm64 variable protection Co-authored-by: chengwu12 Signed-off-by: Liu Zhehui --- .../configs/deepin_arm64_desktop_defconfig | 1 + arch/arm64/include/asm/haoc/haoc-def.h | 3 +++ arch/arm64/include/asm/haoc/haoc.h | 6 +++++ arch/arm64/include/asm/haoc/iee-init.h | 5 +++++ arch/arm64/include/asm/haoc/iee-varp.h | 20 +++++++++++++++++ arch/arm64/kernel/haoc/Kconfig | 11 ++++++++++ arch/arm64/kernel/haoc/Makefile | 1 + arch/arm64/kernel/haoc/haoc-bitmap.c | 7 ++++++ arch/arm64/kernel/haoc/haoc.c | 5 ++++- arch/arm64/kernel/haoc/varp/Makefile | 1 + arch/arm64/kernel/haoc/varp/varp.c | 22 +++++++++++++++++++ arch/arm64/kernel/vmlinux.lds.S | 12 ++++++++++ 12 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/haoc/iee-varp.h create mode 100644 arch/arm64/kernel/haoc/varp/Makefile create mode 100644 arch/arm64/kernel/haoc/varp/varp.c diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index fd5288df06deb..8cd88b24a43d9 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -170,6 +170,7 @@ CONFIG_KEYP=y CONFIG_IEE_SELINUX_P=y CONFIG_PTP=y CONFIG_PTP_RESERVE_ORDER=12 +CONFIG_VARP=y CONFIG_KPROBES=y CONFIG_SHADOW_CALL_STACK=y CONFIG_MODULES=y diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index e1e20940cced5..5be9084e7a241 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -110,6 +110,9 @@ enum { IEE_SEL_SET_INITIALIZED, IEE_SEL_SET_POLICY_CAP, IEE_SEL_RCU_ASSIGN_POLICY, +#endif +#ifdef CONFIG_VARP + IEE_OP_SET_VARP_MODPROBE_PATH, #endif IEE_FLAG_END }; diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index ea81e638ab64b..bd9f684be320a 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -162,4 +162,10 @@ void _iee_set_sel_policy_cap(unsigned long __unused, unsigned int idx, int cap); void _iee_sel_rcu_assign_policy(unsigned long __unused, struct selinux_policy *new_policy, struct selinux_policy *iee_new_policy); #endif + +#ifdef CONFIG_VARP +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, int maxlen, size_t len, + char *buffer, size_t *lenp); +#endif + #endif diff --git a/arch/arm64/include/asm/haoc/iee-init.h b/arch/arm64/include/asm/haoc/iee-init.h index cc27b237ef112..3ac057dd86ed0 100644 --- a/arch/arm64/include/asm/haoc/iee-init.h +++ b/arch/arm64/include/asm/haoc/iee-init.h @@ -31,6 +31,11 @@ extern char __iee_cred_data_end[]; extern char __iee_selinux_data_start[]; extern char __iee_selinux_data_end[]; #endif +#ifdef CONFIG_VARP +extern char __iee_varp_data_start[]; +extern char __iee_varp_data_end[]; +#endif + extern struct mutex fixmap_lock; #endif diff --git a/arch/arm64/include/asm/haoc/iee-varp.h b/arch/arm64/include/asm/haoc/iee-varp.h new file mode 100644 index 0000000000000..9ad9f000c26c6 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-varp.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_VARP_H +#define _LINUX_IEE_VARP_H + +#include + +#ifdef CONFIG_VARP +#define __iee_varp_data __section(".iee.varp") +#endif + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_set_varp_modprobe_path(char *data, int maxlen, + size_t len, char *buffer, + size_t *lenp) +{ + iee_rw_gate(IEE_OP_SET_VARP_MODPROBE_PATH, data, maxlen, len, buffer, + lenp); +} +#endif diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index 4babc31a82282..50d60f391a9ed 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -131,4 +131,15 @@ config IEE_SELINUX_P If unsure, say N. depends on SECURITY_SELINUX depends on IEE + +config VARP + bool "Variable Protection(VARP)" + depends on IEE + help + Protect variables with IEE. At present, only modprobe_path is + covered. Writes to protected variables are routed through IEE + interfaces instead of direct kernel stores. + Enable this option when modprobe_path must be protected by the + isolated execution environment. + endmenu # HAOC diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile index 723dc21ef15ef..ce076e84f351d 100644 --- a/arch/arm64/kernel/haoc/Makefile +++ b/arch/arm64/kernel/haoc/Makefile @@ -4,6 +4,7 @@ obj-y += iee/ obj-$(CONFIG_CREDP) += credp/ obj-$(CONFIG_PTP) += ptp/ obj-$(CONFIG_KEYP) += keyp/ +obj-$(CONFIG_VARP) += varp/ ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ diff --git a/arch/arm64/kernel/haoc/haoc-bitmap.c b/arch/arm64/kernel/haoc/haoc-bitmap.c index d1c162b91ca0a..c1eb4db73d44c 100644 --- a/arch/arm64/kernel/haoc/haoc-bitmap.c +++ b/arch/arm64/kernel/haoc/haoc-bitmap.c @@ -208,6 +208,13 @@ void __init haoc_bitmap_setup(void) num_pages = (end - start) / PAGE_SIZE; iee_set_bitmap_type(start, num_pages, IEE_VARP); #endif + #ifdef CONFIG_VARP + /* Map .iee.varp as RO pages, the variable need to be protected can be added */ + start = (unsigned long)__iee_varp_data_start; + end = (unsigned long)__iee_varp_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_VARP); + #endif /* Mark bitmaps of early allocated iee data. */ setup_iee_early_data_bitmap(); diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index 249bf792aab4c..5d9a96668509d 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -118,5 +118,8 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_sel_policy_cap, (iee_func)_iee_sel_rcu_assign_policy, #endif - NULL +#ifdef CONFIG_VARP + (iee_func)_iee_set_varp_modprobe_path, +#endif + NULL }; diff --git a/arch/arm64/kernel/haoc/varp/Makefile b/arch/arm64/kernel/haoc/varp/Makefile new file mode 100644 index 0000000000000..775b2da3f64b1 --- /dev/null +++ b/arch/arm64/kernel/haoc/varp/Makefile @@ -0,0 +1 @@ +obj-y += varp.o \ No newline at end of file diff --git a/arch/arm64/kernel/haoc/varp/varp.c b/arch/arm64/kernel/haoc/varp/varp.c new file mode 100644 index 0000000000000..714a87571a348 --- /dev/null +++ b/arch/arm64/kernel/haoc/varp/varp.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +void __iee_code _iee_set_varp_modprobe_path(unsigned long __unused, char *data, + int maxlen, size_t len, char *buffer, + size_t *lenp) +{ + char c, *p; + + if (!data || !buffer || !lenp) + return; + data = __ptr_to_iee(data); + p = buffer; + while ((p - buffer) < *lenp && len < maxlen - 1) { + c = *(p++); + if (c == 0 || c == '\n') + break; + data[len++] = c; + } + data[len] = 0; +} diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 15541e91742df..06588d80da678 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -182,6 +182,17 @@ jiffies = jiffies_64; #define IEE_SELINUX_DATA #endif +#ifdef CONFIG_VARP +#define VARP_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_start = .; \ + *(.iee.varp) \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_end = .; +#else +#define VARP_DATA +#endif + #ifdef CONFIG_IEE #define IEE_INIT_DATA \ .iee.data : { \ @@ -190,6 +201,7 @@ jiffies = jiffies_64; PTP_DATA \ CRED_DATA \ IEE_SELINUX_DATA \ + VARP_DATA \ . = ALIGN(PAGE_SIZE); \ iee_init_data_end = .; \ } From b358f2bdf19ed7db60431273045d67ec88d10d12 Mon Sep 17 00:00:00 2001 From: amjac Date: Mon, 1 Jun 2026 20:11:34 +0800 Subject: [PATCH 16/26] HAOC4.0/x86_64/PTP_S: Add supprot for x86 page table protection-separation Introduce CONFIG_PTP_S and IEE_USER_PGTABLE to support staged Page-Table Protection (kernel/user page Separation). Split PTP write paths so user page tables can use direct writes while kernel tables stay on ptp gate paths, with user-side safety checks for alias/integrity. Signed-off-by: Liu Zhehui (cherry picked from commit c03cf3608ac85964f1c34c40cf951126aefc62fc) --- arch/x86/Kconfig | 10 +++ arch/x86/configs/deepin_x86_desktop_defconfig | 5 +- arch/x86/include/asm/haoc/ptp.h | 44 +++++++++++++ arch/x86/include/asm/pgtable.h | 40 +++++++++++- arch/x86/include/asm/pgtable_64.h | 12 ++++ arch/x86/kernel/haoc/iee/iee-func.c | 5 ++ arch/x86/kernel/haoc/iee/iee-init.c | 3 + arch/x86/kernel/haoc/ptp/ptp.c | 65 +++++++++++++++++++ include/linux/ptp-cache.h | 3 + mm/haoc/ptp-pg_cache.c | 39 ++++++++++- 10 files changed, 222 insertions(+), 4 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d221a72c3987b..47a7a68927973 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1622,6 +1622,16 @@ config IEE_SELINUX_P depends on SECURITY_SELINUX depends on IEE +config PTP_S + bool "Page Table Protection - Separation (PTP_S)" + depends on PTP + help + PTP_S (kernel/user page separation). Split PTP write paths so user page + tables can use direct writes while kernel tables stay on ptp gate paths, + with user-side safety checks for alias/integrity. + Enable this option when user page-table writes should avoid the PTP + gate while kernel page-table writes remain protected by PTP. + config VARP bool "variable protection for IEE" depends on IEE diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 8eafd558f19ff..d60bb6a82a98e 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -89,14 +89,15 @@ CONFIG_X86_MSR=m CONFIG_X86_CPUID=m CONFIG_AMD_MEM_ENCRYPT=y CONFIG_IEE=y +CONFIG_PTP=y +CONFIG_PTP_RESERVE_ORDER=12 CONFIG_IEE_PTRP=y CONFIG_IEE_PTRP_W=y CONFIG_IEE_SIP=y CONFIG_CREDP=y CONFIG_KEYP=y CONFIG_IEE_SELINUX_P=y -CONFIG_PTP=y -CONFIG_PTP_RESERVE_ORDER=12 +CONFIG_PTP_S=y CONFIG_VARP=y CONFIG_NUMA=y CONFIG_ARCH_MEMORY_PROBE=y diff --git a/arch/x86/include/asm/haoc/ptp.h b/arch/x86/include/asm/haoc/ptp.h index 20e04e40fa940..1f8d51b89166d 100644 --- a/arch/x86/include/asm/haoc/ptp.h +++ b/arch/x86/include/asm/haoc/ptp.h @@ -3,32 +3,76 @@ #define _LINUX_PTP_H #include +#include #include +#ifdef CONFIG_PTP_S +#include +#endif extern unsigned long long ptp_rw_gate(int flag, ...); +#ifdef CONFIG_PTP_S +extern bool ptp_is_user_pgtable(const void *ptp); +extern void ptp_user_check_pte_update(pte_t *ptep, pte_t pte); +extern void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd); +extern void ptp_user_check_pud_update(pud_t *pudp, pud_t pud); +#endif + static inline void ptp_set_pte(pte_t *ptep, pte_t pte) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, pte); + WRITE_ONCE(*ptep, pte); + return; + } +#endif ptp_rw_gate(IEE_OP_SET_PTE, ptep, pte); } static inline void ptp_set_pmd(pmd_t *pmdp, pmd_t pmd) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, pmd); + WRITE_ONCE(*pmdp, pmd); + return; + } +#endif ptp_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); } static inline void ptp_set_pud(pud_t *pudp, pud_t pud) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pudp)) { + ptp_user_check_pud_update(pudp, pud); + WRITE_ONCE(*pudp, pud); + return; + } +#endif ptp_rw_gate(IEE_OP_SET_PUD, pudp, pud); } static inline void ptp_set_p4d(p4d_t *p4dp, p4d_t p4d) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(p4dp)) { + WRITE_ONCE(*p4dp, p4d); + return; + } +#endif ptp_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); } static inline void ptp_set_pgd(pgd_t *pgdp, pgd_t pgd) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pgdp)) { + WRITE_ONCE(*pgdp, pgd); + return; + } +#endif ptp_rw_gate(IEE_OP_SET_PGD, pgdp, pgd); } diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 765c1652b557a..b3fd5f9f5165b 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -25,6 +25,10 @@ #include #ifdef CONFIG_PTP #include +#include +#ifdef CONFIG_PTP_S +#include +#endif #endif extern pgd_t early_top_pgt[PTRS_PER_PGD]; @@ -1305,10 +1309,24 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, old_pte = READ_ONCE(*ptep); #ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + do { + new_pte = pte_wrprotect(old_pte); + if (ptp_is_user_pgtable(ptep)) { + if (try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, + *(long *)&new_pte)) + break; + } else if (ptp_try_cmpxchg((long *)ptep, pte_val(old_pte), + pte_val(new_pte))) { + break; + } + } while (1); +#else /* CONFIG_PTP && !CONFIG_PTP_S */ do { new_pte = pte_wrprotect(old_pte); } while (!ptp_try_cmpxchg((long *)ptep, pte_val(old_pte), pte_val(new_pte))); -#else +#endif /* CONFIG_PTP_S */ +#else /* !CONFIG_PTP */ do { new_pte = pte_wrprotect(old_pte); } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte)); @@ -1373,9 +1391,23 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, old_pmd = READ_ONCE(*pmdp); #ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + do { + new_pmd = pmd_wrprotect(old_pmd); + if (ptp_is_user_pgtable(pmdp)) { + if (try_cmpxchg((long *)pmdp, (long *)&old_pmd, + *(long *)&new_pmd)) + break; + } else if ((ptp_try_cmpxchg((long *)pmdp, pmd_val(old_pmd), + pmd_val(new_pmd)))) { + break; + } + } while (1); +#else /* CONFIG_PTP && !CONFIG_PTP_S */ do { new_pmd = pmd_wrprotect(old_pmd); } while (!ptp_try_cmpxchg((long *)pmdp, pmd_val(old_pmd), pmd_val(new_pmd))); +#endif /* CONFIG_PTP_S */ #else do { new_pmd = pmd_wrprotect(old_pmd); @@ -1391,6 +1423,12 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); if (IS_ENABLED(CONFIG_SMP)) { #ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, pmd); + return xchg(pmdp, pmd); + } +#endif return native_make_pmd(ptp_xchg((pgprotval_t *)pmdp, pmd_val(pmd))); #else return xchg(pmdp, pmd); diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 6839dd1be00be..a8f04cfca8d8d 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -105,6 +105,10 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) { #ifdef CONFIG_SMP #ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(xp)) + return native_make_pte(xchg(&xp->pte, 0)); +#endif return native_make_pte(ptp_xchg((pgprotval_t *)xp, 0)); #else return native_make_pte(xchg(&xp->pte, 0)); @@ -122,6 +126,10 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) { #ifdef CONFIG_SMP #ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(xp)) + return native_make_pmd(xchg(&xp->pmd, 0)); +#endif return native_make_pmd(ptp_xchg((pgprotval_t *)xp, 0)); #else return native_make_pmd(xchg(&xp->pmd, 0)); @@ -154,6 +162,10 @@ static inline pud_t native_pudp_get_and_clear(pud_t *xp) { #ifdef CONFIG_SMP #ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(xp)) + return native_make_pud(xchg(&xp->pud, 0)); +#endif return native_make_pud(ptp_xchg((pgprotval_t *)xp, 0)); #else return native_make_pud(xchg(&xp->pud, 0)); diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index bf23da605cba3..33d4f024c0921 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -17,6 +17,11 @@ extern bool haoc_enabled; void set_iee_page(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) { +#ifdef CONFIG_PTP_S + if (type == IEE_USER_PGTABLE) + set_memory_rw(addr, num_pages); + else +#endif set_memory_ro(addr, num_pages); iee_set_bitmap_type(__pa(addr), num_pages, type); } diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index d909d4abbc4fc..131d44b634035 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -240,6 +240,9 @@ void __init iee_early_init(void) iee_cache_init(&pg_cache, 0, 4, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); else iee_cache_init(&pg_cache, 0, 3, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); +#ifdef CONFIG_PTP_S + iee_cache_init(&pg_user_cache, 0, 3, IEE_USER_PGTABLE, CONFIG_PTP_RESERVE_ORDER); +#endif /* CONFIG_PTP_S */ #endif } diff --git a/arch/x86/kernel/haoc/ptp/ptp.c b/arch/x86/kernel/haoc/ptp/ptp.c index 7769b1b77ace5..e042d185ae22f 100644 --- a/arch/x86/kernel/haoc/ptp/ptp.c +++ b/arch/x86/kernel/haoc/ptp/ptp.c @@ -288,6 +288,56 @@ static inline void check_pt_integrity(pte_t *ptep, pte_t pte) old_pte.pte, pte.pte); } +#ifdef CONFIG_PTP_S +bool ptp_is_user_pgtable(const void *ptp) +{ + if (!haoc_enabled) + return false; + + if (unlikely(!haoc_init_done)) + return false; + + if (unlikely(!virt_addr_valid(ptp))) + return false; + + return iee_get_bitmap_type(__pa((unsigned long)ptp)) == IEE_USER_PGTABLE; +} +EXPORT_SYMBOL(ptp_is_user_pgtable); + +void ptp_user_check_pte_update(pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) + return; + + check_alias_mapping(__pte_to_phys(pte), __pte_to_phys(pte) + PAGE_SIZE, + pte_write(pte), preserves_existing_writable_mapping(ptep, pte)); + check_pt_integrity(ptep, pte); +} +EXPORT_SYMBOL(ptp_user_check_pte_update); + +void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd) +{ + if (!(pmd_val(pmd) & _PAGE_PRESENT)) + return; + + if (pmd_leaf(pmd)) + check_alias_mapping(__pmd_to_phys(pmd), __pmd_to_phys(pmd) + PMD_SIZE, + pmd_write(pmd), + pmd_preserves_existing_writable_mapping(pmdp, pmd)); +} + +void ptp_user_check_pud_update(pud_t *pudp, pud_t pud) +{ + if (!(pud_val(pud) & _PAGE_PRESENT)) + return; + + if (pud_leaf(pud)) + check_alias_mapping(__pud_to_phys(pud), __pud_to_phys(pud) + PUD_SIZE, + pud_write(pud), + pud_preserves_existing_writable_mapping(pudp, pud)); +} +#endif + void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte) { if (!(pte_val(pte) & _PAGE_PRESENT)) { @@ -455,6 +505,21 @@ void __init haoc_ptp_init(void) int cpu; struct iee_cr0 *iee_cr0; + set_iee_page(pg_cache.reserve_start_addr, + (1 << pg_cache.reserve_order) * pg_cache.levels, IEE_PGTABLE); + + set_iee_page(pgd_cache.reserve_start_addr, + (1 << pgd_cache.reserve_order) * pgd_cache.levels, IEE_PGTABLE); + +#ifdef CONFIG_PTP_S + if (!pg_user_cache.init) + panic("PTP_S: pg_user_cache is not initialized"); + + set_iee_page(pg_user_cache.reserve_start_addr, + (1 << pg_user_cache.reserve_order) * pg_user_cache.levels, + IEE_USER_PGTABLE); +#endif + for_each_possible_cpu(cpu) { iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); iee_cr0->wp_disabled_cnt = 0; diff --git a/include/linux/ptp-cache.h b/include/linux/ptp-cache.h index 928f0191473bb..b769920e4b3f7 100644 --- a/include/linux/ptp-cache.h +++ b/include/linux/ptp-cache.h @@ -31,6 +31,9 @@ struct iee_cache { }; extern struct iee_cache pg_cache; +#ifdef CONFIG_PTP_S +extern struct iee_cache pg_user_cache; +#endif extern void iee_cache_init(struct iee_cache *cache, unsigned long object_order, int levels, enum HAOC_BITMAP_TYPE bitmap_type, unsigned long reserve_order); diff --git a/mm/haoc/ptp-pg_cache.c b/mm/haoc/ptp-pg_cache.c index cde38ee6b0b8f..5ded80fe3f0f9 100644 --- a/mm/haoc/ptp-pg_cache.c +++ b/mm/haoc/ptp-pg_cache.c @@ -14,8 +14,14 @@ #include #include #include +#ifdef CONFIG_PTP_S +#include +#endif struct iee_cache pg_cache; +#ifdef CONFIG_PTP_S +struct iee_cache pg_user_cache; +#endif int expend_num; #ifdef CONFIG_X86_64 @@ -102,6 +108,13 @@ void __init iee_cache_init(struct iee_cache *cache, unsigned long object_order, unsigned long object_size; struct page *page; + if (!haoc_enabled && bitmap_type != IEE_PGTABLE +#ifdef CONFIG_PTP_S + && bitmap_type != IEE_USER_PGTABLE +#endif + ) + return; + object_size = (1 << object_order) * PAGE_SIZE; while (1) { reserve_pages = (1 << reserve_order) * levels; @@ -219,6 +232,12 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) void *object; void *next_object; +#ifdef CONFIG_PTP_S + if (haoc_enabled && haoc_init_done) + if ((gfp & __GFP_ACCOUNT) && cache == &pg_cache) + cache = &pg_user_cache; +#endif + if (!cache->init) return (void *)__get_free_pages(gfp, cache->object_order); @@ -260,8 +279,14 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) if (unlikely(!__update_freelist(cache, object, next_object, tid))) goto redo; prefetchw(next_object); - if (gfp & __GFP_ZERO) + if (gfp & __GFP_ZERO) { +#ifdef CONFIG_PTP_S + if (cache == &pg_user_cache) + clear_page(object); + else +#endif iee_memset(object, 0, object_size); + } } return object; }; @@ -394,6 +419,13 @@ void iee_cache_free(struct iee_cache *cache, void *object) unsigned long tid; void **freelist; +#ifdef CONFIG_PTP_S + if (cache == &pg_cache) { + if (ptp_is_user_pgtable(object)) + cache = &pg_user_cache; + } +#endif + if (!cache->init) { free_pages((unsigned long)object, cache->object_order); return; @@ -424,6 +456,11 @@ void iee_cache_free(struct iee_cache *cache, void *object) tid = READ_ONCE(cache->tid); barrier(); freelist = READ_ONCE(cache->freelist); +#ifdef CONFIG_PTP_S + if (cache == &pg_user_cache) + __set_freepointer(object, freelist); + else +#endif __iee_set_freepointer(object, freelist); #ifdef CONFIG_ARM64 dsb(sy); From 1aa36641453d399aaa81227b5cf2ba94d26cbd2b Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Fri, 5 Jun 2026 14:47:18 +0800 Subject: [PATCH 17/26] HAOC4.0/arm64/PTP_S: Add support for ARM64 Page-Tables-Protection Seperation (PTP_S) Add ARM64 support for HAOC Page-Table Protection Separation (PTP_S). This introduces separated page-table protection semantics so user and kernel page-table updates can follow different enforcement paths while keeping the existing strict gate protection for kernel mappings. The change is designed to preserve current behavior when PTP_S is disabled and only enables the new separation model when configured. Signed-off-by: Liu Zhehui (cherry picked from commit 25d2717f0ec6e2b3abc5fe56312de0ae1310e26e) --- .../configs/deepin_arm64_desktop_defconfig | 5 +- arch/arm64/include/asm/haoc/haoc.h | 2 +- arch/arm64/include/asm/haoc/iee-func.h | 2 + arch/arm64/include/asm/haoc/iee-pgtable.h | 96 +++++++++++--- arch/arm64/kernel/haoc/Kconfig | 31 +++++ arch/arm64/kernel/haoc/iee/iee-func.c | 26 +++- arch/arm64/kernel/haoc/ptp/ptp.c | 121 +++++++++++++++++- arch/arm64/kernel/setup.c | 4 + include/linux/ptp-cache.h | 1 + mm/haoc/ptp-pg_cache.c | 15 ++- 10 files changed, 271 insertions(+), 32 deletions(-) diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index 8cd88b24a43d9..a41bea47a494a 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -162,14 +162,15 @@ CONFIG_PMIC_OPREGION=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y CONFIG_IEE=y +CONFIG_PTP=y +CONFIG_PTP_RESERVE_ORDER=12 CONFIG_IEE_PTRP=y CONFIG_IEE_PTRP_W=y CONFIG_IEE_SIP=y CONFIG_CREDP=y CONFIG_KEYP=y CONFIG_IEE_SELINUX_P=y -CONFIG_PTP=y -CONFIG_PTP_RESERVE_ORDER=12 +CONFIG_PTP_S=y CONFIG_VARP=y CONFIG_KPROBES=y CONFIG_SHADOW_CALL_STACK=y diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index bd9f684be320a..64863021973e7 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -111,7 +111,7 @@ pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, pmdval_t old_pmdval, pmdval_t new_pmdval); void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, - pte_t *iee_ptep, int order, int use_block_pmd); + pte_t *iee_ptep, int order, int use_block_pmd, bool writable); void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd); #endif diff --git a/arch/arm64/include/asm/haoc/iee-func.h b/arch/arm64/include/asm/haoc/iee-func.h index 16cde421e66be..4f726c917cfca 100644 --- a/arch/arm64/include/asm/haoc/iee-func.h +++ b/arch/arm64/include/asm/haoc/iee-func.h @@ -17,6 +17,8 @@ extern void set_iee_address_valid(unsigned long lm_addr, unsigned int order); extern void set_iee_address_invalid(unsigned long lm_addr, unsigned int order); extern void iee_set_logical_mem(unsigned long addr, unsigned int order, bool prot); extern void put_pages_into_iee(unsigned long addr, int order); +extern void put_pages_into_iee_rw(unsigned long addr, int order); +extern void remove_pages_from_iee(unsigned long addr, int order); extern void set_iee_page_type(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type); extern void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type); diff --git a/arch/arm64/include/asm/haoc/iee-pgtable.h b/arch/arm64/include/asm/haoc/iee-pgtable.h index 90fd6d2edf0f5..495874a74b013 100644 --- a/arch/arm64/include/asm/haoc/iee-pgtable.h +++ b/arch/arm64/include/asm/haoc/iee-pgtable.h @@ -45,6 +45,14 @@ typedef unsigned int cydp_t; #define CYDP_CLEAR_DIRTY (1U << 1) #endif +#ifdef CONFIG_PTP_S +extern void ptp_user_check_pte_update(pte_t *ptep, pte_t pte); +extern void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd); +extern void ptp_user_check_pud_update(pud_t *pudp, pud_t pud); +extern void ptp_user_check_p4d_update(p4d_t *p4dp, p4d_t p4d); +extern bool ptp_is_user_pgtable(const void *ptp); +#endif + extern pgd_t tramp_pg_dir[]; extern pgd_t idmap_pg_dir[]; extern pgd_t init_pg_dir[]; @@ -175,10 +183,17 @@ static inline pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval) { pteval_t ret; - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, __pte(pteval)); + return xchg_relaxed((pteval_t *)ptep, pteval); + } +#endif ret = iee_rw_gate(IEE_OP_SET_XCHG, ptep, pteval); - else + } else { ret = xchg_relaxed((pteval_t *)ptep, pteval); + } return (pteval_t)ret; } @@ -186,10 +201,17 @@ static inline pmdval_t iee_set_pmd_xchg_relaxed(pmd_t *pmdp, pmdval_t pmdval) { pmdval_t ret; - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, __pmd(pmdval)); + return xchg_relaxed((pmdval_t *)pmdp, pmdval); + } +#endif ret = iee_rw_gate(IEE_OP_SET_PMD_XCHG, pmdp, pmdval); - else + } else { ret = xchg_relaxed((pmdval_t *)pmdp, pmdval); + } return (pmdval_t)ret; } @@ -198,10 +220,17 @@ static inline pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, { pteval_t ret; - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, __pte(new_pteval)); + return cmpxchg_relaxed((pteval_t *)ptep, old_pteval, new_pteval); + } +#endif ret = iee_rw_gate(IEE_OP_SET_CMPXCHG, ptep, old_pteval, new_pteval); - else + } else { ret = cmpxchg_relaxed((pteval_t *)ptep, old_pteval, new_pteval); + } return ret; } @@ -210,10 +239,17 @@ static inline pmdval_t iee_set_pmd_cmpxchg_relaxed(pmd_t *pmdp, pmdval_t old_pmd { pmdval_t ret; - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, __pmd(new_pmdval)); + return cmpxchg_relaxed((pmdval_t *)pmdp, old_pmdval, new_pmdval); + } +#endif ret = iee_rw_gate(IEE_OP_SET_PMD_CMPXCHG, pmdp, old_pmdval, new_pmdval); - else + } else { ret = cmpxchg_relaxed((pmdval_t *)pmdp, old_pmdval, new_pmdval); + } return ret; } @@ -477,10 +513,17 @@ static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) static inline void __set_pte(pte_t *ptep, pte_t pte) { - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, pte); + WRITE_ONCE(*ptep, pte); + } else +#endif iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); - else + } else { WRITE_ONCE(*ptep, pte); + } dsb(ishst); isb(); @@ -895,10 +938,17 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) } #endif /* __PAGETABLE_PMD_FOLDED */ - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, pmd); + WRITE_ONCE(*pmdp, pmd); + } else +#endif iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); - else + } else { WRITE_ONCE(*pmdp, pmd); + } if (pmd_valid(pmd)) { dsb(ishst); @@ -962,10 +1012,17 @@ static inline void set_pud(pud_t *pudp, pud_t pud) } #endif /* __PAGETABLE_PUD_FOLDED */ - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pudp)) { + ptp_user_check_pud_update(pudp, pud); + WRITE_ONCE(*pudp, pud); + } else +#endif iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); - else + } else { WRITE_ONCE(*pudp, pud); + } if (pud_valid(pud)) { dsb(ishst); @@ -1072,10 +1129,17 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) return; } - if (iee_pgtable_ready()) + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(p4dp)) { + ptp_user_check_p4d_update(p4dp, p4d); + WRITE_ONCE(*p4dp, p4d); + } else +#endif iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); - else + } else { WRITE_ONCE(*p4dp, p4d); + } dsb(ishst); isb(); diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index 50d60f391a9ed..509576decb8f8 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -89,6 +89,37 @@ config IEE_SIP such as writing system control registers. These instructions would be executated inside IEE. +config IEE_SIP + bool "Sensitive Instruction Protection for IEE(IEE_SIP)" + help + Protects critical instructions that may break the isolation of IEE, + such as writing system control registers. These instructions would + be executated inside IEE. + depends on IEE + +config PTP + bool "Page Table Protection(PTP)" + depends on IEE + help + Protect page tables by routing page-table modification paths + through the IEE gate. This keeps page table updates mediated by + the isolated execution environment instead of allowing direct + writes from normal kernel code. + +config PTP_RESERVE_ORDER + depends on PTP + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + range 9 15 + default 12 + +config PTP_S + bool "Page Table Protection - Separation (PTP_S)" + help + PTP_S (kernel/user page separation). Split PTP write paths so user page + tables can use direct writes while kernel tables stay on ptp gate paths, + with user-side safety checks for alias/integrity. + depends on PTP + config CREDP bool "Struct cred protection(CREDP)" depends on IEE diff --git a/arch/arm64/kernel/haoc/iee/iee-func.c b/arch/arm64/kernel/haoc/iee/iee-func.c index 8853b3738ceec..a8458fbfadc28 100644 --- a/arch/arm64/kernel/haoc/iee/iee-func.c +++ b/arch/arm64/kernel/haoc/iee/iee-func.c @@ -224,18 +224,21 @@ void set_iee_address_invalid(unsigned long lm_addr, unsigned int order) * synchronization problems. */ static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, - int use_block_pmd) + int use_block_pmd, bool writable) { #ifdef CONFIG_PTP iee_rw_gate(IEE_OP_SET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, - use_block_pmd); + use_block_pmd, writable); #else int i; if (use_block_pmd) { pmd_t pmd = __pmd(pte_val(__ptep_get(lm_ptep))); - pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + if (writable) + pmd = __pmd((pmd_val(pmd) & ~PMD_SECT_RDONLY) | PTE_DBM); + else + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); for (i = 0; i < (1 << order); i++) { pte_t pte = __ptep_get(iee_ptep); @@ -248,7 +251,10 @@ static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, for (i = 0; i < (1 << order); i++) { pte_t pte = __ptep_get(lm_ptep); - pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); + if (writable) + pte = __pte((pte_val(pte) & ~PTE_RDONLY) | PTE_DBM); + else + pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, pte); pte = __ptep_get(iee_ptep); pte = __pte(pte_val(pte) | PTE_VALID); @@ -349,7 +355,7 @@ void put_pages_into_iee_block(unsigned long addr, int order) pmdp = pmd_offset(pudp, iee_addr); iee_ptep = pte_offset_kernel(pmdp, iee_addr); /* Valid the IEE mappings of these pages to enable IEE access. */ - iee_set_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd); + iee_set_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd, false); flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); isb(); } @@ -382,10 +388,15 @@ void put_pages_into_iee(unsigned long addr, int order) } } +void put_pages_into_iee_rw(unsigned long addr, int order) +{ + set_iee_address_valid(addr, order); +} + /* The reverse operation of put_pages_into_iee(). * Call this function when you are returning pages back to kernel. */ -static void remove_pages_from_iee(unsigned long addr, int order) +void remove_pages_from_iee(unsigned long addr, int order) { pgd_t *pgdir = swapper_pg_dir; pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); @@ -425,6 +436,9 @@ void set_iee_page_type(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type /* See put_pages_into_iee(). */ void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) { + if (type == IEE_USER_PGTABLE) + put_pages_into_iee_rw(addr, order); + else put_pages_into_iee(addr, order); iee_set_bitmap_type(addr, 1UL << order, type); } diff --git a/arch/arm64/kernel/haoc/ptp/ptp.c b/arch/arm64/kernel/haoc/ptp/ptp.c index 0c2a2e50e33bb..6bc000906c8ba 100644 --- a/arch/arm64/kernel/haoc/ptp/ptp.c +++ b/arch/arm64/kernel/haoc/ptp/ptp.c @@ -7,6 +7,110 @@ #include #include +#ifdef CONFIG_PTP_S +static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte); +static inline bool check_pte_dep(char *addr, pte_t pte); +static inline bool check_pmd_dep(char *addr, pmd_t pmd); + +bool ptp_is_user_pgtable(const void *ptp) +{ + unsigned long check_addr = (unsigned long)ptp; + + if (!(haoc_enabled && haoc_bitmap_ready && iee_init_done)) + return false; + + /* + * iee_get_bitmap_type() relies on __va_to_haoc_bitmap(). + * Normalize IEE alias addresses back to linear-map address first. + */ + if (unlikely(!__is_lm_address((u64)check_addr))) { + if (!(check_addr & IEE_OFFSET)) + return false; + + check_addr = (unsigned long)__iee_to_virt(check_addr); + if (unlikely(!__is_lm_address((u64)check_addr))) + return false; + } + + if (unlikely(!virt_addr_valid((void *)check_addr))) + return false; + + return iee_get_bitmap_type(check_addr) == IEE_USER_PGTABLE; +} + +/* + * User fast-path pre-check for PTE updates under CONFIG_PTP_S. + * Preserve alias/integrity checks before direct write. + */ +void ptp_user_check_pte_update(pte_t *ptep, pte_t pte) +{ + char *addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); + + if (!(pte_val(pte) & PTE_VALID)) + return; + + if (haoc_enabled && !is_changing_pte_prot(ptep, pte) && + check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) + panic("You are remmaping IEE page to other VA.\n"); + + // Avoid mapping a writable VA to kernel code PA. + check_pte_dep(addr, pte); + +} + +/* + * User fast-path pre-check for PMD updates under CONFIG_PTP_S. + * For leaf PMD mappings, preserve alias checking before direct write. + */ +void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd) +{ + char *addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); + + if (!(pmd_val(pmd) & PMD_SECT_VALID)) + return; + + // Check if the pte table is legally allocated. + if (haoc_enabled && (pmd_val(pmd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) + panic("You can't use non-iee-pgtable\n"); + + // Avoid mapping a huge pmd to IEE physical page. + // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) + // panic("Mapping IEE physical page to a huge pmd.\n"); + + check_pmd_dep(addr, pmd); +} + +/* + * User fast-path pre-check for PUD updates under CONFIG_PTP_S. + * For leaf PUD mappings, preserve alias checking before direct write. + */ +void ptp_user_check_pud_update(pud_t *pudp, pud_t pud) +{ + if (!(pud_val(pud) & PMD_SECT_VALID)) + return; + + if (haoc_enabled && (pud_val(pud) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) + panic("You can't use non-iee-pgtable\n"); +} + +/* + * User fast-path pre-check for P4D/PGD updates under CONFIG_PTP_S. + * A bad top-level table entry poisons later fault walks before lower-level + * checks can run, so keep the same target-page validation here too. + */ +void ptp_user_check_p4d_update(p4d_t *p4dp, p4d_t p4d) +{ + if (!(p4d_val(p4d) & PMD_SECT_VALID)) + return; + + if (haoc_enabled && (p4d_val(p4d) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) + panic("You can't use non-iee-pgtable\n"); +} +#endif + bool check_addr_in_iee_valid(unsigned long addr) { pgd_t *pgdir = swapper_pg_dir; @@ -209,7 +313,7 @@ pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, } void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, - int order, int use_block_pmd) + int order, int use_block_pmd, bool writable) { int i; @@ -219,13 +323,19 @@ void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep #ifdef CONFIG_IEE_ALLOW_SPLIT_LM pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); - pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + if (writable) + pmd = __pmd((pmd_val(pmd) & ~PMD_SECT_RDONLY) | PTE_DBM); + else + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); #else if (order == IEE_DATA_ORDER) { pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); - pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + if (writable) + pmd = __pmd((pmd_val(pmd) & ~PMD_SECT_RDONLY) | PTE_DBM); + else + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); } else { /* Give up RO protection on linear mapping to avoid page table spliting. */ @@ -243,7 +353,10 @@ void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep for (i = 0; i < (1 << order); i++) { pte_t pte = READ_ONCE(*lm_ptep); - pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); + if (writable) + pte = __pte((pte_val(pte) & ~PTE_RDONLY) | PTE_DBM); + else + pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, pte); pte = READ_ONCE(*iee_ptep); pte = __pte(pte_val(pte) | PTE_VALID); diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 68aae0e4caa54..0e38fe006e9b5 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -376,6 +376,10 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) if (haoc_enabled) { iee_cache_init(&pg_cache, 0, CONFIG_PGTABLE_LEVELS, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); +#ifdef CONFIG_PTP_S + iee_cache_init(&pg_user_cache, 0, CONFIG_PGTABLE_LEVELS, IEE_USER_PGTABLE, + CONFIG_PTP_RESERVE_ORDER); +#endif } #endif diff --git a/include/linux/ptp-cache.h b/include/linux/ptp-cache.h index b769920e4b3f7..b102b4e9bf1af 100644 --- a/include/linux/ptp-cache.h +++ b/include/linux/ptp-cache.h @@ -33,6 +33,7 @@ struct iee_cache { extern struct iee_cache pg_cache; #ifdef CONFIG_PTP_S extern struct iee_cache pg_user_cache; +extern bool ptp_is_user_pgtable(const void *ptp); #endif extern void iee_cache_init(struct iee_cache *cache, unsigned long object_order, int levels, enum HAOC_BITMAP_TYPE bitmap_type, diff --git a/mm/haoc/ptp-pg_cache.c b/mm/haoc/ptp-pg_cache.c index 5ded80fe3f0f9..fcbf1823f5998 100644 --- a/mm/haoc/ptp-pg_cache.c +++ b/mm/haoc/ptp-pg_cache.c @@ -14,7 +14,7 @@ #include #include #include -#ifdef CONFIG_PTP_S +#if defined(CONFIG_PTP_S) && defined(CONFIG_X86_64) #include #endif @@ -214,6 +214,11 @@ static __maybe_unused int __ref iee_cache_expand(struct iee_cache *cache, gfp_t tid = READ_ONCE(cache->tid); barrier(); freelist = READ_ONCE(cache->freelist); +#ifdef CONFIG_PTP_S + if (cache->name == IEE_USER_PGTABLE) + __set_freepointer((void *)(end_addr - object_size), freelist); + else +#endif __iee_set_freepointer((void *)(end_addr - object_size), freelist); if (unlikely(!__update_freelist(cache, freelist, (void *)start_addr, tid))) goto redo; @@ -233,7 +238,11 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) void *next_object; #ifdef CONFIG_PTP_S +#ifdef CONFIG_ARM64 + if (haoc_enabled && iee_init_done) +#else if (haoc_enabled && haoc_init_done) +#endif if ((gfp & __GFP_ACCOUNT) && cache == &pg_cache) cache = &pg_user_cache; #endif @@ -259,7 +268,7 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) if (!object) return NULL; set_iee_address_valid((unsigned long)object, cache->object_order); - iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, IEE_PGTABLE); + iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, cache->name); #ifdef DEBUG WARN_ONCE(1, "IEE: Failed on HAOC_BITMAP_TYPE %u expansion.", (unsigned int)cache->name); @@ -274,7 +283,7 @@ void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) set_iee_page((unsigned long)object, 1 << cache->object_order, cache->name); #endif } else { - // fast path alloc + /* fast path alloc */ next_object = __get_freepointer(object); if (unlikely(!__update_freelist(cache, object, next_object, tid))) goto redo; From 7390aec98b9a51f264be5c6a3af974dd9482588e Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Mon, 17 Aug 2026 23:44:29 +0800 Subject: [PATCH 18/26] HAOC/arm64/IEE: Fix: Mark __create_pgd_mapping_for_iee() as __init Mark the HAOC4.0 mapping wrapper as init-only to match its only caller and lifetime. The shared locked helper remains available to runtime users, so this adaptation does not copy the original annotation beyond the wrapper. Original commit message: arm64/iee: mark __create_pgd_mapping_for_iee as __init modpost reports a section mismatch: WARNING: modpost: vmlinux: section mismatch in reference: __create_pgd_mapping_for_iee+0x54 (section: .text.unlikely.) -> __create_pgd_mapping_for_iee_locked (section: .init.text) __create_pgd_mapping_for_iee() lacks a __init annotation, so the compiler places it in a non-init section (.text.unlikely) while it calls __create_pgd_mapping_for_iee_locked(), which lives in .init.text. After init memory is freed, that reference would dangle, hence the mismatch warning. The only caller is __map_memblock_for_iee(), itself __init, reached exclusively from iee_init_mappings() during paging_init(). The call can never happen at runtime, so the wrapper belongs in .init.text too. Annotate it __init instead of __ref: the reference is genuinely init-only, and this also lets the wrapper's code be freed by free_initmem(), slightly reducing the runtime memory footprint. Fixes: 76baf5e2e1ad ("HAOC: Add support for AArch64 Isolated Execution Environment(IEE).") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/arm64/kernel/haoc/iee/iee-mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/haoc/iee/iee-mmu.c b/arch/arm64/kernel/haoc/iee/iee-mmu.c index e3b5d675ea0fe..7e47b0e6f217b 100644 --- a/arch/arm64/kernel/haoc/iee/iee-mmu.c +++ b/arch/arm64/kernel/haoc/iee/iee-mmu.c @@ -497,7 +497,7 @@ void __iee_create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, } while (pgdp++, addr = next, addr != end); } -static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, +static void __init __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(int), From b0f55b39af9831fc59015578fcf5504d9f7f1434 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Mon, 17 Aug 2026 23:46:15 +0800 Subject: [PATCH 19/26] HAOC/arm64/IEE: Fix: Mark iee_init_mappings() declaration as __init Add the missing __init annotation to the public declaration so it matches the definition and the declaration in iee-mmu.h. This documents the init-only lifetime without changing behavior. Original commit message: arm64/iee: add __init to iee_init_mappings() declaration in iee.h The declaration of iee_init_mappings() in asm/haoc/iee.h lacks the __init annotation present on its definition in iee-mmu.c and on the duplicate declaration in asm/haoc/iee-mmu.h. Keep the annotations consistent so the init-only nature of the function is visible at every declaration site. No functional change. Fixes: 76baf5e2e1ad ("HAOC: Add support for AArch64 Isolated Execution Environment(IEE).") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/arm64/include/asm/haoc/iee.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/haoc/iee.h b/arch/arm64/include/asm/haoc/iee.h index c0024ff619ff9..be5fde9e1f4e1 100644 --- a/arch/arm64/include/asm/haoc/iee.h +++ b/arch/arm64/include/asm/haoc/iee.h @@ -74,7 +74,7 @@ extern bool haoc_enabled; #define TCR_HPD1 (UL(1) << 42) -void iee_init_mappings(pgd_t *pgdp); +void __init iee_init_mappings(pgd_t *pgdp); void iee_init_post(void); void iee_stack_init(void); void iee_init_tcr(void); From bbcb4268177da4f82e526b1d741db04f68f4aa2f Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Mon, 17 Aug 2026 23:49:26 +0800 Subject: [PATCH 20/26] HAOC/x86/IEE-PTP: Fix: Make gate frame setup objtool-clean Build valid frame-pointer chains in the HAOC4.0 x86 gates before saving gate state or calling C code. Unlike the original patch, the current gates no longer switch stacks and therefore remain regular SYM_FUNC symbols that objtool can validate. Original commit message: HAOC: IEE: Make x86 gates frame-pointer and objtool clean The hand-written x86 IEE gates are declared as SYM_FUNC but never set up a frame pointer, and they switch to the per-cpu IEE stack in the middle of the function before calling into C code: vmlinux.o: warning: objtool: iee_rwx_gate+0x38: call without frame pointer save/setup With CONFIG_STACK_VALIDATION=y and CONFIG_FRAME_POINTER=y, objtool requires a valid stack frame before any call instruction. The same defect exists in iee_rw_gate and surfaces as soon as its dispatch call becomes a direct one. Establish a real %rbp frame chain in both gates so the frame-pointer unwinder can walk across the stack switch at runtime: the saved %rbp value links the callee frame back to the caller frame regardless of which stack each frame lives on. objtool itself provably cannot model a mid-function switch to a per-cpu stack: once the CFA is %rbp-anchored, "mov %reg, %rsp" degrades it to CFI_UNDEFINED, and the only modelled stack-switch idiom (the "stack swizzle") yields CFI_SP_INDIRECT, which also fails has_valid_stack_frame(). Follow the x86 entry code convention for such code: declare the gates SYM_CODE_* (STT_NOTYPE) to exempt them from function stack validation while keeping the manually maintained frame chain. SYM_CODE_START does not emit ENDBR64. Add it explicitly to both gates; iee_rw_gate is exported, and CONFIG_X86_KERNEL_IBT=y makes objtool flag data relocations (from the __ksymtab entry) to non-ENDBR targets: vmlinux.o: warning: objtool: .export_symbol+0x1348: data relocation to !ENDBR: iee_rw_gate+0x0 Fixes: a8edec335283 ("HAOC: Add support for x86 Isolated Execution Environment") Fixes: 7d1832b64006 ("Haoc: Add support for x86 Sensitive Instruction Protection(IEE_SIP)") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/x86/kernel/haoc/iee/iee-gate.S | 12 ++++++++++++ arch/x86/kernel/haoc/ptp/ptp-gate.S | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index bb0e2742dccaa..62aa1b1f7ab96 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -2,12 +2,16 @@ #include "../gate.h" SYM_FUNC_START(iee_rw_gate) + pushq %rbp + movq %rsp, %rbp + cmpb $0, haoc_enabled(%rip) jnz .L_haoc_enabled_rw .L_haoc_disabled_rw: leaq iee_funcs(%rip), %rax call *(%rax, %rdi, 8) + popq %rbp jmp __x86_return_thunk .L_haoc_enabled_rw: @@ -28,6 +32,7 @@ SYM_FUNC_START(iee_rw_gate) popq %r12 popfq + popq %rbp jmp __x86_return_thunk rw_wp_enabled: @@ -45,6 +50,7 @@ rw_wp_enabled: /* restore irq*/ popfq + popq %rbp jmp __x86_return_thunk SYM_FUNC_END(iee_rw_gate) #if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) @@ -53,11 +59,15 @@ EXPORT_SYMBOL(iee_rw_gate) #ifdef CONFIG_IEE_SIP SYM_FUNC_START(iee_rwx_gate) + pushq %rbp + movq %rsp, %rbp + cmpb $0, haoc_enabled(%rip) jnz .L_haoc_enabled_rwx .L_haoc_disabled_rwx: call _iee_si_handler + popq %rbp jmp __x86_return_thunk .L_haoc_enabled_rwx: @@ -88,6 +98,7 @@ SYM_FUNC_START(iee_rwx_gate) popfq popq %r12 + popq %rbp jmp __x86_return_thunk /* ret */ rwx_wp_enabled: @@ -115,6 +126,7 @@ rwx_wp_enabled: popfq popq %r12 + popq %rbp jmp __x86_return_thunk /* ret */ SYM_FUNC_END(iee_rwx_gate) #endif diff --git a/arch/x86/kernel/haoc/ptp/ptp-gate.S b/arch/x86/kernel/haoc/ptp/ptp-gate.S index e56f184f909e0..b9d0eda154ecf 100644 --- a/arch/x86/kernel/haoc/ptp/ptp-gate.S +++ b/arch/x86/kernel/haoc/ptp/ptp-gate.S @@ -85,12 +85,16 @@ cmpxchg_wp_enabled: SYM_FUNC_END(ptp_try_cmpxchg) SYM_FUNC_START(ptp_rw_gate) + pushq %rbp + movq %rsp, %rbp + cmpb $0, haoc_enabled(%rip) jnz .L_haoc_enabled_rw .L_haoc_disabled_rw: leaq iee_funcs(%rip), %rax call *(%rax, %rdi, 8) + popq %rbp jmp __x86_return_thunk .L_haoc_enabled_rw: @@ -110,6 +114,7 @@ SYM_FUNC_START(ptp_rw_gate) popq %r12 popfq + popq %rbp jmp __x86_return_thunk ptprw_wp_enabled: @@ -126,6 +131,7 @@ ptprw_wp_enabled: /* restore irq*/ popfq + popq %rbp jmp __x86_return_thunk /* ret */ SYM_FUNC_END(ptp_rw_gate) EXPORT_SYMBOL(ptp_rw_gate) From 025dd1ed972525dd7cc56e3c0d8a5a51fe5434a0 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Tue, 18 Aug 2026 00:34:51 +0800 Subject: [PATCH 21/26] HAOC/x86/IEE-PTP: Fix: Replace the writable gate dispatch table Replace the writable iee_funcs table and raw indirect gate calls with an exhaustive C dispatcher adapted to the HAOC4.0 operation set. Preserve the six-register gate ABI, cover the PTP gate, and fail closed for unknown operations. Original commit message: HAOC: IEE: Replace indirect function table with a C dispatcher The x86 IEE access gate dispatches IEE operations through iee_funcs[], a plain writable .data function pointer table, using a raw indirect call in hand-written assembly: vmlinux.o: warning: objtool: iee_rw_gate+0x2a: indirect call found in RETPOLINE build In a CONFIG_RETPOLINE=y build every indirect branch must go through the retpoline thunks; the gate's "call *(%rax,%rdi,8)" bypasses them, leaving an unprotected indirect branch (Spectre v2/BTI) inside the entry point of a security mechanism. The table makes this worse: under HAOC's own threat model an attacker with arbitrary kernel write (or ROP into the gate with a controlled %rdi) can redirect or index off the writable table and turn the gate into a call-anywhere gadget that runs with CR0.WP cleared. Replace the table with iee_dispatch(), a C function that switches on the op flag and calls each _iee_*() implementation directly. All callees take at most three register arguments and absorb the flag in their first (unused) parameter, so the existing gate ABI is preserved bit-for-bit while the compiler generates retpoline/objtool/IBT compliant code. This also removes the writable dispatch table from the kernel image entirely. An unknown flag used to dereference past the table (a NULL entry or adjacent .data) and jump to a wild pointer. Fail closed instead: panic() unconditionally. BUG() was considered and rejected: with CONFIG_PANIC_ON_OOPS unset and nearly all gate callers in process context, die() would merely kill the current task, leaving the CPU running with CR0.WP cleared, interrupts disabled and the IEE stack leaked -- exactly the state IEE_SIP exists to prevent. Fixes: a8edec335283 ("HAOC: Add support for x86 Isolated Execution Environment") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/x86/include/asm/haoc/haoc.h | 4 + arch/x86/kernel/haoc/haoc.c | 423 ++++++++++++++++++++++------ arch/x86/kernel/haoc/iee/iee-gate.S | 9 +- arch/x86/kernel/haoc/ptp/ptp-gate.S | 9 +- 4 files changed, 341 insertions(+), 104 deletions(-) diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index 82565551065f8..55c1cb0f79b05 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -155,4 +155,8 @@ void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, struct key_restriction *restrict_link); unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); #endif + +unsigned long iee_dispatch(unsigned long flag, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5); #endif diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 280c83a31cf0d..014f8d2f624ab 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -7,111 +7,350 @@ * Hu Bing */ +#include +#include #include +#include +#include -typedef void (*iee_func)(void); -iee_func iee_funcs[] = { - (iee_func)_iee_memcpy, - (iee_func)_iee_memset, - (iee_func)_iee_set_freeptr, - (iee_func)_iee_test_and_clear_bit, +notrace unsigned long iee_dispatch(unsigned long flag, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5) +{ + switch (flag) { + case IEE_OP_MEMCPY: + _iee_memcpy(flag, (void *)arg1, (void *)arg2, (size_t)arg3); + break; + case IEE_OP_MEMSET: + _iee_memset(flag, (void *)arg1, (int)arg2, (size_t)arg3); + break; + case IEE_OP_SET_FREEPTR: + _iee_set_freeptr(flag, (void **)arg1, (void *)arg2); + break; + case IEE_OP_TEST_CLEAR_BIT: + return _iee_test_and_clear_bit(flag, (long)arg1, + (unsigned long *)arg2); #ifdef CONFIG_IEE_PTRP - (iee_func)_iee_set_token_pgd, - (iee_func)_iee_invalidate_token, - (iee_func)_iee_validate_token, + case IEE_OP_SET_TOKEN_PGD: + _iee_set_token_pgd(flag, (struct task_struct *)arg1, + (pgd_t *)arg2); + break; + case IEE_OP_INVALIDATE_TOKEN: + _iee_invalidate_token(flag, (struct task_struct *)arg1); + break; + case IEE_OP_VALIDATE_TOKEN: + _iee_validate_token(flag, (struct task_struct *)arg1); + break; #endif - (iee_func)_iee_set_bitmap_type, + case IEE_OP_SET_BITMAP_TYPE: + _iee_set_bitmap_type(flag, (uint8_t *)arg1, + (enum HAOC_BITMAP_TYPE)arg2, (int)arg3); + break; #ifdef CONFIG_CREDP - (iee_func)_iee_copy_cred, - (iee_func)_iee_copy_kernel_cred, - (iee_func)_iee_init_copied_cred, - (iee_func)_iee_commit_creds, - (iee_func)_iee_abort_cred, - (iee_func)_iee_fill_cred_for_session_keyring, - (iee_func)_iee_override_creds, - (iee_func)_iee_revert_creds, - (iee_func)_iee_set_cred_uid, - (iee_func)_iee_set_cred_gid, - (iee_func)_iee_set_cred_suid, - (iee_func)_iee_set_cred_sgid, - (iee_func)_iee_set_cred_euid, - (iee_func)_iee_set_cred_egid, - (iee_func)_iee_set_cred_fsuid, - (iee_func)_iee_set_cred_fsgid, - (iee_func)_iee_set_cred_user, - (iee_func)_iee_set_cred_user_ns, - (iee_func)_iee_set_cred_group_info, - (iee_func)_iee_set_cred_securebits, - (iee_func)_iee_set_cred_cap_inheritable, - (iee_func)_iee_set_cred_cap_permitted, - (iee_func)_iee_set_cred_cap_effective, - (iee_func)_iee_set_cred_cap_bset, - (iee_func)_iee_set_cred_cap_ambient, - (iee_func)_iee_set_cred_jit_keyring, - (iee_func)_iee_set_cred_session_keyring, - (iee_func)_iee_set_cred_process_keyring, - (iee_func)_iee_set_cred_thread_keyring, - (iee_func)_iee_set_cred_request_key_auth, - (iee_func)_iee_set_cred_non_rcu, - (iee_func)_iee_set_cred_atomic_set_usage, - (iee_func)_iee_set_cred_atomic_op_usage, - (iee_func)_iee_set_cred_security, - (iee_func)_iee_set_cred_rcu, - (iee_func)_iee_set_cred_ucounts, + case IEE_OP_COPY_CRED: + _iee_copy_cred(flag, (struct cred *)arg1); + break; + case IEE_OP_COPY_KERNEL_CRED: + _iee_copy_kernel_cred(flag, (const struct cred *)arg1, + (struct cred *)arg2); + break; + case IEE_OP_INIT_COPIED_CRED: + _iee_init_copied_cred(flag, (struct task_struct *)arg1, + (struct cred *)arg2); + break; + case IEE_OP_COMMIT_CRED: + _iee_commit_creds(flag, (const struct cred *)arg1); + break; + case IEE_OP_ABORT_CRED: + _iee_abort_cred(flag, (const struct cred *)arg1); + break; + case IEE_OP_FILL_SESSION_KEYRING_CRED: + _iee_fill_cred_for_session_keyring(flag, (struct cred *)arg1, + (const struct cred *)arg2); + break; + case IEE_OP_OVERRIDE_CRED: + _iee_override_creds(flag, (const struct cred *)arg1); + break; + case IEE_OP_REVERT_CRED: + _iee_revert_creds(flag, (const struct cred *)arg1); + break; + case IEE_OP_SET_CRED_UID: + _iee_set_cred_uid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_GID: + _iee_set_cred_gid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_SUID: + _iee_set_cred_suid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_SGID: + _iee_set_cred_sgid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_EUID: + _iee_set_cred_euid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_EGID: + _iee_set_cred_egid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_FSUID: + _iee_set_cred_fsuid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_FSGID: + _iee_set_cred_fsgid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_USER: + _iee_set_cred_user(flag, (struct cred *)arg1, + (struct user_struct *)arg2); + break; + case IEE_OP_SET_CRED_USER_NS: + _iee_set_cred_user_ns(flag, (struct cred *)arg1, + (struct user_namespace *)arg2); + break; + case IEE_OP_SET_CRED_GROUP_INFO: + _iee_set_cred_group_info(flag, (struct cred *)arg1, + (struct group_info *)arg2); + break; + case IEE_OP_SET_CRED_SECUREBITS: + _iee_set_cred_securebits(flag, (struct cred *)arg1, + (unsigned int)arg2); + break; + case IEE_OP_SET_CRED_CAP_INHER: + _iee_set_cred_cap_inheritable(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_PERM: + _iee_set_cred_cap_permitted(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_EFFECT: + _iee_set_cred_cap_effective(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_BSET: + _iee_set_cred_cap_bset(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_AMBIENT: + _iee_set_cred_cap_ambient(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_JIT_KEYRING: + _iee_set_cred_jit_keyring(flag, (struct cred *)arg1, + (unsigned char)arg2); + break; + case IEE_OP_SET_CRED_SESS_KEYRING: + _iee_set_cred_session_keyring(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_PROC_KEYRING: + _iee_set_cred_process_keyring(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_THREAD_KEYRING: + _iee_set_cred_thread_keyring(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_REQ_KEYRING: + _iee_set_cred_request_key_auth(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_NON_RCU: + _iee_set_cred_non_rcu(flag, (struct cred *)arg1, (int)arg2); + break; + case IEE_OP_SET_CRED_ATSET_USAGE: + _iee_set_cred_atomic_set_usage(flag, (struct cred *)arg1, + (int)arg2); + break; + case IEE_OP_SET_CRED_ATOP_USAGE: + return _iee_set_cred_atomic_op_usage(flag, (struct cred *)arg1, + (int)arg2, (int)arg3); + case IEE_OP_SET_CRED_SECURITY: + _iee_set_cred_security(flag, (struct cred *)arg1, (void *)arg2); + break; + case IEE_OP_SET_CRED_RCU: + _iee_set_cred_rcu(flag, (struct cred *)arg1, + (struct rcu_head *)arg2); + break; + case IEE_OP_SET_CRED_UCOUNTS: + _iee_set_cred_ucounts(flag, (struct cred *)arg1, + (struct ucounts *)arg2); + break; #endif #ifdef CONFIG_KEYP - (iee_func)_iee_set_key_union, - (iee_func)_iee_set_key_struct, - (iee_func)_iee_set_key_payload, - (iee_func)_iee_set_key_usage, - (iee_func)_iee_set_key_serial, - (iee_func)_iee_set_key_watchers, - (iee_func)_iee_set_key_user, - (iee_func)_iee_set_key_security, - (iee_func)_iee_set_key_expiry, - (iee_func)_iee_set_key_revoked_at, - (iee_func)_iee_set_key_last_used_at, - (iee_func)_iee_set_key_uid, - (iee_func)_iee_set_key_gid, - (iee_func)_iee_set_key_perm, - (iee_func)_iee_set_key_quotalen, - (iee_func)_iee_set_key_datalen, - (iee_func)_iee_set_key_state, - (iee_func)_iee_set_key_magic, - (iee_func)_iee_set_key_flags, - (iee_func)_iee_set_key_index_key, - (iee_func)_iee_set_key_hash, - (iee_func)_iee_set_key_len_desc, - (iee_func)_iee_set_key_type, - (iee_func)_iee_set_key_domain_tag, - (iee_func)_iee_set_key_description, - (iee_func)_iee_set_key_restrict_link, - (iee_func)_iee_set_key_flag_bit, + case IEE_OP_SET_KEY_UNION: + _iee_set_key_union(flag, (struct key *)arg1, + (struct key_union *)arg2); + break; + case IEE_OP_SET_KEY_STRUCT: + _iee_set_key_struct(flag, (struct key *)arg1, + (struct key_struct *)arg2); + break; + case IEE_OP_SET_KEY_PAYLOAD: + _iee_set_key_payload(flag, (struct key *)arg1, + (union key_payload *)arg2); + break; + case IEE_OP_SET_KEY_USAGE: + return _iee_set_key_usage(flag, (struct key *)arg1, (int)arg2, + (int)arg3); + case IEE_OP_SET_KEY_SERIAL: + _iee_set_key_serial(flag, (struct key *)arg1, + (key_serial_t)arg2); + break; + case IEE_OP_SET_KEY_WATCHERS: + _iee_set_key_watchers(flag, (struct key *)arg1, + (struct watch_list *)arg2); + break; + case IEE_OP_SET_KEY_USERS: + _iee_set_key_user(flag, (struct key *)arg1, + (struct key_user *)arg2); + break; + case IEE_OP_SET_KEY_SECURITY: + _iee_set_key_security(flag, (struct key *)arg1, (void *)arg2); + break; + case IEE_OP_SET_KEY_EXPIRY: + _iee_set_key_expiry(flag, (struct key *)arg1, (time64_t)arg2); + break; + case IEE_OP_SET_KEY_REVOKED_AT: + _iee_set_key_revoked_at(flag, (struct key *)arg1, + (time64_t)arg2); + break; + case IEE_OP_SET_KEY_LAST_USED_AT: + _iee_set_key_last_used_at(flag, (struct key *)arg1, + (time64_t)arg2); + break; + case IEE_OP_SET_KEY_UID: + _iee_set_key_uid(flag, (struct key *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_KEY_GID: + _iee_set_key_gid(flag, (struct key *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_KEY_PERM: + _iee_set_key_perm(flag, (struct key *)arg1, (key_perm_t)arg2); + break; + case IEE_OP_SET_KEY_QUOTALEN: + _iee_set_key_quotalen(flag, (struct key *)arg1, + (unsigned short)arg2); + break; + case IEE_OP_SET_KEY_DATALEN: + _iee_set_key_datalen(flag, (struct key *)arg1, + (unsigned short)arg2); + break; + case IEE_OP_SET_KEY_STATE: + _iee_set_key_state(flag, (struct key *)arg1, (short)arg2); + break; + case IEE_OP_SET_KEY_MAGIC: + _iee_set_key_magic(flag, (struct key *)arg1, + (unsigned int)arg2); + break; + case IEE_OP_SET_KEY_FLAGS: + _iee_set_key_flags(flag, (struct key *)arg1, arg2); + break; + case IEE_OP_SET_KEY_INDEX_KEY: + _iee_set_key_index_key(flag, (struct key *)arg1, + (struct keyring_index_key *)arg2); + break; + case IEE_OP_SET_KEY_HASH: + _iee_set_key_hash(flag, (struct key *)arg1, arg2); + break; + case IEE_OP_SET_KEY_LEN_DESC: + _iee_set_key_len_desc(flag, (struct key *)arg1, arg2); + break; + case IEE_OP_SET_KEY_TYPE: + _iee_set_key_type(flag, (struct key *)arg1, + (struct key_type *)arg2); + break; + case IEE_OP_SET_KEY_TAG: + _iee_set_key_domain_tag(flag, (struct key *)arg1, + (struct key_tag *)arg2); + break; + case IEE_OP_SET_KEY_DESCRIPTION: + _iee_set_key_description(flag, (struct key *)arg1, (char *)arg2); + break; + case IEE_OP_SET_KEY_RESTRICT_LINK: + _iee_set_key_restrict_link(flag, (struct key *)arg1, + (struct key_restriction *)arg2); + break; + case IEE_OP_SET_KEY_FLAG_BIT: + return _iee_set_key_flag_bit(flag, (struct key *)arg1, (long)arg2, + (int)arg3); #endif #ifdef CONFIG_IEE_SELINUX_P - (iee_func)_iee_set_selinux_status_pg, - (iee_func)_iee_set_selinux_enforcing, - (iee_func)_iee_mark_selinux_initialized, - (iee_func)_iee_set_sel_policy_cap, - (iee_func)_iee_sel_rcu_assign_policy, + case IEE_SEL_SET_STATUS_PG: + _iee_set_selinux_status_pg(flag, (struct page *)arg1); + break; + case IEE_SEL_SET_ENFORCING: + _iee_set_selinux_enforcing(flag, (bool)arg1); + break; + case IEE_SEL_SET_INITIALIZED: + _iee_mark_selinux_initialized(flag); + break; + case IEE_SEL_SET_POLICY_CAP: + _iee_set_sel_policy_cap(flag, (unsigned int)arg1, (int)arg2); + break; + case IEE_SEL_RCU_ASSIGN_POLICY: + _iee_sel_rcu_assign_policy(flag, (struct selinux_policy *)arg1, + (struct selinux_policy *)arg2); + break; #endif #ifdef CONFIG_PTP #ifdef CONFIG_IEE_PTRP - (iee_func)_iee_unset_token, - (iee_func)_iee_set_token, + case IEE_OP_UNSET_TOKEN: + _iee_unset_token(flag, (pte_t *)arg1, (pte_t *)arg2, arg3, + (unsigned int)arg4); + break; + case IEE_OP_SET_TOKEN: + _iee_set_token(flag, (pte_t *)arg1, (pte_t *)arg2, arg3, + (unsigned int)arg4); + break; #endif - (iee_func)_iee_set_pte, - (iee_func)_iee_set_pmd, - (iee_func)_iee_set_pud, - (iee_func)_iee_set_p4d, - (iee_func)_iee_set_pgd, - (iee_func)_iee_set_pte_text_poke, - (iee_func)_iee_set_pmd_ident, - (iee_func)_iee_set_pud_ident, - (iee_func)_iee_set_sx_pte, + case IEE_OP_SET_PTE: + _iee_set_pte(flag, (pte_t *)arg1, __pte(arg2)); + break; + case IEE_OP_SET_PMD: + _iee_set_pmd(flag, (pmd_t *)arg1, __pmd(arg2)); + break; + case IEE_OP_SET_PUD: + _iee_set_pud(flag, (pud_t *)arg1, __pud(arg2)); + break; + case IEE_OP_SET_P4D: + _iee_set_p4d(flag, (p4d_t *)arg1, __p4d(arg2)); + break; + case IEE_OP_SET_PGD: + _iee_set_pgd(flag, (pgd_t *)arg1, __pgd(arg2)); + break; + case IEE_OP_SET_PTE_TEXT_POKE: + _iee_set_pte_text_poke(flag, (pte_t *)arg1, __pte(arg2)); + break; + case IEE_OP_SET_PMD_IDENT: + _iee_set_pmd_ident(flag, (pmd_t *)arg1, __pmd(arg2)); + break; + case IEE_OP_SET_PUD_IDENT: + _iee_set_pud_ident(flag, (pud_t *)arg1, __pud(arg2)); + break; + case IEE_OP_SET_SX_PTE: + _iee_set_sx_pte(flag, (pte_t *)arg1, __pte(arg2)); + break; #endif #ifdef CONFIG_VARP - (iee_func)_iee_set_varp_modprobe_path, + case IEE_OP_SET_VARP_MODPROBE_PATH: + _iee_set_varp_modprobe_path(flag, (char *)arg1, (int)arg2, + (size_t)arg3, (char *)arg4, + (size_t *)arg5); + break; #endif - NULL -}; + default: + panic("iee_dispatch: invalid flag %lu\n", flag); + } + + return 0; +} diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index 62aa1b1f7ab96..7ebb5df3a31a2 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -9,8 +9,7 @@ SYM_FUNC_START(iee_rw_gate) jnz .L_haoc_enabled_rw .L_haoc_disabled_rw: - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + call iee_dispatch popq %rbp jmp __x86_return_thunk @@ -27,8 +26,7 @@ SYM_FUNC_START(iee_rw_gate) testq %r12, %r12 jz rw_wp_enabled - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + call iee_dispatch popq %r12 popfq @@ -40,8 +38,7 @@ rw_wp_enabled: DISABLE_WP r12 /* call iee func */ - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + call iee_dispatch ENABLE_WP r12 diff --git a/arch/x86/kernel/haoc/ptp/ptp-gate.S b/arch/x86/kernel/haoc/ptp/ptp-gate.S index b9d0eda154ecf..1655465b175e6 100644 --- a/arch/x86/kernel/haoc/ptp/ptp-gate.S +++ b/arch/x86/kernel/haoc/ptp/ptp-gate.S @@ -92,8 +92,7 @@ SYM_FUNC_START(ptp_rw_gate) jnz .L_haoc_enabled_rw .L_haoc_disabled_rw: - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + call iee_dispatch popq %rbp jmp __x86_return_thunk @@ -109,8 +108,7 @@ SYM_FUNC_START(ptp_rw_gate) testq %r12, %r12 jz ptprw_wp_enabled - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + call iee_dispatch popq %r12 popfq @@ -121,8 +119,7 @@ ptprw_wp_enabled: DISABLE_WP r12 /* call iee func */ - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + call iee_dispatch ENABLE_WP r12 From 614ffd7bd575889dff3244d8827763052ac24e4c Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Tue, 18 Aug 2026 01:30:36 +0800 Subject: [PATCH 22/26] HAOC/x86/IEE_PTRP: Fix: Allocate a dedicated init_task token page Allocate and map a dedicated HAOC4.0 token page for the static init_task before its first credential operation. Enable HAOC before validating the mapping so writes use the protected gate; keep the existing optional IEE_PTRP configuration semantics. Original commit message: HAOC: IEE: Give init_task a dedicated token page on x86 With CREDP enabled, every credential operation performed by swapper/0 corrupts init_task. The CREDP token accessors locate a task's struct task_token through __addr_to_iee(tsk): for slab-allocated tasks the IEE alias was remapped to a dedicated token page by iee_alloc_task_token_slab(), but init_task is a kernel image symbol, so its IEE alias (via __kimg_to_iee()) still points at its own physical page, and the token writes the gate performs with CR0.WP cleared land on init_task itself. struct task_token overlays the start of task_struct: token.new_cred (+40) covers task_struct.usage (+40) and task_struct.flags (+44). While swapper forks PID 1/2, _iee_copy_cred() stores the new cred pointer into token.new_cred, i.e. init_task.usage/flags transiently hold the two halves of a direct-mapped pointer until _iee_init_copied_cred() clears them; afterwards init_task.flags is left as 0 (even PF_KTHREAD is lost). The window between the two gate calls is wide (LSM hooks, key management, ...), and if the timer tick lands in it, scheduler_tick() reads that garbage flags word on current. Whenever the randomized direct-map base makes the PF_WQ_WORKER bit (0x20) of the transient value set -- which happens on roughly half of the boots since CONFIG_RANDOMIZE_MEMORY randomizes page_offset_base -- wq_worker_tick() runs on swapper and dereferences kthread_data(&init_task), which is legitimately NULL: [ 0.578469] Oops: 0000 [#1] PREEMPT SMP NOPTI [ 0.578469] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G S [ 0.578469] RIP: 0010:wq_worker_tick+0x19/0x180 [ 0.578469] RAX: 0000000000000000 RBX: ffff88a8bec52a40 [ 0.578469] CR2: 0000000000000020 CR4: 00000000001006f0 [ 0.578469] Call Trace: [ 0.578469] [ 0.578469] scheduler_tick+0x13d/0x340 [ 0.578469] update_process_times+0x7b/0x90 [ 0.578469] tick_periodic+0x6b/0x80 [ 0.578469] tick_handle_periodic+0x29/0x90 [ 0.578469] timer_interrupt+0x1d/0x30 [ 0.578469] __common_interrupt+0x49/0xc0 [ 0.578469] common_interrupt+0x92/0xb0 [ 0.578469] [ 0.578469] [ 0.578469] asm_common_interrupt+0x2b/0x40 [ 0.578469] RIP: 0010:iee_rw_gate+0x42/0x50 [ 0.578469] ? prepare_creds+0x87/0x300 [ 0.578469] copy_creds+0x85/0x3a0 [ 0.578469] copy_process+0x32a/0x1070 [ 0.578469] Kernel panic - not syncing: Fatal exception in interrupt The arm64 port already handles this with iee_prepare_init_task_token(), which allocates a dedicated token page for init_task and remaps its IEE alias onto it. Port that to x86, reusing the existing iee_set_token_page_valid() helper, and call it from iee_init(): iee_init() runs in mem_init(), which is before swapper's first credential operation in rest_init(). Note that CREDP's token accesses depend on the token-page remapping machinery, which is only built with IEE_PTRP; without it every token write would write through the 1:1 alias. Make that dependency explicit in Kconfig. Fixes: 416542ddedfbe ("HAOC: Add support for x86 CRED Protection (CREDP).") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/x86/include/asm/haoc/iee-token.h | 4 +++- arch/x86/kernel/haoc/iee/iee-init.c | 6 ++++++ arch/x86/kernel/haoc/iee/iee-token.c | 27 ++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h index 2045a9ae7874c..21dc25f816a02 100644 --- a/arch/x86/include/asm/haoc/iee-token.h +++ b/arch/x86/include/asm/haoc/iee-token.h @@ -3,6 +3,7 @@ #define _LINUX_IEE_TOKEN_H #include #include +#include #include #include #include @@ -13,8 +14,9 @@ extern unsigned long long iee_rw_gate(int flag, ...); #ifdef CONFIG_IEE_PTRP extern struct kmem_cache *task_struct_cachep; +extern void __init iee_prepare_init_task_token(void); extern void iee_set_token_page_valid(unsigned long token, unsigned long new, - unsigned int order); + unsigned int order); extern void iee_set_token_page_invalid(unsigned long token_addr, unsigned long __unused, unsigned int order); diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 131d44b634035..727d53d7de022 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -21,6 +21,9 @@ #ifdef CONFIG_IEE_SIP #include #endif +#ifdef CONFIG_IEE_PTRP +#include +#endif #ifdef CONFIG_PTP #include #include @@ -266,6 +269,9 @@ void __init iee_init(void) cet_disable(); haoc_enabled = true; +#ifdef CONFIG_IEE_PTRP + iee_prepare_init_task_token(); +#endif #ifdef CONFIG_IEE_SIP iee_setup_sip_cr4_pinning(); #endif diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c index d1eceb58c102d..9a2ca697ca0c1 100644 --- a/arch/x86/kernel/haoc/iee/iee-token.c +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -8,8 +9,32 @@ #include #include "slab.h" +/* + * Slab-allocated task_struct objects use dedicated token pages, but init_task + * is a kernel image symbol whose IEE alias initially maps its own page. Give + * init_task a dedicated token page before its first credential operation. + */ +void __init iee_prepare_init_task_token(void) +{ + unsigned long token = (unsigned long)__kimg_to_iee(&init_task); + unsigned long token_page; + unsigned int order = 0; + + if (ALIGN(token + sizeof(struct task_token), PAGE_SIZE) != + ALIGN(token + 1, PAGE_SIZE)) + order = 1; + + token_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!token_page) + panic("IEE: failed to allocate token page for init_task\n"); + + iee_set_token_page_valid(ALIGN_DOWN(token, PAGE_SIZE), token_page, + order); + iee_validate_token(&init_task); +} + void iee_set_token_page_valid(unsigned long token, unsigned long token_page, - unsigned int order) + unsigned int order) { set_memory_4k(token, 1 << order); set_memory_4k(token_page, 1 << order); From 60206c36119dee44e13f35d42886f06ab0986b3c Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Tue, 18 Aug 2026 00:38:04 +0800 Subject: [PATCH 23/26] HAOC/CREDP: Fix: Clear the abort token before dropping the cred reference Move the architecture-independent CREDP token cleanup before put_cred() so both x86 and arm64 finish the protected operation while the credential object is still alive. Original commit message: HAOC: CREDP: Clear abort token before dropping the cred reference abort_creds() drops the caller's reference with put_cred() before asking the IEE to clear token.new_cred. put_cred() may be the last reference and release the object, so iee_abort_creds() potentially runs after the cred has been freed. Clear the token first, while the object is still alive, mirroring how commit_creds() performs the IEE operation before dropping references. Fixes: a7369f49fb0a ("HAOC: CREDP: protect commit_creds() from ROP attack.") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- kernel/cred.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/cred.c b/kernel/cred.c index 7069195fd055b..a9fa46f465d2c 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -619,11 +619,11 @@ void abort_creds(struct cred *new) atomic_long_read(&new->usage)); BUG_ON(atomic_long_read(&new->usage) < 1); - put_cred(new); #ifdef CONFIG_CREDP if (haoc_enabled) iee_abort_creds(new); #endif + put_cred(new); } EXPORT_SYMBOL(abort_creds); From 9484680a42cec5cd5cb451b4edfce3d829d16d70 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Tue, 18 Aug 2026 00:39:55 +0800 Subject: [PATCH 24/26] HAOC/x86/IEE_SIP: Fix: Enforce CR4 pinning before the SI write Validate and correct protected CR4 bits in the IEE_SIP handler before the requested value reaches hardware. Use a local mask for the handler's SMEP and SMAP execution constraints without weakening the global CR4 pinning policy. Original commit message: HAOC: IEE: Enforce CR4 pinning in the SI handler's CR4 write path When haoc_enabled is set, native_write_cr4() routes every CR4 write through iee_write_cr4() -> _iee_si_handler(IEE_WRITE_CR4) before its own pinning check runs. The handler's CR4 branch only stripped SMEP and wrote the value straight to hardware -- unlike the sibling IEE_WRITE_CR0 branch, which faithfully replicates the CR0.WP pinning enforcement. The gate exit path only restores SMEP/SMAP (via iee_cr4_set_mask), so a write that cleared UMIP or FSGSBASE, or set CET, took effect silently and permanently. This renders CR4 pinning -- x86's defense-in-depth against CR4 tampering after a control-flow hijack -- void on haoc kernels with zero warning: - clearing UMIP re-enables userspace SIDT/SGDT/SLDT/SMSW/STR and effectively defeats KASLR; - clearing FSGSBASE #UDs the kernel's own ALTERNATIVE-patched {RD,WR}GSBASE sites; - setting CET makes the gate's CR0.WP toggle #GP (SDM Vol. 3A section 2.5), wedging the IEE gate. The defect was left half-finished from day one: the introducing commit turned cr4_pinned_mask/cr4_pinned_bits/cr_pinning into globals and added extern declarations for them in iee-si.h, yet the handler only ever consumed cr_pinning (in the CR0 branch). Replicate the pinning enforcement in the handler, with two deviations forced by the execution environment: - SMEP is excluded from the check/force set: the handler executes from user-mapped .iee.si_text with SMEP already cleared by the gate, so writing CR4 with SMEP=1 would fault on the very next instruction fetch. The gate exit restores SMEP/SMAP per iee_cr4_set_mask and is deliberately left untouched -- forcing pinned bits in the gate would break setup_smep()/setup_smap(), which legitimately run through the gate before pinning starts. - The value is corrected *before* writing instead of the native write-check-rewrite sequence, so a malicious value never reaches hardware at all; WARN_ONCE() then fires as usual. The check is gated on the cr_pinning static key, exactly like the native path, so early-boot CR4 writes (PSE/PGE/FSGSBASE/SMEP/SMAP/ UMIP setup, all before setup_cr_pinning()) are unaffected. Note that under IEE, CET is pinned to 0 by design: setup_cet() refuses to enable it because the gate's WP toggling conflicts with CR4.CET=1 (commit a8edec335283b: "IEE depends on CR0.wp"). This change preserves that behavior and only restores enforcement for the remaining pinned bits. Verified by booting with haoc=1 under QEMU: IEE shadow mapping, .iee.text U-page remapping and si_test all succeed. Fixes: 7d1832b64006 ("Haoc: Add support for x86 Sensitive Instruction Protection(IEE_SIP)") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/x86/kernel/haoc/iee/iee-si.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/haoc/iee/iee-si.c b/arch/x86/kernel/haoc/iee/iee-si.c index 1a92b76ff4d30..393f683ae9a80 100644 --- a/arch/x86/kernel/haoc/iee/iee-si.c +++ b/arch/x86/kernel/haoc/iee/iee-si.c @@ -38,20 +38,24 @@ unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) } case IEE_WRITE_CR4: { unsigned long bits_changed = 0; + const unsigned long check_mask = + cr4_pinned_mask & ~X86_CR4_SMEP; + const unsigned long check_bits = + cr4_pinned_bits & ~X86_CR4_SMEP; val = va_arg(pArgs, u64); - val &= ~(X86_CR4_SMEP); -set_register_cr4: - asm volatile("mov %0,%%cr4" : "+r" (val) : : "memory"); + val &= ~X86_CR4_SMEP; if (static_branch_likely(&cr_pinning)) { - if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) { - bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits; - val = (val & ~cr4_pinned_mask) | cr4_pinned_bits; - goto set_register_cr4; + if (unlikely((val & check_mask) != check_bits)) { + bits_changed = (val & check_mask) ^ check_bits; + val = (val & ~check_mask) | check_bits; } - WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n", - bits_changed); } + asm volatile("mov %0,%%cr4" : "+r" (val) : : "memory"); + if (static_branch_likely(&cr_pinning)) + WARN_ONCE(bits_changed, + "pinned CR4 bits changed: 0x%lx!?\n", + bits_changed); break; } case IEE_LOAD_IDT: { From ebda71e5c26a03ff967692a1a892ba1c7595fe30 Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Tue, 18 Aug 2026 00:42:11 +0800 Subject: [PATCH 25/26] HAOC/x86/IEE_SIP: Fix: Restore const on cr4_pinned_mask Remove the remaining runtime mutation of the global CR4 pinning mask and make it const again. The preceding handler fix now keeps its execution-specific exceptions local, so the system-wide policy can stay read-only. Original commit message: HAOC: IEE: Restore const on cr4_pinned_mask Upstream keeps cr4_pinned_mask "static const". The IEE_SIP commit dropped both qualifiers so the SI handler could reference it from another TU, and added a runtime reassignment in parse_haoc_enabled() -- which stores exactly the value the initializer already carries (the IEE pinned set is identical to the upstream one), leaving the mask in writable .data for no reason. An attacker with an arbitrary kernel write could zero it and defeat CR4 pinning even on the non-haoc path: a hardening regression against the upstream baseline, not merely a missing improvement. Drop the redundant early-param assignment along with the extern declaration it needed in iee-init.c. With no writer left anywhere, the mask can simply be const again: only the "static" stays dropped (the handler needs cross-TU access), and the extern declaration in iee-si.h gains the const qualifier to match. This is strictly stronger than __ro_after_init: the object lives in .rodata, i.e. read-only for the kernel's entire lifetime rather than only after mark_rodata_ro(), and any future write attempt fails at compile time. Fixes: 7d1832b64006 ("Haoc: Add support for x86 Sensitive Instruction Protection(IEE_SIP)") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/x86/include/asm/haoc/iee-si.h | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/haoc/iee/iee-init.c | 10 ---------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/haoc/iee-si.h b/arch/x86/include/asm/haoc/iee-si.h index 3f60275696b6e..b20966778c6cd 100644 --- a/arch/x86/include/asm/haoc/iee-si.h +++ b/arch/x86/include/asm/haoc/iee-si.h @@ -7,7 +7,7 @@ #define __iee_si_code __section(".iee.si_text") #define __iee_si_data __section(".iee.si_data") -extern unsigned long cr4_pinned_mask; +extern const unsigned long cr4_pinned_mask; extern struct static_key_false cr_pinning; extern unsigned long cr4_pinned_bits; diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index bd02c484b78c7..3d57a3cc0b3b1 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -406,7 +406,7 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c) /* These bits should not change their value after CPU init is finished. */ #ifdef CONFIG_IEE_SIP -unsigned long cr4_pinned_mask = +const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE | X86_CR4_CET; DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 727d53d7de022..ebe34a511d728 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -219,13 +219,6 @@ static bool __init check_haoc_hardware_support(void) cpu_feature_enabled(X86_FEATURE_SMEP); } -#ifdef CONFIG_IEE_SIP -static void __init iee_setup_sip_cr4_pinning(void) -{ - cr4_pinned_mask &= ~(X86_CR4_SMEP | X86_CR4_SMAP); -} -#endif - void __init iee_early_init(void) { if (!cmd_haoc_enabled) @@ -271,9 +264,6 @@ void __init iee_init(void) haoc_enabled = true; #ifdef CONFIG_IEE_PTRP iee_prepare_init_task_token(); -#endif -#ifdef CONFIG_IEE_SIP - iee_setup_sip_cr4_pinning(); #endif pr_info("HAOC initialized"); } From af0a14e26f8865622c63e1dc56129e2ed395b18e Mon Sep 17 00:00:00 2001 From: Liu Zhehui Date: Tue, 18 Aug 2026 00:51:10 +0800 Subject: [PATCH 26/26] HAOC/x86/IEE_SIP: Fix: Exclude the decompressor from runtime hooks Remove IEE_SIP runtime CR3 hooks from freestanding decompressor code and guard shared-header hooks with __DISABLE_EXPORTS. Preserve the separate HAOC4.0 PTP decompressor paths while eliminating unresolvable main-kernel references. Original commit message: x86/boot/compressed: Drop IEE_SIP CR3-write hooks from the decompressor The decompressor is linked separately (eg. ld.lld -pie) from its own object list and never sees arch/x86/kernel/haoc/iee/ objects. The IEE_SIP commit nevertheless templated #ifdef CONFIG_IEE_SIP if (haoc_enabled) iee_write_cr3_early(...); else write_cr3(...); #endif onto three CR3 writes in ident_map_64.c and one in pgtable_64.c, apparently as part of mechanically covering every raw CR3 write in the tree. The references can never be satisfied in the decompressor: haoc_enabled is defined only in iee-init.c (main kernel) and is parsed from the "haoc=" early_param in setup_arch(), long after the decompressor has exited. No IEE state (shadow mappings, IEE stacks, SIP page setup) exists at decompression time at all, so the branch is dead by design -- the original commit even chose the plain "_early" CR3 write for the true arm, which is exactly what the code now does unconditionally. A regular -O2 build links today only by an optimizer accident: both arms lower to the identical "mov %0,%%cr3" asm (the nested iee_rwx_gate() call inside native_write_cr3() is dead under the outer else), so clang folds the branch and drops the dead load of haoc_enabled. Rebuilding the same translation units at -O0 exposes the latent breakage immediately: ld.lld: error: undefined hidden symbol: haoc_enabled >>> referenced by ident_map_64.c >>> ident_map_64.o:(initialize_identity_maps) >>> referenced by ident_map_64.c >>> ident_map_64.o:(set_clr_page_flags) >>> referenced by pgtable_64.c >>> pgtable_64.o:(configure_5level_paging) >>> referenced 3 more times ld.lld: error: undefined hidden symbol: iee_rwx_gate >>> referenced by pgtable_64.c >>> pgtable_64.o:(iee_write_cr3) >>> referenced by ident_map_64.c >>> ident_map_64.o:(iee_write_cr3) Any change that makes the two arms diverge, a different compiler, or a different optimization level re-exposes it. Remove the four decompressor sites, and fence the IEE hunks in the shared headers behind !__DISABLE_EXPORTS so the inline chain (write_cr3() -> native_write_cr3() -> iee_write_cr3()) can never reintroduce the references into freestanding translation units. __DISABLE_EXPORTS is already defined for all of boot/compressed (as well as real-mode and purgatory, both likewise IEE-free), the main kernel never defines it, and asm/ibt.h already uses exactly this idiom to keep kernel-runtime mechanisms out of freestanding builds. desc.h gets the same guard defensively; nothing in the decompressor includes it today. No functional change to the decompressor: with the branch folded away at -O2, the generated code is identical before and after. Verified with a full LLVM=1 build and a haoc=1 QEMU boot. Fixes: 7d1832b64006 ("Haoc: Add support for x86 Sensitive Instruction Protection(IEE_SIP)") Co-authored-by: WangYuli Signed-off-by: Liu Zhehui --- arch/x86/boot/compressed/ident_map_64.c | 23 +---------------------- arch/x86/boot/compressed/pgtable_64.c | 7 ------- arch/x86/include/asm/desc.h | 8 ++++---- arch/x86/include/asm/special_insns.h | 6 +++--- 4 files changed, 8 insertions(+), 36 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index bf3c12a654414..1494e396768af 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -190,14 +190,7 @@ void initialize_identity_maps(void *rmode) sev_prep_identity_maps(top_level_pgt); /* Load the new page-table. */ - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early(top_level_pgt); - else - write_cr3(top_level_pgt); - #else write_cr3(top_level_pgt); - #endif /* * Now that the required page table mappings are established and a @@ -247,16 +240,9 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, ptp_set_pmd_pre_init(pmdp, pmd); #else set_pmd(pmdp, pmd); -#endif + #endif /* Flush TLB to establish the new PMD */ - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early(top_level_pgt); - else - write_cr3(top_level_pgt); - #else write_cr3(top_level_pgt); - #endif return pte + pte_index(__address); } @@ -361,14 +347,7 @@ static int set_clr_page_flags(struct x86_mapping_info *info, snp_set_page_private(__pa(address & PAGE_MASK)); /* Flush TLB after changing encryption attribute */ - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early(top_level_pgt); - else - write_cr3(top_level_pgt); - #else write_cr3(top_level_pgt); - #endif return 0; } diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index ab495a0b16f22..dc21b0d89d667 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -200,14 +200,7 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) * Move the top level page table out of trampoline memory. */ memcpy(pgtable, trampoline_32bit, PAGE_SIZE); - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early((unsigned long)pgtable); - else - native_write_cr3((unsigned long)pgtable); - #else native_write_cr3((unsigned long)pgtable); - #endif /* Restore trampoline memory */ memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index 4a90cf3c78dc7..94a8674872fd8 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -13,7 +13,7 @@ #include #include -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) #include extern bool haoc_enabled; #endif @@ -215,7 +215,7 @@ static inline void native_load_gdt(const struct desc_ptr *dtr) asm volatile("lgdt %0"::"m" (*dtr)); } -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) static __always_inline void iee_load_idt_early(const struct desc_ptr *dtr) { asm volatile("lidt %0"::"m" (*dtr)); @@ -224,7 +224,7 @@ static __always_inline void iee_load_idt_early(const struct desc_ptr *dtr) static __always_inline void native_load_idt(const struct desc_ptr *dtr) { - #ifdef CONFIG_IEE_SIP + #if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) if(haoc_enabled) iee_load_idt((void *)dtr); else @@ -261,7 +261,7 @@ static inline void native_idt_invalidate(void) .size = 0 }; -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) // The native_idt_invalidate() is only called by machine_kexec(). // In the kdump path, IEE_SIP should not be used, so we directly // call iee_load_idt_early(), which is the original version of diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index 1ab08117dea8c..0a3737ba7a7c9 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -9,7 +9,7 @@ #include #include #include -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) #include extern bool haoc_enabled; #endif @@ -53,7 +53,7 @@ static inline unsigned long __native_read_cr3(void) return val; } -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) static inline void iee_write_cr3_early(unsigned long val) { asm volatile("mov %0,%%cr3" : : "r" (val) : "memory"); @@ -62,7 +62,7 @@ static inline void iee_write_cr3_early(unsigned long val) static inline void native_write_cr3(unsigned long val) { - #ifdef CONFIG_IEE_SIP + #if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) if(haoc_enabled) iee_write_cr3(val); else