diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index c2d0cd3be5e38..a41bea47a494a 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -162,9 +162,16 @@ CONFIG_PMIC_OPREGION=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y CONFIG_IEE=y +CONFIG_PTP=y +CONFIG_PTP_RESERVE_ORDER=12 CONFIG_IEE_PTRP=y +CONFIG_IEE_PTRP_W=y CONFIG_IEE_SIP=y CONFIG_CREDP=y +CONFIG_KEYP=y +CONFIG_IEE_SELINUX_P=y +CONFIG_PTP_S=y +CONFIG_VARP=y CONFIG_KPROBES=y CONFIG_SHADOW_CALL_STACK=y CONFIG_MODULES=y diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 58c294a966768..85d92fb420249 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -109,6 +109,10 @@ void __init fixmap_copy(pgd_t *pgdir); extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); +#ifdef CONFIG_PTP +#include +#endif + #include #endif /* !__ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/haoc/haoc-bitmap.h b/arch/arm64/include/asm/haoc/haoc-bitmap.h new file mode 100644 index 0000000000000..75c855d50a833 --- /dev/null +++ b/arch/arm64/include/asm/haoc/haoc-bitmap.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HAOC_BITMAP_H +#define _LINUX_HAOC_BITMAP_H + +#include +#include + +/* HAOC_NORMAL means */ +enum HAOC_BITMAP_TYPE { + IEE_NORMAL = 0, /* Non-IEE pages */ + IEE_DATA, + IEE_TOKEN, + IEE_PGTABLE, + IEE_USER_PGTABLE, /* for PTP_S */ + IEE_KEY, + IEE_SELINUX, + IEE_VARP, + IEE_CRED, +}; + +#include + +#define HAOC_BITMAP_START VMEMMAP_END +#define haoc_bitmap_base ((uint8_t *)HAOC_BITMAP_START - \ + (memstart_addr >> PAGE_SHIFT)) + +#define __pfn_to_haoc_bitmap(pfn) (haoc_bitmap_base + (pfn)) + +#define __va_to_haoc_bitmap(va) ({ \ + uint8_t *__ret; \ + if (__is_lm_address((u64)va)) \ + __ret = __pfn_to_haoc_bitmap(PHYS_PFN(__pa(va))); \ + else \ + __ret = __pfn_to_haoc_bitmap(PHYS_PFN(__pa_symbol(va))); \ + __ret; \ +}) + +extern bool haoc_enabled; +extern bool haoc_bitmap_ready; + +extern int haoc_bitmap_sparse_init(void); +extern void haoc_bitmap_setup(void); +extern void setup_iee_early_data_bitmap(void); + +static inline enum HAOC_BITMAP_TYPE iee_get_bitmap_type(unsigned long va) +{ + return *__va_to_haoc_bitmap(va); +} + +void _iee_set_bitmap_type(unsigned long __unused, + u64 va, enum HAOC_BITMAP_TYPE type, int num_pages); + +static inline void iee_set_bitmap_type(unsigned long va, + int num_pages, enum HAOC_BITMAP_TYPE type) +{ + iee_rw_gate(IEE_OP_SET_BITMAP_TYPE, va, type, num_pages); +} + +static inline void iee_verify_type(unsigned long va, enum HAOC_BITMAP_TYPE type, + const char *name) +{ + if (haoc_enabled) { + uint8_t bit_type = iee_get_bitmap_type(va); + + if (unlikely(bit_type != type)) + panic("IEE detected type: %d, fake %s: va(0x%lx)", bit_type, name, va); + } +} + +static inline unsigned long iee_test_not_normal(void *start, void *end) +{ + unsigned long addr; + + if (!haoc_enabled) + return 0; + addr = (unsigned long)start; + while (addr < (unsigned long)end) { + if (unlikely(iee_get_bitmap_type(addr) == IEE_NORMAL)) + return addr; + addr += PAGE_SIZE; + } + return 0; +} + +#define iee_verify_not_normal(start, end) ({ \ + unsigned long __addr = iee_test_not_normal(start, end); \ + if (__addr) \ + panic("HAOC: (%s)operate on IEE bitmap type normal 0x%lx.", __func__, __addr); \ +}) + +#endif diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h index 61b201914d5ed..5be9084e7a241 100644 --- a/arch/arm64/include/asm/haoc/haoc-def.h +++ b/arch/arm64/include/asm/haoc/haoc-def.h @@ -21,12 +21,16 @@ enum { IEE_OP_INVALIDATE_TOKEN, IEE_OP_VALIDATE_TOKEN, #endif + IEE_OP_SET_BITMAP_TYPE, #ifdef CONFIG_CREDP IEE_OP_COPY_CRED, IEE_OP_COPY_KERNEL_CRED, IEE_OP_INIT_COPIED_CRED, IEE_OP_COMMIT_CRED, IEE_OP_ABORT_CRED, + IEE_OP_FILL_SESSION_KEYRING_CRED, + IEE_OP_OVERRIDE_CRED, + IEE_OP_REVERT_CRED, IEE_OP_SET_CRED_UID, IEE_OP_SET_CRED_GID, IEE_OP_SET_CRED_SUID, @@ -55,6 +59,60 @@ enum { IEE_OP_SET_CRED_SECURITY, IEE_OP_SET_CRED_RCU, IEE_OP_SET_CRED_UCOUNTS, +#endif +#ifdef CONFIG_PTP + IEE_OP_SET_TRAMP_PGD, + IEE_OP_SET_BM_PTE, + IEE_OP_SET_PTE, + IEE_OP_SET_PMD, + IEE_OP_SET_PUD, + IEE_OP_SET_P4D, + IEE_OP_SET_SWAPPER_PGD, + IEE_OP_SET_XCHG, + IEE_OP_SET_PMD_XCHG, + IEE_OP_SET_CMPXCHG, + IEE_OP_SET_PMD_CMPXCHG, + IEE_OP_SET_SENSITIVE_PTE, + IEE_OP_UNSET_SENSITIVE_PTE, +#endif +#ifdef CONFIG_KEYP + IEE_OP_SET_KEY_UNION, + IEE_OP_SET_KEY_STRUCT, + IEE_OP_SET_KEY_PAYLOAD, + IEE_OP_SET_KEY_USAGE, + IEE_OP_SET_KEY_SERIAL, + IEE_OP_SET_KEY_WATCHERS, + IEE_OP_SET_KEY_USERS, + IEE_OP_SET_KEY_SECURITY, + IEE_OP_SET_KEY_EXPIRY, + IEE_OP_SET_KEY_REVOKED_AT, + IEE_OP_SET_KEY_LAST_USED_AT, + IEE_OP_SET_KEY_UID, + IEE_OP_SET_KEY_GID, + IEE_OP_SET_KEY_PERM, + IEE_OP_SET_KEY_QUOTALEN, + IEE_OP_SET_KEY_DATALEN, + IEE_OP_SET_KEY_STATE, + IEE_OP_SET_KEY_MAGIC, + IEE_OP_SET_KEY_FLAGS, + IEE_OP_SET_KEY_INDEX_KEY, + IEE_OP_SET_KEY_HASH, + IEE_OP_SET_KEY_LEN_DESC, + IEE_OP_SET_KEY_TYPE, + IEE_OP_SET_KEY_TAG, + IEE_OP_SET_KEY_DESCRIPTION, + IEE_OP_SET_KEY_RESTRICT_LINK, + IEE_OP_SET_KEY_FLAG_BIT, +#endif +#ifdef CONFIG_IEE_SELINUX_P + IEE_SEL_SET_STATUS_PG, + IEE_SEL_SET_ENFORCING, + IEE_SEL_SET_INITIALIZED, + IEE_SEL_SET_POLICY_CAP, + IEE_SEL_RCU_ASSIGN_POLICY, +#endif +#ifdef CONFIG_VARP + IEE_OP_SET_VARP_MODPROBE_PATH, #endif IEE_FLAG_END }; @@ -65,6 +123,17 @@ enum { #define AT_SUB_AND_TEST 3 #endif +#ifdef CONFIG_KEYP +#define REFCOUNT_INC 1 +#define REFCOUNT_SET 2 +#define REFCOUNT_DEC_AND_TEST 3 +#define REFCOUNT_INC_NOT_ZERO 4 + +#define SET_BIT_OP 1 +#define TEST_AND_CLEAR_BIT 2 +#define TEST_AND_SET_BIT 3 +#endif + /* The entry gate of all IEE APIs. The first parameter must be a valid * IEE function index. */ diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h index a8d5fcabbf95d..64863021973e7 100644 --- a/arch/arm64/include/asm/haoc/haoc.h +++ b/arch/arm64/include/asm/haoc/haoc.h @@ -10,6 +10,7 @@ #ifndef _LINUX_HAOC_H #define _LINUX_HAOC_H +#include #include #include @@ -26,6 +27,8 @@ void _iee_init_token(unsigned long __unused, struct task_struct *tsk); void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); #endif +void _iee_set_bitmap_type(unsigned long __unused, u64 va, + enum HAOC_BITMAP_TYPE type, int num_pages); #ifdef CONFIG_CREDP #include @@ -33,9 +36,16 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); void _iee_copy_cred(unsigned long __unused, struct cred *new); void _iee_copy_kernel_cred(unsigned long iee_offset, const struct cred *old, struct cred *new); void _iee_abort_cred(unsigned long iee_offset, const struct cred *cred); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, + struct cred *new, + const struct cred *old); void _iee_init_copied_cred(unsigned long iee_offset, struct task_struct *new_task, struct cred *new); void _iee_commit_creds(unsigned long iee_offset, const struct cred *new); +void _iee_override_creds(unsigned long iee_offset, const struct cred *new); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old); +void _iee_revert_creds(unsigned long iee_offset, const struct cred *old); void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); @@ -81,4 +91,81 @@ void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_hea void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); #endif + +#ifdef CONFIG_PTP +#include + +void __iee_code _iee_set_static_pgd(int flag, pgd_t *pgdp, pgd_t pgd); +void __iee_code _iee_set_bm_pte(int flag, pte_t *ptep, pte_t pte); +void __iee_code _iee_set_pte(int flag, pte_t *ptep, pte_t pte); +void __iee_code _iee_set_pmd(int flag, pmd_t *pmdp, pmd_t pmd); +void __iee_code _iee_set_pud(int flag, pud_t *pudp, pud_t pud); +void __iee_code _iee_set_p4d(int flag, p4d_t *p4dp, p4d_t p4d); +void __iee_code _iee_set_swapper_pgd(int flag, pgd_t *pgdp, pgd_t pgd); +pteval_t __iee_code _iee_set_xchg_relaxed(int flag, pte_t *ptep, + pteval_t pteval); +pmdval_t __iee_code _iee_set_pmd_xchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t pmdval); +pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, + pteval_t old_pteval, pteval_t new_pteval); +pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t old_pmdval, pmdval_t new_pmdval); +void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, + pte_t *iee_ptep, int order, int use_block_pmd, bool writable); +void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, + pte_t *iee_ptep, int order, int use_block_pmd); +#endif + +#ifdef CONFIG_KEYP +#include +struct watch_list; + +void _iee_set_key_union(unsigned long __unused, struct key *key, struct key_union *key_union); +void _iee_set_key_struct(unsigned long __unused, struct key *key, struct key_struct *key_struct); +void _iee_set_key_payload(unsigned long __unused, struct key *key, union key_payload *key_payload); +unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, int n, int flag); +void _iee_set_key_serial(unsigned long __unused, struct key *key, key_serial_t serial); +void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers); +void _iee_set_key_user(unsigned long __unused, struct key *key, struct key_user *user); +void _iee_set_key_security(unsigned long __unused, struct key *key, void *security); +void _iee_set_key_expiry(unsigned long __unused, struct key *key, time64_t expiry); +void _iee_set_key_revoked_at(unsigned long __unused, struct key *key, time64_t revoked_at); +void _iee_set_key_last_used_at(unsigned long __unused, struct key *key, time64_t last_used_at); +void _iee_set_key_uid(unsigned long __unused, struct key *key, kuid_t uid); +void _iee_set_key_gid(unsigned long __unused, struct key *key, kgid_t gid); +void _iee_set_key_perm(unsigned long __unused, struct key *key, key_perm_t perm); +void _iee_set_key_quotalen(unsigned long __unused, struct key *key, unsigned short quotalen); +void _iee_set_key_datalen(unsigned long __unused, struct key *key, unsigned short datalen); +void _iee_set_key_state(unsigned long __unused, struct key *key, short state); +void _iee_set_key_magic(unsigned long __unused, struct key *key, unsigned int magic); +void _iee_set_key_flags(unsigned long __unused, struct key *key, unsigned long flags); +void _iee_set_key_index_key(unsigned long __unused, struct key *key, + struct keyring_index_key *index_key); +void _iee_set_key_hash(unsigned long __unused, struct key *key, unsigned long hash); +void _iee_set_key_len_desc(unsigned long __unused, struct key *key, unsigned long len_desc); +void _iee_set_key_type(unsigned long __unused, struct key *key, struct key_type *type); +void _iee_set_key_domain_tag(unsigned long __unused, struct key *key, struct key_tag *domain_tag); +void _iee_set_key_description(unsigned long __unused, struct key *key, char *description); +void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, + struct key_restriction *restrict_link); +unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); +#endif + +#ifdef CONFIG_IEE_SELINUX_P +struct page; +struct selinux_policy; + +void _iee_set_selinux_status_pg(unsigned long __unused, struct page *new_page); +void _iee_set_selinux_enforcing(unsigned long __unused, bool value); +void _iee_mark_selinux_initialized(unsigned long __unused); +void _iee_set_sel_policy_cap(unsigned long __unused, unsigned int idx, int cap); +void _iee_sel_rcu_assign_policy(unsigned long __unused, + struct selinux_policy *new_policy, struct selinux_policy *iee_new_policy); +#endif + +#ifdef CONFIG_VARP +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, int maxlen, size_t len, + char *buffer, size_t *lenp); +#endif + #endif diff --git a/arch/arm64/include/asm/haoc/iee-cred.h b/arch/arm64/include/asm/haoc/iee-cred.h index e5297dabafc37..7384515c4512d 100644 --- a/arch/arm64/include/asm/haoc/iee-cred.h +++ b/arch/arm64/include/asm/haoc/iee-cred.h @@ -6,9 +6,65 @@ #include #include #include +#include extern unsigned long long iee_rw_gate(int flag, ...); +static inline void iee_detect_cred_uaf(struct cred *cred, const char *caller) +{ + if (!haoc_enabled || !iee_init_done || !haoc_bitmap_ready) + return; + + if (atomic_long_read(&cred->usage) != 0) + panic("IEE: (%s) Cred usage %ld.", caller, + atomic_long_read(&cred->usage)); +} + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +static inline void iee_verify_cred(void) +{ + struct task_struct *curr_task; + struct task_token *token; + const struct cred *curr_cred; + + if (!haoc_enabled || !iee_init_done || !haoc_bitmap_ready) + return; + + curr_task = current; + if (curr_task == &init_task) + return; + + token = (struct task_token *)iee_get_task_token(curr_task); + curr_cred = current_cred(); + if (!token->valid) + panic("IEE: (%s) Invalid Token.", __func__); + if (token->curr_cred != curr_cred) + panic("IEE: (%s) Task cred corrupted. token cred 0x%llx, task cred 0x%llx", + __func__, (u64)token->curr_cred, (u64)curr_cred); +} + +static inline void iee_verify_update_cred(struct cred *new) +{ + struct task_struct *curr_task; + struct task_token *token; + + if (!haoc_enabled || !iee_init_done || !haoc_bitmap_ready) + return; + + iee_verify_cred(); + curr_task = current; + if (curr_task == &init_task) + return; + + token = (struct task_token *)iee_get_task_token(curr_task); + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); +} +#else +static inline void iee_verify_cred(void) { } +static inline void iee_verify_update_cred(struct cred *new) { } +#endif + static void __maybe_unused iee_copy_cred(struct cred *new) { // if(!haoc_enabled) @@ -36,10 +92,26 @@ static void __maybe_unused iee_abort_creds(struct cred *cred) iee_rw_gate(IEE_OP_ABORT_CRED, cred); } +static void __maybe_unused iee_fill_cred_for_session_keyring(struct cred *new, + const struct cred *old) +{ + iee_rw_gate(IEE_OP_FILL_SESSION_KEYRING_CRED, new, old); +} + static void __maybe_unused iee_commit_creds(const struct cred *new) { iee_rw_gate(IEE_OP_COMMIT_CRED, new); } + +static void __maybe_unused iee_override_creds(const struct cred *new) +{ + iee_rw_gate(IEE_OP_OVERRIDE_CRED, new); +} + +static void __maybe_unused iee_revert_creds(const struct cred *old) +{ + iee_rw_gate(IEE_OP_REVERT_CRED, old); +} static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) { if(!haoc_enabled) @@ -305,4 +377,4 @@ static void __maybe_unused iee_set_cred_security(struct cred *cred, void *securi } #endif -#endif \ No newline at end of file +#endif diff --git a/arch/arm64/include/asm/haoc/iee-fixmap.h b/arch/arm64/include/asm/haoc/iee-fixmap.h new file mode 100644 index 0000000000000..628ff4a5c6875 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-fixmap.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_FIXMAP_H +#define _LINUX_IEE_FIXMAP_H + +#include + +#ifndef clear_fixmap_pre_init +#define clear_fixmap_pre_init(idx) \ + __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR) +#endif + +#define __iee_set_fixmap_offset_pre_init(idx, phys, flags) \ +({ \ + unsigned long ________addr; \ + __iee_set_fixmap_pre_init(idx, phys, flags); \ + ________addr = fix_to_virt(idx) + ((phys) & (PAGE_SIZE - 1)); \ + ________addr; \ +}) + +#define iee_set_fixmap_offset_pre_init(idx, phys) \ + __iee_set_fixmap_offset_pre_init(idx, phys, FIXMAP_PAGE_NORMAL) + +#endif diff --git a/arch/arm64/include/asm/haoc/iee-func.h b/arch/arm64/include/asm/haoc/iee-func.h index 2c57ea4aff81e..4f726c917cfca 100644 --- a/arch/arm64/include/asm/haoc/iee-func.h +++ b/arch/arm64/include/asm/haoc/iee-func.h @@ -10,10 +10,18 @@ #ifndef _LINUX_IEE_ASM_FUNC_H #define _LINUX_IEE_ASM_FUNC_H +#include + extern void set_iee_address(unsigned long addr, unsigned int order, bool valid); +extern void set_iee_address_valid(unsigned long lm_addr, unsigned int order); +extern void set_iee_address_invalid(unsigned long lm_addr, unsigned int order); extern void iee_set_logical_mem(unsigned long addr, unsigned int order, bool prot); extern void put_pages_into_iee(unsigned long addr, int order); -extern void set_iee_page(unsigned long addr, int order); +extern void put_pages_into_iee_rw(unsigned long addr, int order); +extern void remove_pages_from_iee(unsigned long addr, int order); +extern void set_iee_page_type(unsigned long addr, int order, + enum HAOC_BITMAP_TYPE type); +extern void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type); extern void unset_iee_page(unsigned long addr, int order); #endif diff --git a/arch/arm64/include/asm/haoc/iee-init.h b/arch/arm64/include/asm/haoc/iee-init.h index 31467199c11a4..3ac057dd86ed0 100644 --- a/arch/arm64/include/asm/haoc/iee-init.h +++ b/arch/arm64/include/asm/haoc/iee-init.h @@ -13,8 +13,29 @@ #define NO_BLOCK_MAPPINGS BIT(0) #define NO_CONT_MAPPINGS BIT(1) #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ +#define IS_IEE_MAPPINGS BIT(3) + +#define TOP_PAGE_TABLE(level) ((level) == CONFIG_PGTABLE_LEVELS) extern char iee_init_data_begin[]; extern char iee_init_data_end[]; +#ifdef CONFIG_PTP +extern char __iee_ptp_data_start[]; +extern char __iee_ptp_data_end[]; +#endif +#ifdef CONFIG_CREDP +extern char __iee_cred_data_start[]; +extern char __iee_cred_data_end[]; +#endif +#ifdef CONFIG_IEE_SELINUX_P +extern char __iee_selinux_data_start[]; +extern char __iee_selinux_data_end[]; +#endif +#ifdef CONFIG_VARP +extern char __iee_varp_data_start[]; +extern char __iee_varp_data_end[]; +#endif + +extern struct mutex fixmap_lock; #endif diff --git a/arch/arm64/include/asm/haoc/iee-key.h b/arch/arm64/include/asm/haoc/iee-key.h new file mode 100644 index 0000000000000..ef30ed898e3fd --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-key.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_KEY_H +#define _LINUX_IEE_KEY_H + +#include +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_set_key_union(struct key *key, struct key_union *key_union) +{ + if (!haoc_enabled) { + key->graveyard_link.next = (struct list_head *)key_union; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); +} + +static void __maybe_unused iee_set_key_struct(struct key *key, struct key_struct *key_struct) +{ + if (!haoc_enabled) { + key->name_link.prev = (struct list_head *)key_struct; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); +} + +static void __maybe_unused iee_set_key_payload(struct key *key, union key_payload *key_payload) +{ + if (!haoc_enabled) { + key->name_link.next = (struct list_head *)key_payload; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); +} + +extern bool iee_set_key_usage(struct key *key, int n, int flag); + +static void __maybe_unused iee_set_key_serial(struct key *key, key_serial_t serial) +{ + if (!haoc_enabled) { + key->serial = serial; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); +} + +#ifdef CONFIG_KEY_NOTIFICATIONS +static void __maybe_unused iee_set_key_watchers(struct key *key, struct watch_list *watchers) +{ + if (!haoc_enabled) { + key->watchers = watchers; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); +} +#endif + +static void __maybe_unused iee_set_key_user(struct key *key, struct key_user *user) +{ + if (!haoc_enabled) { + key->user = user; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); +} + +static void __maybe_unused iee_set_key_security(struct key *key, void *security) +{ + if (!haoc_enabled) { + key->security = security; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); +} + +static void __maybe_unused iee_set_key_expiry(struct key *key, time64_t expiry) +{ + if (!haoc_enabled) { + key->expiry = expiry; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); +} + +static void __maybe_unused iee_set_key_revoked_at(struct key *key, time64_t revoked_at) +{ + if (!haoc_enabled) { + key->revoked_at = revoked_at; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); +} + +static void __maybe_unused iee_set_key_last_used_at(struct key *key, time64_t last_used_at) +{ + if (!haoc_enabled) { + key->last_used_at = last_used_at; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); +} + +static void __maybe_unused iee_set_key_uid(struct key *key, kuid_t uid) +{ + if (!haoc_enabled) { + key->uid = uid; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); +} + +static void __maybe_unused iee_set_key_gid(struct key *key, kgid_t gid) +{ + if (!haoc_enabled) { + key->gid = gid; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); +} + +static void __maybe_unused iee_set_key_perm(struct key *key, key_perm_t perm) +{ + if (!haoc_enabled) { + key->perm = perm; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); +} + +static void __maybe_unused iee_set_key_quotalen(struct key *key, unsigned short quotalen) +{ + if (!haoc_enabled) { + key->quotalen = quotalen; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); +} + +static void __maybe_unused iee_set_key_datalen(struct key *key, unsigned short datalen) +{ + if (!haoc_enabled) { + key->datalen = datalen; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); +} + +static void __maybe_unused iee_set_key_state(struct key *key, short state) +{ + if (!haoc_enabled) { + smp_store_release(&key->state, state); + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); +} + +#ifdef KEY_DEBUGGING +static void __maybe_unused iee_set_key_magic(struct key *key, unsigned int magic) +{ + if (!haoc_enabled) { + key->magic = magic; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); +} +#endif + +static void __maybe_unused iee_set_key_flags(struct key *key, unsigned long flags) +{ + if (!haoc_enabled) { + key->flags = flags; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); +} + +static void __maybe_unused iee_set_key_index_key(struct key *key, + struct keyring_index_key *index_key) +{ + if (!haoc_enabled) { + key->index_key = *index_key; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); +} + +static void __maybe_unused iee_set_key_hash(struct key *key, unsigned long hash) +{ + if (!haoc_enabled) { + key->hash = hash; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); +} + +static void __maybe_unused iee_set_key_len_desc(struct key *key, unsigned long len_desc) +{ + if (!haoc_enabled) { + key->len_desc = len_desc; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); +} + +static void __maybe_unused iee_set_key_type(struct key *key, struct key_type *type) +{ + if (!haoc_enabled) { + key->type = type; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); +} + +static void __maybe_unused iee_set_key_domain_tag(struct key *key, struct key_tag *domain_tag) +{ + if (!haoc_enabled) { + key->domain_tag = domain_tag; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); +} + +static void __maybe_unused iee_set_key_description(struct key *key, char *description) +{ + if (!haoc_enabled) { + key->description = description; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); +} + +static void __maybe_unused iee_set_key_restrict_link(struct key *key, + struct key_restriction *restrict_link) +{ + if (!haoc_enabled) { + key->restrict_link = restrict_link; + return; + } + + iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); +} + +static bool __maybe_unused iee_set_key_flag_bit(struct key *key, long nr, int flag) +{ + bool ret; + + if (!haoc_enabled) { + switch (flag) { + case SET_BIT_OP: + set_bit(nr, &key->flags); + break; + case TEST_AND_CLEAR_BIT: + return test_and_clear_bit(nr, &key->flags); + case TEST_AND_SET_BIT: + return test_and_set_bit(nr, &key->flags); + } + return false; + } + + ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); + return ret; +} + +#endif diff --git a/arch/arm64/include/asm/haoc/iee-mmu.h b/arch/arm64/include/asm/haoc/iee-mmu.h index bc43314bd3446..33f291ee72f29 100644 --- a/arch/arm64/include/asm/haoc/iee-mmu.h +++ b/arch/arm64/include/asm/haoc/iee-mmu.h @@ -10,8 +10,13 @@ #ifndef _LINUX_IEE_MMU_H #define _LINUX_IEE_MMU_H +#ifdef CONFIG_PTP +#include +#endif + extern phys_addr_t __init early_iee_stack_alloc(int order); -extern phys_addr_t __init early_iee_data_alloc(int shift); +extern phys_addr_t __init early_iee_data_alloc(int order); +extern phys_addr_t __init early_iee_pgtable_alloc(int shift); extern void __iee_create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, diff --git a/arch/arm64/include/asm/haoc/iee-pgtable.h b/arch/arm64/include/asm/haoc/iee-pgtable.h new file mode 100644 index 0000000000000..495874a74b013 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-pgtable.h @@ -0,0 +1,2131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_IEE_PGTABLE_H +#define __ASM_IEE_PGTABLE_H + +#include +#include + +#include +#include +#include +#include +#include + +/* + * VMALLOC range. + * + * VMALLOC_START: beginning of the kernel vmalloc space + * VMALLOC_END: extends to the available space below vmemmap + */ +#define VMALLOC_START (MODULES_END) +#if VA_BITS == VA_BITS_MIN +#define VMALLOC_END (VMEMMAP_START - SZ_8M) +#else +#define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT) +#define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * \ + sizeof(struct page) - SZ_8M) +#endif + +#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef CYDP_CLEAR_YOUNG +typedef unsigned int cydp_t; +#define CYDP_CLEAR_YOUNG (1U << 0) +#define CYDP_CLEAR_DIRTY (1U << 1) +#endif + +#ifdef CONFIG_PTP_S +extern void ptp_user_check_pte_update(pte_t *ptep, pte_t pte); +extern void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd); +extern void ptp_user_check_pud_update(pud_t *pudp, pud_t pud); +extern void ptp_user_check_p4d_update(p4d_t *p4dp, p4d_t p4d); +extern bool ptp_is_user_pgtable(const void *ptp); +#endif + +extern pgd_t tramp_pg_dir[]; +extern pgd_t idmap_pg_dir[]; +extern pgd_t init_pg_dir[]; +extern pgd_t init_pg_end[]; + +extern int __pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty); +extern bool haoc_enabled; +extern bool iee_init_done; + +static inline bool iee_pgtable_ready(void) +{ + return haoc_enabled && iee_init_done; +} + +static inline bool in_swapper_pgdir(void *addr); +static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd); + +static inline bool in_tramp_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)tramp_pg_dir & PAGE_MASK); +} + +static inline bool in_idmap_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)idmap_pg_dir & PAGE_MASK); +} + +static inline void iee_set_pte_pre_init(pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*ptep, pte); + + /* + * Only if the new pte is valid and kernel, otherwise TLB maintenance + * or update_mmu_cache() have the necessary barriers. + */ + dsb(ishst); + isb(); +} + +static inline void iee_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) +{ +#ifdef __PAGETABLE_PMD_FOLDED + if (in_swapper_pgdir(pmdp)) { + iee_set_swapper_pgd_pre_init((pgd_t *)pmdp, __pgd(pmd_val(pmd))); + return; + } +#endif /* __PAGETABLE_PMD_FOLDED */ + + WRITE_ONCE(*pmdp, pmd); + + dsb(ishst); + isb(); +} + +static inline void iee_set_pud_pre_init(pud_t *pudp, pud_t pud) +{ + #ifdef __PAGETABLE_PUD_FOLDED + if (in_swapper_pgdir(pudp)) { + iee_set_swapper_pgd_pre_init((pgd_t *)pudp, __pgd(pud_val(pud))); + return; + } + #endif + + WRITE_ONCE(*pudp, pud); + + dsb(ishst); + isb(); +} + +static inline void __maybe_unused iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) +{ + if (in_swapper_pgdir(p4dp)) { + iee_set_swapper_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + return; + } + + WRITE_ONCE(*p4dp, p4d); + dsb(ishst); + isb(); +} + +static inline void iee_set_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ + if (in_swapper_pgdir(pgdp)) { + iee_set_swapper_pgd_pre_init(pgdp, __pgd(pgd_val(pgd))); + return; + } + + WRITE_ONCE(*pgdp, pgd); + dsb(ishst); + isb(); +} + +#define pte_set_fixmap_pre_init(addr) \ + ((pte_t *)iee_set_fixmap_offset_pre_init(FIX_PTE, addr)) +#define pte_set_fixmap_offset_pre_init(pmd, addr) \ + pte_set_fixmap_pre_init(pte_offset_phys(pmd, addr)) +#define pte_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PTE) + +#define pmd_set_fixmap_pre_init(addr) \ + ((pmd_t *)iee_set_fixmap_offset_pre_init(FIX_PMD, addr)) +#define pmd_set_fixmap_offset_pre_init(pud, addr) \ + pmd_set_fixmap_pre_init(pmd_offset_phys(pud, addr)) +#define pmd_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PMD) + +#define pud_set_fixmap_pre_init(addr) \ + ((pud_t *)iee_set_fixmap_offset_pre_init(FIX_PUD, addr)) +#define pud_set_fixmap_offset_pre_init(p4d, addr)\ + pud_set_fixmap_pre_init(pud_offset_phys(p4d, addr)) +#define pud_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PUD) + +#define pgd_set_fixmap_pre_init(addr) \ + ((pgd_t *)iee_set_fixmap_offset_pre_init(FIX_PGD, addr)) +#define pgd_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PGD) + +static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ + WRITE_ONCE(*pgdp, pgd); + dsb(ishst); + isb(); +} + +static inline pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval) +{ + pteval_t ret; + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, __pte(pteval)); + return xchg_relaxed((pteval_t *)ptep, pteval); + } +#endif + ret = iee_rw_gate(IEE_OP_SET_XCHG, ptep, pteval); + } else { + ret = xchg_relaxed((pteval_t *)ptep, pteval); + } + return (pteval_t)ret; +} + +static inline pmdval_t iee_set_pmd_xchg_relaxed(pmd_t *pmdp, pmdval_t pmdval) +{ + pmdval_t ret; + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, __pmd(pmdval)); + return xchg_relaxed((pmdval_t *)pmdp, pmdval); + } +#endif + ret = iee_rw_gate(IEE_OP_SET_PMD_XCHG, pmdp, pmdval); + } else { + ret = xchg_relaxed((pmdval_t *)pmdp, pmdval); + } + return (pmdval_t)ret; +} + +static inline pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, + pteval_t new_pteval) +{ + pteval_t ret; + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, __pte(new_pteval)); + return cmpxchg_relaxed((pteval_t *)ptep, old_pteval, new_pteval); + } +#endif + ret = iee_rw_gate(IEE_OP_SET_CMPXCHG, ptep, old_pteval, new_pteval); + } else { + ret = cmpxchg_relaxed((pteval_t *)ptep, old_pteval, new_pteval); + } + return ret; +} + +static inline pmdval_t iee_set_pmd_cmpxchg_relaxed(pmd_t *pmdp, pmdval_t old_pmdval, + pmdval_t new_pmdval) +{ + pmdval_t ret; + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, __pmd(new_pmdval)); + return cmpxchg_relaxed((pmdval_t *)pmdp, old_pmdval, new_pmdval); + } +#endif + ret = iee_rw_gate(IEE_OP_SET_PMD_CMPXCHG, pmdp, old_pmdval, new_pmdval); + } else { + ret = cmpxchg_relaxed((pmdval_t *)pmdp, old_pmdval, new_pmdval); + } + return ret; +} + +static inline void iee_set_static_pgd(pgd_t *pgdp, pgd_t pgd) +{ + iee_rw_gate(IEE_OP_SET_TRAMP_PGD, pgdp, pgd); +} + +static inline void iee_set_bm_pte(pte_t *ptep, pte_t pte) +{ + iee_rw_gate(IEE_OP_SET_BM_PTE, ptep, pte); + + /* + * Only if the new pte is valid and kernel, otherwise TLB maintenance + * or update_mmu_cache() have the necessary barriers. + */ + dsb(ishst); + isb(); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE + +/* Set stride and tlb_level in flush_*_tlb_range */ +#define flush_pmd_tlb_range(vma, addr, end) \ + __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2) +#define flush_pud_tlb_range(vma, addr, end) \ + __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +/* + * Outside of a few very special situations (e.g. hibernation), we always + * use broadcast TLB invalidation instructions, therefore a spurious page + * fault on one CPU which has been handled concurrently by another CPU + * does not need to perform additional invalidation. + */ +#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) + +/* + * ZERO_PAGE is a global shared page that is always zero: used + * for zero-mapped memory areas etc.. + */ +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page)) + +#define pte_ERROR(e) \ + pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) + +/* + * Macros to convert between a physical address and its placement in a + * page table entry, taking care of 52-bit addresses. + */ +#ifdef CONFIG_ARM64_PA_BITS_52 +static inline phys_addr_t __pte_to_phys(pte_t pte) +{ + pte_val(pte) &= ~PTE_MAYBE_SHARED; + return (pte_val(pte) & PTE_ADDR_LOW) | + ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT); +} +static inline pteval_t __phys_to_pte_val(phys_addr_t phys) +{ + return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK; +} +#else +#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK) +#define __phys_to_pte_val(phys) (phys) +#endif + +#define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT) +#define pfn_pte(pfn, prot) \ + __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) + +#define pte_none(pte) (!pte_val(pte)) +#define __pte_clear(mm, addr, ptep) \ + __set_pte(ptep, __pte(0)) +#define pte_page(pte) (pfn_to_page(pte_pfn(pte))) + +/* + * The following only work if pte_present(). Undefined behaviour otherwise. + */ +#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE))) +#define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) +#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) +#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) +#define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY)) +#define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) +#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) +#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) +#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP)) +#define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \ + PTE_ATTRINDX(MT_NORMAL_TAGGED)) + +#define pte_cont_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +#define pmd_cont_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +#define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte)) +#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) +#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) + +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) +/* + * Execute-only user mappings do not have the PTE_USER bit set. All valid + * kernel mappings have the PTE_UXN bit set. + */ +#define pte_valid_not_user(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN)) +/* + * Returns true if the pte is valid and has the contiguous bit set. + */ +#define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) +/* + * Could the pte be present in the TLB? We must check mm_tlb_flush_pending + * so that we don't erroneously return false for pages that have been + * remapped as PROT_NONE but are yet to be flushed from the TLB. + * Note that we can't make any assumptions based on the state of the access + * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the + * TLB. + */ +#define pte_accessible(mm, pte) \ + (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) + +/* + * p??_access_permitted() is true for valid user mappings (PTE_USER + * bit set, subject to the write permission check). For execute-only + * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits + * not set) must return false. PROT_NONE mappings do not have the + * PTE_VALID bit set. + */ +#define pte_access_permitted(pte, write) \ + (((pte_val(pte) & (PTE_VALID | PTE_USER)) ==\ + (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte))) +#define pmd_access_permitted(pmd, write) \ + (pte_access_permitted(pmd_pte(pmd), (write))) +#define pud_access_permitted(pud, write) \ + (pte_access_permitted(pud_pte(pud), (write))) + +static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) +{ + pte_val(pte) &= ~pgprot_val(prot); + return pte; +} + +static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) +{ + pte_val(pte) |= pgprot_val(prot); + return pte; +} + +static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot) +{ + pmd_val(pmd) &= ~pgprot_val(prot); + return pmd; +} + +static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) +{ + pmd_val(pmd) |= pgprot_val(prot); + return pmd; +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); + if (pte_sw_dirty(pte)) + pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); + return pte; +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY)); + pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); + + return pte; +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + + if (pte_write(pte)) + pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); + + return pte; +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + /* + * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY + * clear), set the PTE_DIRTY bit. + */ + if (pte_hw_dirty(pte)) + pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + + pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); + pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); + return pte; +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_AF)); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_AF)); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); +} + +static inline pte_t pte_mkcont(pte_t pte) +{ + pte = set_pte_bit(pte, __pgprot(PTE_CONT)); + return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE)); +} + +static inline pte_t pte_mknoncont(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_CONT)); +} + +static inline pte_t pte_mkpresent(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_VALID)); +} + +static inline pmd_t pmd_mkcont(pmd_t pmd) +{ + return __pmd(pmd_val(pmd) | PMD_SECT_CONT); +} + +static inline pte_t pte_mkdevmap(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL)); +} + +static inline void set_pte_nosync(pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*ptep, pte); +} + +static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) +{ + if (iee_pgtable_ready()) + iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); + else + WRITE_ONCE(*ptep, pte); +} + +static inline void __set_pte(pte_t *ptep, pte_t pte) +{ + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, pte); + WRITE_ONCE(*ptep, pte); + } else +#endif + iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); + } else { + WRITE_ONCE(*ptep, pte); + } + + dsb(ishst); + isb(); +} + +static inline pte_t __ptep_get(pte_t *ptep) +{ + return READ_ONCE(*ptep); +} + +extern void __sync_icache_dcache(pte_t pteval); +bool pgattr_change_is_safe(u64 old, u64 new); + +/* + * PTE bits configuration in the presence of hardware Dirty Bit Management + * (PTE_WRITE == PTE_DBM): + * + * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) + * 0 0 | 1 0 0 + * 0 1 | 1 1 0 + * 1 0 | 1 0 1 + * 1 1 | 0 1 x + * + * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via + * the page fault mechanism. Checking the dirty status of a pte becomes: + * + * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY) + */ + +static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, + pte_t pte) +{ + pte_t old_pte; + + if (!IS_ENABLED(CONFIG_DEBUG_VM)) + return; + + old_pte = __ptep_get(ptep); + + if (!pte_valid(old_pte) || !pte_valid(pte)) + return; + if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1) + return; + + /* + * Check for potential race with hardware updates of the pte + * (__ptep_set_access_flags safely changes valid ptes without going + * through an invalid entry). + */ + VM_WARN_ONCE(!pte_young(pte), + "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", + __func__, pte_val(old_pte), pte_val(pte)); + VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), + "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", + __func__, pte_val(old_pte), pte_val(pte)); + VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)), + "%s: unsafe attribute change: 0x%016llx -> 0x%016llx", + __func__, pte_val(old_pte), pte_val(pte)); +} + +static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages) +{ + if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) + __sync_icache_dcache(pte); + + /* + * If the PTE would provide user space access to the tags associated + * with it then ensure that the MTE tags are synchronised. Although + * pte_access_permitted() returns false for exec only mappings, they + * don't expose tags (instruction fetches don't check tags). + */ + if (system_supports_mte() && pte_access_permitted(pte, false) && + !pte_special(pte) && pte_tagged(pte)) + mte_sync_tags(pte, nr_pages); +} + +/* + * Select all bits except the pfn + */ +#define pte_pgprot pte_pgprot +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + +#define pte_advance_pfn pte_advance_pfn +static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) +{ + return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); +} + +static inline void __set_ptes(struct mm_struct *mm, + unsigned long __always_unused addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + page_table_check_ptes_set(mm, ptep, pte, nr); + __sync_cache_and_tags(pte, nr); + + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn(pte, 1); + } +} + +/* + * Huge pte definitions. + */ +#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT)) + +/* + * Hugetlb definitions. + */ +#define HUGE_MAX_HSTATE 4 +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +static inline pte_t pgd_pte(pgd_t pgd) +{ + return __pte(pgd_val(pgd)); +} + +static inline pte_t p4d_pte(p4d_t p4d) +{ + return __pte(p4d_val(p4d)); +} + +static inline pte_t pud_pte(pud_t pud) +{ + return __pte(pud_val(pud)); +} + +static inline pud_t pte_pud(pte_t pte) +{ + return __pud(pte_val(pte)); +} + +static inline pmd_t pud_pmd(pud_t pud) +{ + return __pmd(pud_val(pud)); +} + +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + +static inline pgprot_t mk_pud_sect_prot(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT); +} + +static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT); +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); +} + +static inline int pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & PTE_SWP_EXCLUSIVE; +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); +} + +#ifdef CONFIG_NUMA_BALANCING +/* + * See the comment in include/linux/pgtable.h + */ +static inline int pte_protnone(pte_t pte) +{ + return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE; +} + +static inline int pmd_protnone(pmd_t pmd) +{ + return pte_protnone(pmd_pte(pmd)); +} +#endif + +#define pmd_present_invalid(pmd) (!!(pmd_val(pmd) & PMD_PRESENT_INVALID)) + +static inline int pmd_present(pmd_t pmd) +{ + return pte_present(pmd_pte(pmd)) || pmd_present_invalid(pmd); +} + +/* + * THP definitions. + */ + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pmd_trans_huge(pmd_t pmd) +{ + return pmd_val(pmd) && pmd_present(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) +#define pmd_young(pmd) pte_young(pmd_pte(pmd)) +#define pmd_valid(pmd) pte_valid(pmd_pte(pmd)) +#define pmd_user(pmd) pte_user(pmd_pte(pmd)) +#define pmd_user_exec(pmd) pte_user_exec(pmd_pte(pmd)) +#define pmd_cont(pmd) pte_cont(pmd_pte(pmd)) +#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) +#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) +#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))) +#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) +#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) +#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) + +static inline pmd_t pmd_mkinvalid(pmd_t pmd) +{ + pmd = set_pmd_bit(pmd, __pgprot(PMD_PRESENT_INVALID)); + pmd = clear_pmd_bit(pmd, __pgprot(PMD_SECT_VALID)); + + return pmd; +} + +#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) + +#define pmd_write(pmd) pte_write(pmd_pte(pmd)) + +#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd)) +#endif +static inline pmd_t pmd_mkdevmap(pmd_t pmd) +{ + return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP))); +} + +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP +#define pmd_special(pte) (!!((pmd_val(pte) & PTE_SPECIAL))) +static inline pmd_t pmd_mkspecial(pmd_t pmd) +{ + return set_pmd_bit(pmd, __pgprot(PTE_SPECIAL)); +} +#endif + +#define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd)) +#define __phys_to_pmd_val(phys) __phys_to_pte_val(phys) +#define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT) +#define pfn_pmd(pfn, prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT)\ + | pgprot_val(prot)) +#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) + +#define pud_young(pud) pte_young(pud_pte(pud)) +#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) +#define pud_write(pud) pte_write(pud_pte(pud)) + +#define pud_mkhuge(pud) (__pud(pud_val(pud) & ~PUD_TABLE_BIT)) + +#define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud)) +#define __phys_to_pud_val(phys) __phys_to_pte_val(phys) +#define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT) +#define pfn_pud(pfn, prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT)\ + | pgprot_val(prot)) + +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t pmd) +{ + unsigned long pfn = pmd_pfn(pmd); + + return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd)); +} + +#define pud_pgprot pud_pgprot +static inline pgprot_t pud_pgprot(pud_t pud) +{ + unsigned long pfn = pud_pfn(pud); + + return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud)); +} + +static inline void __set_pte_at(struct mm_struct *mm, + unsigned long __always_unused addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + __sync_cache_and_tags(pte, nr); + __check_safe_pte_update(mm, ptep, pte); + __set_pte(ptep, pte); +} + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + page_table_check_pmd_set(mm, pmdp, pmd); + return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd), + PMD_SIZE >> PAGE_SHIFT); +} + +static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t pud) +{ + page_table_check_pud_set(mm, pudp, pud); + return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud), + PUD_SIZE >> PAGE_SHIFT); +} + +#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d)) +#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys) + +#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd)) +#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys) + +#define __pgprot_modify(prot, mask, bits) \ + __pgprot((pgprot_val(prot) & ~(mask)) | (bits)) + +#define pgprot_nx(prot) \ + __pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN) + +#ifdef PROT_NS_SHARED +#define pgprot_decrypted(prot) \ + __pgprot_modify(prot, PROT_NS_SHARED, PROT_NS_SHARED) +#define pgprot_encrypted(prot) \ + __pgprot_modify(prot, PROT_NS_SHARED, 0) +#endif + +/* + * Mark the prot value as uncacheable and unbufferable. + */ +#define pgprot_noncached(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN) +#define pgprot_writecombine(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) +#define pgprot_device(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN) +#define pgprot_tagged(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED)) +#define pgprot_mhp pgprot_tagged +/* + * DMA allocations for non-coherent devices use what the Arm architecture calls + * "Normal non-cacheable" memory, which permits speculation, unaligned accesses + * and merging of writes. This is different from "Device-nGnR[nE]" memory which + * is intended for MMIO and thus forbids speculation, preserves access size, + * requires strict alignment and can also force write responses to come from the + * endpoint. + */ +#define pgprot_dmacoherent(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, \ + PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) + +#define __HAVE_PHYS_MEM_ACCESS_PROT +struct file; +extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t vma_prot); + +#define pmd_none(pmd) (!pmd_val(pmd)) + +#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ + PMD_TYPE_TABLE) +#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ + PMD_TYPE_SECT) +#define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd)) +#define pmd_bad(pmd) (!pmd_table(pmd)) + +#define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE) +#define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE) + +#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3 +static inline bool pud_sect(pud_t pud) { return false; } +static inline bool pud_table(pud_t pud) { return true; } +#else +#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ + PUD_TYPE_SECT) +#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ + PUD_TYPE_TABLE) +#endif + +extern pgd_t init_pg_dir[PTRS_PER_PGD]; +extern pgd_t init_pg_end[]; +extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; +extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; +extern pgd_t tramp_pg_dir[PTRS_PER_PGD]; +extern pgd_t reserved_pg_dir[PTRS_PER_PGD]; + +extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); + +static inline bool in_swapper_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)swapper_pg_dir & PAGE_MASK); +} + +static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) +{ +#ifdef __PAGETABLE_PMD_FOLDED + if (in_swapper_pgdir(pmdp)) { + set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); + return; + } +#endif /* __PAGETABLE_PMD_FOLDED */ + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, pmd); + WRITE_ONCE(*pmdp, pmd); + } else +#endif + iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); + } else { + WRITE_ONCE(*pmdp, pmd); + } + + if (pmd_valid(pmd)) { + dsb(ishst); + isb(); + } +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(0)); +} + +static inline phys_addr_t pmd_page_paddr(pmd_t pmd) +{ + return __pmd_to_phys(pmd); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)__va(pmd_page_paddr(pmd)); +} + +/* Find an entry in the third-level page table. */ +#define pte_offset_phys(dir, addr) (pmd_page_paddr(READ_ONCE(*(dir))) \ + + pte_index(addr) * sizeof(pte_t)) + +#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) +#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) +#define pte_clear_fixmap() clear_fixmap(FIX_PTE) + +#define pmd_page(pmd) phys_to_page(__pmd_to_phys(pmd)) + +/* use ONLY for statically allocated translation tables */ +#define pte_offset_kimg(dir, addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr)))) + +/* + * Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + */ +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) + +#if CONFIG_PGTABLE_LEVELS > 2 + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) + +#define pud_none(pud) (!pud_val(pud)) +#define pud_bad(pud) (!pud_table(pud)) +#define pud_present(pud) pte_present(pud_pte(pud)) +#define pud_leaf(pud) (pud_present(pud) && !pud_table(pud)) +#define pud_valid(pud) pte_valid(pud_pte(pud)) +#define pud_user(pud) pte_user(pud_pte(pud)) +#define pud_user_exec(pud) pte_user_exec(pud_pte(pud)) + +static inline void set_pud(pud_t *pudp, pud_t pud) +{ +#ifdef __PAGETABLE_PUD_FOLDED + if (in_swapper_pgdir(pudp)) { + set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); + return; + } +#endif /* __PAGETABLE_PUD_FOLDED */ + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pudp)) { + ptp_user_check_pud_update(pudp, pud); + WRITE_ONCE(*pudp, pud); + } else +#endif + iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); + } else { + WRITE_ONCE(*pudp, pud); + } + + if (pud_valid(pud)) { + dsb(ishst); + isb(); + } +} + +static inline void pud_clear(pud_t *pudp) +{ + set_pud(pudp, __pud(0)); +} + +static inline phys_addr_t pud_page_paddr(pud_t pud) +{ + return __pud_to_phys(pud); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)__va(pud_page_paddr(pud)); +} + +/* Find an entry in the second-level page table. */ +#define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) \ + + pmd_index(addr) * sizeof(pmd_t)) + +#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) +#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) +#define pmd_clear_fixmap() clear_fixmap(FIX_PMD) + +#define pud_page(pud) phys_to_page(__pud_to_phys(pud)) + +/* use ONLY for statically allocated translation tables */ +#define pmd_offset_kimg(dir, addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr)))) + +#else + +#define pud_page_paddr(pud) ({ BUILD_BUG(); 0; }) +#define pud_user_exec(pud) pud_user(pud) /* Always 0 with folding */ + +/* Match pmd_offset folding in */ +#define pmd_set_fixmap(addr) NULL +#define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp) +#define pmd_clear_fixmap() + +#define pmd_offset_kimg(dir, addr) ((pmd_t *)dir) + +#endif /* CONFIG_PGTABLE_LEVELS > 2 */ + +#if CONFIG_PGTABLE_LEVELS > 3 + +static __always_inline bool pgtable_l4_enabled(void) +{ +#if defined(CONFIG_ARM64_LPA2) && defined(ARM64_HAS_VA52) + if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) + return true; + if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) + return vabits_actual == VA_BITS; + return alternative_has_cap_unlikely(ARM64_HAS_VA52); +#else + return true; +#endif +} + +static inline bool mm_pud_folded(const struct mm_struct *mm) +{ + return !pgtable_l4_enabled(); +} +#define mm_pud_folded mm_pud_folded + +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e)) + +#define p4d_none(p4d) (pgtable_l4_enabled() && !p4d_val(p4d)) +#define p4d_bad(p4d) (pgtable_l4_enabled() && !(p4d_val(p4d) & 2)) +#define p4d_present(p4d) (!p4d_none(p4d)) +extern bool check_addr_in_iee_valid(unsigned long addr); +static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) +{ + if (in_swapper_pgdir(p4dp)) { + set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + return; + } + + if (in_tramp_pgdir(p4dp)) { + if (iee_pgtable_ready()) + iee_set_static_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + else { + WRITE_ONCE(*(pgd_t *)p4dp, __pgd(p4d_val(p4d))); + dsb(ishst); + isb(); + } + return; + } + + if (in_idmap_pgdir(p4dp)) { + if (iee_pgtable_ready()) + iee_set_static_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + else { + WRITE_ONCE(*(pgd_t *)p4dp, __pgd(p4d_val(p4d))); + dsb(ishst); + isb(); + } + return; + } + + if (iee_pgtable_ready()) { +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(p4dp)) { + ptp_user_check_p4d_update(p4dp, p4d); + WRITE_ONCE(*p4dp, p4d); + } else +#endif + iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); + } else { + WRITE_ONCE(*p4dp, p4d); + } + + dsb(ishst); + isb(); +} + +static inline void p4d_clear(p4d_t *p4dp) +{ + if (pgtable_l4_enabled()) + set_p4d(p4dp, __p4d(0)); +} + +static inline phys_addr_t p4d_page_paddr(p4d_t p4d) +{ + return __p4d_to_phys(p4d); +} + +#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) + +static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) +{ + return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr); +} + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + return (pud_t *)__va(p4d_page_paddr(p4d)); +} + +/* Find an entry in the first-level page table. */ +static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) +{ + BUG_ON(!pgtable_l4_enabled()); + + return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); +} + +static inline +pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr) +{ + if (!pgtable_l4_enabled()) + return p4d_to_folded_pud(p4dp, addr); + return (pud_t *)__va(p4d_page_paddr(p4d)) + pud_index(addr); +} +#define pud_offset_lockless pud_offset_lockless + +static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long addr) +{ + return pud_offset_lockless(p4dp, READ_ONCE(*p4dp), addr); +} +#define pud_offset pud_offset + +static inline pud_t *pud_set_fixmap(unsigned long addr) +{ + if (!pgtable_l4_enabled()) + return NULL; + return (pud_t *)set_fixmap_offset(FIX_PUD, addr); +} + +static inline pud_t *pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr) +{ + if (!pgtable_l4_enabled()) + return p4d_to_folded_pud(p4dp, addr); + return pud_set_fixmap(pud_offset_phys(p4dp, addr)); +} + +static inline void pud_clear_fixmap(void) +{ + if (pgtable_l4_enabled()) + clear_fixmap(FIX_PUD); +} + +#define p4d_page(p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d))) + +/* use ONLY for statically allocated translation tables */ +static inline pud_t *pud_offset_kimg(p4d_t *p4dp, u64 addr) +{ + if (!pgtable_l4_enabled()) + return p4d_to_folded_pud(p4dp, addr); + return (pud_t *)__phys_to_kimg(pud_offset_phys(p4dp, addr)); +} + +#else + +static inline bool pgtable_l4_enabled(void) { return false; } + +#define p4d_page_paddr(p4d) ({ BUILD_BUG(); 0; }) +#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0; }) + +/* Match pud_offset folding in */ +#define pud_set_fixmap(addr) NULL +#define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) +#define pud_clear_fixmap() + +#define pud_offset_kimg(dir, addr) ((pud_t *)dir) + +#endif /* CONFIG_PGTABLE_LEVELS > 3 */ + +#if CONFIG_PGTABLE_LEVELS > 4 + +static __always_inline bool pgtable_l5_enabled(void) +{ +#if defined(ARM64_HAS_VA52) + if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) + return vabits_actual == VA_BITS; + return alternative_has_cap_unlikely(ARM64_HAS_VA52); +#else + return true; +#endif +} + +static inline bool mm_p4d_folded(const struct mm_struct *mm) +{ + return !pgtable_l5_enabled(); +} +#define mm_p4d_folded mm_p4d_folded + +#define p4d_ERROR(e) \ + pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e)) + +#define pgd_none(pgd) (pgtable_l5_enabled() && !pgd_val(pgd)) +#define pgd_bad(pgd) (pgtable_l5_enabled() && !(pgd_val(pgd) & 2)) +#define pgd_present(pgd) (!pgd_none(pgd)) + +static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) +{ + if (in_swapper_pgdir(pgdp)) { + set_swapper_pgd(pgdp, __pgd(pgd_val(pgd))); + return; + } + + WRITE_ONCE(*pgdp, pgd); + dsb(ishst); + isb(); +} + +static inline void pgd_clear(pgd_t *pgdp) +{ + if (pgtable_l5_enabled()) + set_pgd(pgdp, __pgd(0)); +} + +static inline phys_addr_t pgd_page_paddr(pgd_t pgd) +{ + return __pgd_to_phys(pgd); +} + +#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) + +static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr) +{ + return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr); +} + +static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr) +{ + BUG_ON(!pgtable_l5_enabled()); + + return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t); +} + +static inline +p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long addr) +{ + if (!pgtable_l5_enabled()) + return pgd_to_folded_p4d(pgdp, addr); + return (p4d_t *)__va(pgd_page_paddr(pgd)) + p4d_index(addr); +} +#define p4d_offset_lockless p4d_offset_lockless + +static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr) +{ + return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr); +} + +static inline p4d_t *p4d_set_fixmap(unsigned long addr) +{ + if (!pgtable_l5_enabled()) + return NULL; + return (p4d_t *)set_fixmap_offset(FIX_P4D, addr); +} + +static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr) +{ + if (!pgtable_l5_enabled()) + return pgd_to_folded_p4d(pgdp, addr); + return p4d_set_fixmap(p4d_offset_phys(pgdp, addr)); +} + +static inline void p4d_clear_fixmap(void) +{ + if (pgtable_l5_enabled()) + clear_fixmap(FIX_P4D); +} + +/* use ONLY for statically allocated translation tables */ +static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr) +{ + if (!pgtable_l5_enabled()) + return pgd_to_folded_p4d(pgdp, addr); + return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr)); +} + +#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd))) + +#else + +static inline bool pgtable_l5_enabled(void) { return false; } + +/* Match p4d_offset folding in */ +#define p4d_set_fixmap(addr) NULL +#define p4d_set_fixmap_offset(p4dp, addr) ((p4d_t *)p4dp) +#define p4d_clear_fixmap() + +#define p4d_offset_kimg(dir, addr) ((p4d_t *)dir) + +static inline +p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) +{ + /* + * With runtime folding of the pud, pud_offset_lockless() passes + * the 'pgd_t *' we return here to p4d_to_folded_pud(), which + * will offset the pointer assuming that it points into + * a page-table page. However, the fast GUP path passes us a + * pgd_t allocated on the stack and so we must use the original + * pointer in 'pgdp' to construct the p4d pointer instead of + * using the generic p4d_offset_lockless() implementation. + * + * Note: reusing the original pointer means that we may + * dereference the same (live) page-table entry multiple times. + * This is safe because it is still only loaded once in the + * context of each level and the CPU guarantees same-address + * read-after-read ordering. + */ + return p4d_offset(pgdp, addr); +} +#define p4d_offset_lockless p4d_offset_lockless_folded + +#endif /* CONFIG_PGTABLE_LEVELS > 4 */ + +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) + +#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) +#define pgd_clear_fixmap() clear_fixmap(FIX_PGD) + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + /* + * Normal and Normal-Tagged are two different memory types and indices + * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK. + */ + const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | + PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP | + PTE_ATTRINDX_MASK; + /* preserve the hardware dirty information */ + if (pte_hw_dirty(pte)) + pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + + pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); + /* + * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware + * dirtiness again. + */ + if (pte_sw_dirty(pte)) + pte = pte_mkdirty(pte); + return pte; +} + +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); +} + +extern int __ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +static inline int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + return __pmdp_set_access_flags(vma, address, pmdp, + entry, dirty); +} + +static inline int pud_devmap(pud_t pud) +{ + return 0; +} + +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} +#endif + +#ifdef CONFIG_PAGE_TABLE_CHECK +static inline bool pte_user_accessible_page(pte_t pte) +{ + return pte_present(pte) && (pte_user(pte) || pte_user_exec(pte)); +} + +static inline bool pmd_user_accessible_page(pmd_t pmd) +{ + return pmd_leaf(pmd) && !pmd_present_invalid(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd)); +} + +static inline bool pud_user_accessible_page(pud_t pud) +{ + return pud_leaf(pud) && (pud_user(pud) || pud_user_exec(pud)); +} +#endif + +/* + * Atomic pte/pmd modifications. + */ +static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + pte_t old_pte, pte; + + pte = __ptep_get(ptep); + do { + old_pte = pte; + pte = pte_mkold(pte); + pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, + pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); + + return pte_young(pte); +} + +static inline int __ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + int young = __ptep_test_and_clear_young(vma, address, ptep); + + if (young) { + /* + * We can elide the trailing DSB here since the worst that can + * happen is that a CPU continues to use the young entry in its + * TLB and we mistakenly reclaim the associated page. The + * window for such an event is bounded by the next + * context-switch, which provides a DSB to complete the TLB + * invalidation. + */ + flush_tlb_page_nosync(vma, address); + } + + return young; +} + +static inline int __pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + pmd_t old_pmd, pmd; + + pmd = READ_ONCE(*pmdp); + do { + old_pmd = pmd; + pmd = __pmd(pmd_val(pmd) & ~PTE_AF); + pmd_val(pmd) = iee_set_pmd_cmpxchg_relaxed(pmdp, + pmd_val(old_pmd), pmd_val(pmd)); + } while (pmd_val(pmd) != pmd_val(old_pmd)); + + return pmd_young(pmd); +} + +static inline void __pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp, + pmd_t pmd) +{ + pmd_t old_pmd; + + do { + old_pmd = pmd; + pmd = pmd_wrprotect(pmd); + pmd_val(pmd) = iee_set_pmd_cmpxchg_relaxed(pmdp, pmd_val(old_pmd), pmd_val(pmd)); + } while (pmd_val(pmd) != pmd_val(old_pmd)); +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) +#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + /* Operation applies to PMD table entry only if FEAT_HAFT is enabled */ + VM_WARN_ON(pmd_table(READ_ONCE(*pmdp)) && !system_supports_haft()); + return __pmdp_test_and_clear_young(vma, address, pmdp); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ + +static inline pte_t __ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + pteval_t pteval = iee_set_xchg_relaxed((pte_t *)&pte_val(*ptep), (pteval_t)0); + pte_t pte = __pte(pteval); + + page_table_check_pte_clear(mm, pte); + + return pte; +} + +static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full) +{ + for (;;) { + __ptep_get_and_clear(mm, addr, ptep); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} + +static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned int nr, int full) +{ + pte_t pte, tmp_pte; + + pte = __ptep_get_and_clear(mm, addr, ptep); + while (--nr) { + ptep++; + addr += PAGE_SIZE; + tmp_pte = __ptep_get_and_clear(mm, addr, ptep); + if (pte_dirty(tmp_pte)) + pte = pte_mkdirty(pte); + if (pte_young(tmp_pte)) + pte = pte_mkyoung(pte); + } + return pte; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + pteval_t pteval = iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pteval_t)0); + pmd_t pmd = __pmd(pteval); + + page_table_check_pmd_clear(mm, pmd); + + return pmd; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +static inline void ___ptep_set_wrprotect(struct mm_struct *mm, + unsigned long address, pte_t *ptep, + pte_t pte) +{ + pte_t old_pte; + + do { + old_pte = pte; + + pte = pte_wrprotect(pte); + pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); +} + +/* + * __ptep_set_wrprotect - mark read-only while trasferring potential hardware + * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. + */ +static inline void __ptep_set_wrprotect(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + ___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep)); +} + +static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, + pte_t *ptep, unsigned int nr) +{ + unsigned int i; + + for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + __ptep_set_wrprotect(mm, address, ptep); +} + +static inline void __clear_young_dirty_pte(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, cydp_t flags) +{ + pte_t old_pte; + + do { + old_pte = pte; + + if (flags & CYDP_CLEAR_YOUNG) + pte = pte_mkold(pte); + if (flags & CYDP_CLEAR_DIRTY) + pte = pte_mkclean(pte); + + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), + pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); +} + +static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + pte_t pte; + + for (;;) { + pte = __ptep_get(ptep); + + if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY)) + __set_pte(ptep, pte_mkclean(pte_mkold(pte))); + else + __clear_young_dirty_pte(vma, addr, ptep, pte, flags); + + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_SET_WRPROTECT +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + __pmdp_set_wrprotect(mm, address, pmdp, READ_ONCE(*pmdp)); +} + +#define pmdp_establish pmdp_establish +static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + return __pmd(iee_set_pmd_xchg_relaxed((pmd_t *)&pmd_val(*pmdp), pmd_val(pmd))); +} +#endif + +/* + * Encode and decode a swap entry: + * bits 0-1: present (must be zero) + * bits 2: remember PG_anon_exclusive + * bits 3-7: swap type + * bits 8-57: swap offset + * bit 58: PTE_PROT_NONE (must be zero) + */ +#define __SWP_TYPE_SHIFT 3 +#define __SWP_TYPE_BITS 5 +#define __SWP_OFFSET_BITS 50 +#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) +#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) +#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) + +#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) +#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK) +#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) \ + | ((offset) << __SWP_OFFSET_SHIFT) }) + +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) + +#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) +#define __swp_entry_to_pmd(swp) __pmd((swp).val) +#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ + +/* + * Ensure that there are not more swap files than can be encoded in the kernel + * PTEs. + */ +#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) + +#ifdef CONFIG_ARM64_MTE + +#define __HAVE_ARCH_PREPARE_TO_SWAP +static inline int arch_prepare_to_swap(struct page *page) +{ + if (system_supports_mte()) + return mte_save_tags(page); + return 0; +} + +#define __HAVE_ARCH_SWAP_INVALIDATE +static inline void arch_swap_invalidate_page(int type, pgoff_t offset) +{ + if (system_supports_mte()) + mte_invalidate_tags(type, offset); +} + +static inline void arch_swap_invalidate_area(int type) +{ + if (system_supports_mte()) + mte_invalidate_tags_area(type); +} + +#define __HAVE_ARCH_SWAP_RESTORE +static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) +{ + if (system_supports_mte()) + mte_restore_tags(entry, &folio->page); +} + +#endif /* CONFIG_ARM64_MTE */ + +/* + * On AArch64, the cache coherency is handled via the __set_ptes() function. + */ +static inline void update_mmu_cache_range(struct vm_fault *vmf, + struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, + unsigned int nr) +{ + /* + * We don't do anything here, so there's a very small chance of + * us retaking a user fault which we just fixed up. The alternative + * is doing a dsb(ishst), but that penalises the fastpath. + */ +} + +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) +#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) + +#ifdef CONFIG_ARM64_PA_BITS_52 +#define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52) +#else +#define phys_to_ttbr(addr) (addr) +#endif + +/* + * On arm64 without hardware Access Flag, copying from user will fail because + * the pte is old and cannot be marked young. So we always end up with zeroed + * page after fork() + CoW for pfn mappings. We don't always have a + * hardware-managed access flag on arm64. + */ +#define arch_has_hw_pte_young cpu_has_hw_af + +#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG +#define arch_has_hw_nonleaf_pmd_young system_supports_haft +#endif + +/* + * Experimentally, it's cheap to set the access flag in hardware and we + * benefit from prefaulting mappings as 'old' to start with. + */ +#define arch_wants_old_prefaulted_pte cpu_has_hw_af + +/* + * Request exec memory is read into pagecache in at least 64K folios. The + * trade-off here is performance improvement due to storing translations more + * effciently in the iTLB vs the potential for read amplification due to reading + * data from disk that won't be used. The latter is independent of base page + * size, so we set a page-size independent block size of 64K. This size can be + * contpte-mapped when 4K base pages are in use (16 pages into 1 iTLB entry), + * and HPA can coalesce it (4 pages into 1 TLB entry) when 16K base pages are in + * use. + */ +#define arch_wants_exec_folio_order() ilog2(SZ_64K >> PAGE_SHIFT) + +static inline bool pud_sect_supported(void) +{ + return PAGE_SIZE == SZ_4K; +} + + +#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION +#define ptep_modify_prot_start ptep_modify_prot_start +extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); + +#define ptep_modify_prot_commit ptep_modify_prot_commit +extern void ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte); + +#define modify_prot_start_ptes modify_prot_start_ptes +extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr); + +#define modify_prot_commit_ptes modify_prot_commit_ptes +extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep, pte_t old_pte, pte_t pte, + unsigned int nr); + +#ifdef CONFIG_ARM64_CONTPTE + +/* + * The contpte APIs are used to transparently manage the contiguous bit in ptes + * where it is possible and makes sense to do so. The PTE_CONT bit is considered + * a private implementation detail of the public ptep API (see below). + */ +extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); +extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); +extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte); +extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep); +extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr); +extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full); +extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned int nr, int full); +extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr); +extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty); +extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags); + +static __always_inline void contpte_try_fold(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pte) +{ + /* + * Only bother trying if both the virtual and physical addresses are + * aligned and correspond to the last entry in a contig range. The core + * code mostly modifies ranges from low to high, so this is the likely + * the last modification in the contig range, so a good time to fold. + * We can't fold special mappings, because there is no associated folio. + */ + + const unsigned long contmask = CONT_PTES - 1; + bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + + if (unlikely(valign)) { + bool palign = (pte_pfn(pte) & contmask) == contmask; + + if (unlikely(palign && + pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) + __contpte_try_fold(mm, addr, ptep, pte); + } +} + +static __always_inline void contpte_try_unfold(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pte) +{ + if (unlikely(pte_valid_cont(pte))) + __contpte_try_unfold(mm, addr, ptep, pte); +} + +#define pte_batch_hint pte_batch_hint +static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) +{ + if (!pte_valid_cont(pte)) + return 1; + + return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); +} + +/* + * The below functions constitute the public API that arm64 presents to the + * core-mm to manipulate PTE entries within their page tables (or at least this + * is the subset of the API that arm64 needs to implement). These public + * versions will automatically and transparently apply the contiguous bit where + * it makes sense to do so. Therefore any users that are contig-aware (e.g. + * hugetlb, kernel mapper) should NOT use these APIs, but instead use the + * private versions, which are prefixed with double underscore. All of these + * APIs except for ptep_get_lockless() are expected to be called with the PTL + * held. Although the contiguous bit is considered private to the + * implementation, it is deliberately allowed to leak through the getters (e.g. + * ptep_get()), back to core code. This is required so that pte_leaf_size() can + * provide an accurate size for perf_get_pgtable_size(). But this leakage means + * its possible a pte will be passed to a setter with the contiguous bit set, so + * we explicitly clear the contiguous bit in those cases to prevent accidentally + * setting it in the pgtable. + */ + +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + pte_t pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(pte))) + return pte; + + return contpte_ptep_get(ptep, pte); +} + +#define ptep_get_lockless ptep_get_lockless +static inline pte_t ptep_get_lockless(pte_t *ptep) +{ + pte_t pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(pte))) + return pte; + + return contpte_ptep_get_lockless(ptep); +} + +static inline void set_pte(pte_t *ptep, pte_t pte) +{ + /* + * We don't have the mm or vaddr so cannot unfold contig entries (since + * it requires tlb maintenance). set_pte() is not used in core code, so + * this should never even be called. Regardless do our best to service + * any call and emit a warning if there is any attempt to set a pte on + * top of an existing contig range. + */ + pte_t orig_pte = __ptep_get(ptep); + + WARN_ON_ONCE(pte_valid_cont(orig_pte)); + __set_pte(ptep, pte_mknoncont(pte)); +} + +#define set_ptes set_ptes +static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + pte = pte_mknoncont(pte); + + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __set_ptes(mm, addr, ptep, pte, 1); + contpte_try_fold(mm, addr, ptep, pte); + } else { + contpte_set_ptes(mm, addr, ptep, pte, nr); + } +} + +static inline void pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __pte_clear(mm, addr, ptep); +} + +#define clear_full_ptes clear_full_ptes +static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full) +{ + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __clear_full_ptes(mm, addr, ptep, nr, full); + } else { + contpte_clear_full_ptes(mm, addr, ptep, nr, full); + } +} + +#define get_and_clear_full_ptes get_and_clear_full_ptes +static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned int nr, int full) +{ + pte_t pte; + + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full); + } else { + pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full); + } + + return pte; +} + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + return __ptep_get_and_clear(mm, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t orig_pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_test_and_clear_young(vma, addr, ptep); + + return contpte_ptep_test_and_clear_young(vma, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +static inline int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t orig_pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_clear_flush_young(vma, addr, ptep); + + return contpte_ptep_clear_flush_young(vma, addr, ptep); +} + +#define wrprotect_ptes wrprotect_ptes +static __always_inline void wrprotect_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) +{ + if (likely(nr == 1)) { + /* + * Optimization: wrprotect_ptes() can only be called for present + * ptes so we only need to check contig bit as condition for + * unfold, and we can remove the contig bit from the pte we read + * to avoid re-reading. This speeds up fork() which is sensitive + * for order-0 folios. Equivalent to contpte_try_unfold(). + */ + pte_t orig_pte = __ptep_get(ptep); + + if (unlikely(pte_cont(orig_pte))) { + __contpte_try_unfold(mm, addr, ptep, orig_pte); + orig_pte = pte_mknoncont(orig_pte); + } + ___ptep_set_wrprotect(mm, addr, ptep, orig_pte); + } else { + contpte_wrprotect_ptes(mm, addr, ptep, nr); + } +} + +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + wrprotect_ptes(mm, addr, ptep, 1); +} + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty) +{ + pte_t orig_pte = __ptep_get(ptep); + + entry = pte_mknoncont(entry); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_set_access_flags(vma, addr, ptep, entry, dirty); + + return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty); +} + +#define clear_young_dirty_ptes clear_young_dirty_ptes +static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + if (likely(nr == 1 && !pte_cont(__ptep_get(ptep)))) + __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); + else + contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags); +} + +#else /* CONFIG_ARM64_CONTPTE */ + +#define ptep_get __ptep_get +#define set_pte __set_pte +#define set_ptes __set_ptes +#define pte_clear __pte_clear +#define clear_full_ptes __clear_full_ptes +#define get_and_clear_full_ptes __get_and_clear_full_ptes +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +#define ptep_get_and_clear __ptep_get_and_clear +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +#define ptep_test_and_clear_young __ptep_test_and_clear_young +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +#define ptep_clear_flush_young __ptep_clear_flush_young +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +#define ptep_set_wrprotect __ptep_set_wrprotect +#define wrprotect_ptes __wrprotect_ptes +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +#define ptep_set_access_flags __ptep_set_access_flags +#define clear_young_dirty_ptes __clear_young_dirty_ptes + +#endif /* CONFIG_ARM64_CONTPTE */ + +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +void vmemmap_update_pmd(unsigned long addr, pmd_t *pmdp, pte_t *ptep); +#define vmemmap_update_pmd vmemmap_update_pmd +void vmemmap_update_pte(unsigned long addr, pte_t *ptep, pte_t pte); +#define vmemmap_update_pte vmemmap_update_pte +#endif + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/include/asm/haoc/iee-ptp-init.h b/arch/arm64/include/asm/haoc/iee-ptp-init.h new file mode 100644 index 0000000000000..bfd554611d2ab --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-ptp-init.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_PTP_INIT_H +#define _LINUX_IEE_PTP_INIT_H + +#include + +#define NR_BM_PTE_TABLES \ + SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PMD_SHIFT) +#define NR_BM_PMD_TABLES \ + SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PUD_SHIFT) + +static_assert(NR_BM_PMD_TABLES == 1); + +#define __BM_TABLE_IDX(addr, shift) \ + (((addr) >> (shift)) - (FIXADDR_TOT_START >> (shift))) + +#define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) + +extern pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __section(".iee.ptp") __aligned(PAGE_SIZE); +extern pmd_t bm_pmd[PTRS_PER_PMD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +extern pud_t bm_pud[PTRS_PER_PUD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; + +extern void __iee_pgd_populate_pre_init(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot); +extern void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot); +extern void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot); +extern void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, + pmdval_t prot); +extern int iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot); +extern int iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot); + +extern void __iee_set_fixmap_pre_init(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags); + +extern void iee_set_pgtable_pre_init(unsigned long *addr, unsigned long content); +extern void set_iee_address_pre_init(unsigned long addr, bool valid); +extern void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags); +extern void __create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags); + +extern void __init efi_memmap_unmap_after_init(void); + +extern int early_ioremap_debug __initdata; +extern int after_paging_init __initdata; +extern void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; +extern unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; +extern unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; + +extern bool pgattr_change_is_safe(u64 old, u64 new); + +#endif diff --git a/arch/arm64/include/asm/haoc/iee-selinux.h b/arch/arm64/include/asm/haoc/iee-selinux.h new file mode 100644 index 0000000000000..d169d23790e45 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-selinux.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_SELINUX_P_H +#define _LINUX_IEE_SELINUX_P_H + +#include +#include +#include "security.h" +#include "ss/services.h" + +extern struct kmem_cache *policy_jar; +extern unsigned long long iee_rw_gate(int flag, ...); + +static inline struct mutex *iee_get_selinux_policy_lock(void) +{ + return (struct mutex *)(selinux_state.policy_mutex.owner.counter); +} + +static inline struct mutex *iee_get_selinux_status_lock(void) +{ + return (struct mutex *)(selinux_state.status_lock.owner.counter); +} + +static inline void iee_set_selinux_status_pg(struct page *new_page) +{ + iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); +} + +static inline void enforcing_set(bool value) +{ + iee_rw_gate(IEE_SEL_SET_ENFORCING, value); +} + +static inline void selinux_mark_initialized(void) +{ + iee_rw_gate(IEE_SEL_SET_INITIALIZED); +} + +static inline void iee_set_sel_policy_cap(unsigned int idx, int cap) +{ + iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); +} + +static inline void iee_sel_rcu_assign_policy(struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); +} + +#endif diff --git a/arch/arm64/include/asm/haoc/iee-token.h b/arch/arm64/include/asm/haoc/iee-token.h index 440fcd6e395de..df1c4f947d294 100644 --- a/arch/arm64/include/asm/haoc/iee-token.h +++ b/arch/arm64/include/asm/haoc/iee-token.h @@ -3,7 +3,10 @@ #define _LINUX_IEE_TOKEN_H #include +#include +#include +#ifdef CONFIG_IEE_PTRP extern struct kmem_cache *task_struct_cachep; extern void __init iee_prepare_init_task_token(void); @@ -13,6 +16,20 @@ extern void iee_set_token_page_invalid(unsigned long token_addr, unsigned long token_page, unsigned int order); extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, unsigned int order); +extern void iee_free_task_token_slab(struct kmem_cache *s, struct slab *slab, + unsigned int order); + +#ifndef IEE_TOKEN_BLOCK_SIZE +#define IEE_TOKEN_BLOCK_SIZE 64 +#endif + +#ifndef IEE_TOKEN_ORDER +#define IEE_TOKEN_ORDER(task_order) \ + order_base_2(((1U << (task_order)) * IEE_TOKEN_BLOCK_SIZE) / PAGE_SIZE) +#endif + +extern struct task_token *iee_get_task_token(struct task_struct *task); +#endif /* CONFIG_IEE_PTRP */ #ifdef CONFIG_IEE struct task_token { @@ -22,27 +39,32 @@ struct task_token { bool valid; void *kernel_stack; /* VA */ #ifdef CONFIG_CREDP - struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *curr_cred; /* The current subjective credentials. */ +#endif +#ifdef CONFIG_IEE_PTRP + seqcount_t seq; #endif }; #endif /* CONFIG_IEE */ +#ifdef CONFIG_IEE_PTRP #ifndef CONFIG_IEE_SIP #include -static inline void iee_verify_token_pgd(struct task_struct *tsk) +static inline void iee_verify_pgd(struct task_struct *tsk) { struct task_token *token; if (tsk == &init_task) return; - token = (struct task_token *)__addr_to_iee(tsk); + token = (struct task_token *)iee_get_task_token(tsk); if (token->pgd != tsk->mm->pgd) panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); } #else -static inline void iee_verify_token_pgd(struct task_struct *tsk) +static inline void iee_verify_pgd(struct task_struct *tsk) { } @@ -67,5 +89,6 @@ static inline void iee_validate_token(struct task_struct *tsk) { iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); } +#endif /* CONFIG_IEE_PTRP */ #endif diff --git a/arch/arm64/include/asm/haoc/iee-varp.h b/arch/arm64/include/asm/haoc/iee-varp.h new file mode 100644 index 0000000000000..9ad9f000c26c6 --- /dev/null +++ b/arch/arm64/include/asm/haoc/iee-varp.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_VARP_H +#define _LINUX_IEE_VARP_H + +#include + +#ifdef CONFIG_VARP +#define __iee_varp_data __section(".iee.varp") +#endif + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_set_varp_modprobe_path(char *data, int maxlen, + size_t len, char *buffer, + size_t *lenp) +{ + iee_rw_gate(IEE_OP_SET_VARP_MODPROBE_PATH, data, maxlen, len, buffer, + lenp); +} +#endif diff --git a/arch/arm64/include/asm/haoc/iee.h b/arch/arm64/include/asm/haoc/iee.h index 7137aa186d29a..be5fde9e1f4e1 100644 --- a/arch/arm64/include/asm/haoc/iee.h +++ b/arch/arm64/include/asm/haoc/iee.h @@ -34,19 +34,27 @@ extern bool haoc_enabled; /* Support conversion from both kernel and linear addresses. */ #define __ptr_to_iee(x) ({ \ typeof(x) __val; \ - if (__is_lm_address((u64)x)) \ + if (!haoc_enabled) \ + __val = x; \ + else if (__is_lm_address((u64)x)) \ __val = ((typeof(x))(__virt_to_iee((u64)x))); \ - else \ + else if ((u64)x < (u64)KERNEL_END && (u64)x >= (u64)KERNEL_START) \ __val = ((typeof(x))(__kimg_to_iee((u64)x))); \ + else \ + __val = x; \ __val; \ }) #define __addr_to_iee(x) ({ \ u64 __val; \ - if (__is_lm_address((u64)x)) \ + if (!haoc_enabled) \ + __val = (u64)x; \ + else if (__is_lm_address((u64)x)) \ __val = __virt_to_iee((u64)x); \ - else \ + else if ((u64)x < (u64)KERNEL_END && (u64)x >= (u64)KERNEL_START) \ __val = __kimg_to_iee((u64)x); \ + else \ + __val = (u64)x; \ __val; \ }) @@ -66,7 +74,7 @@ extern bool haoc_enabled; #define TCR_HPD1 (UL(1) << 42) -void iee_init_mappings(pgd_t *pgdp); +void __init iee_init_mappings(pgd_t *pgdp); void iee_init_post(void); void iee_stack_init(void); void iee_init_tcr(void); diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index f905e7814a78d..7875ca1692042 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -54,11 +54,15 @@ static inline void cpu_set_reserved_ttbr0_nosync(void) { unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); +#ifdef CONFIG_IEE_SIP + iee_rwx_gate(IEE_SI_SET_TTBR0, ttbr); +#else #ifdef CONFIG_IEE if (iee_init_done) ttbr |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); #endif write_sysreg(ttbr, ttbr0_el1); +#endif } static inline void cpu_set_reserved_ttbr0(void) diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 661964e99b9d5..a171c57f635ba 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -14,6 +14,9 @@ #include #define __HAVE_ARCH_PGD_FREE +#if CONFIG_PGTABLE_LEVELS > 3 +#define __HAVE_ARCH_PUD_FREE +#endif #include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -53,6 +56,18 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; __p4d_populate(p4dp, __pa(pudp), p4dval); } + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + if (!pgtable_l4_enabled()) + return; + BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); +#ifdef CONFIG_PTP + pagetable_free(virt_to_ptdesc(pud)); +#else + free_page((unsigned long)pud); +#endif +} #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) { diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 6e92be4a1fa81..129ea4adf4a19 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -5,6 +5,10 @@ #ifndef __ASM_PGTABLE_H #define __ASM_PGTABLE_H +#ifdef CONFIG_PTP +#include +#else + #include #include @@ -1471,4 +1475,6 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma, #endif /* !__ASSEMBLY__ */ +#endif /* !CONFIG_PTP */ + #endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index ce21e6be73116..030081d8ce9c2 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,10 +10,18 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif static inline void __tlb_remove_table(void *_table) { + #ifdef CONFIG_PTP + // set_iee_address_invalid((u64)page_to_virt((struct page *)_table)); + iee_cache_free(&pg_cache, page_to_virt((struct page *)_table)); + #else free_page_and_swap_cache((struct page *)_table); + #endif } #define tlb_flush tlb_flush diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index ac148814ffbf2..c1e42241256cc 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -80,6 +80,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif #include #include #include @@ -1835,7 +1838,12 @@ static int __init __kpti_install_ng_mappings(void *__unused) remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); if (!cpu) { + #ifdef CONFIG_PTP + /* CPU 0 is never hotpluged so we could call init function here. */ + alloc = (u64)__va(early_iee_pgtable_alloc(0)); + #else alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); + #endif kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE); kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd); diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig index 7aa900608650f..509576decb8f8 100644 --- a/arch/arm64/kernel/haoc/Kconfig +++ b/arch/arm64/kernel/haoc/Kconfig @@ -14,14 +14,72 @@ config IEE verified in IEE APIs. Needs hardware support FEAT_HPDS. +config IEE_ALLOW_SPLIT_LM + bool "Allow IEE split linear mapping" + depends on IEE + def_bool n + help + Allow the IEE initialization path to split arm64 linear-map block + mappings when a protected range cannot be described safely at the + current table level. This is mainly useful for bring-up on layouts + where protected code or data crosses an existing block boundary. + +config IEE_DEBUG + bool "Skip IEE stack switch for debug" + depends on IEE + def_bool n + help + Build the arm64 IEE gate with a debug path that skips the IEE stack + switch and clears HPD before reporting bad gate state. This keeps + early gate failures easier to inspect while debugging IEE bring-up + code, and should remain disabled for normal kernels. + config IEE_PTRP bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE help - Provide IEE matadata for each process called task_token to allow - verifing pointers inside task_struct, like struct cred* that determines - the capabilities of a process. - Could be a enhancement of other sub-module of HAOC. + Enable task_token pointer protection for IEE. When the write-only + sub-option IEE_PTRP_W is disabled, both read and write checks are + active. When IEE_PTRP_W is enabled, only write protection is active. + +config IEE_PTRP_W + bool "Pointer Write-Only Protection for IEE to improve performance.(IEE_PTRP_W)" + depends on IEE_PTRP + help + Compared to IEE_PTRP, this option only performs write protection on pointers. + Enabling this option improves performance and reduces the protection surface. + Disable this option when read-side pointer verification is also required. + Use the full IEE_PTRP mode when both read and write checks are needed. + +config IEE_CYCLE_CHECK + bool "Cycle check all process cred struct.(IEE_CYCLE_CHECK)" + depends on IEE_PTRP && !IEE_PTRP_W && CREDP + def_bool y + help + Perform periodic read protection checks on all processes. This mode + scans task credentials while pointer write-only protection is disabled. + It is enabled by default when credential protection can use read checks. + Disable pointer write-only protection to keep this periodic scan active. + +config IEE_CS_CHECK + bool "Context-Switch check cred struct.(CONFIG_IEE_CS_CHECK)" + depends on IEE_PTRP && !IEE_PTRP_W && CREDP + def_bool y + help + Perform read protection checks on processes during context switching. + This mode validates task credentials at context-switch boundaries. + It is enabled by default when credential protection can use read checks. + Disable pointer write-only protection to keep context-switch checks active. + +config IEE_IO_CHECK + bool "IO check cred struct.(CONFIG_IEE_IO_CHECK)" + depends on IEE_PTRP && !IEE_PTRP_W && CREDP + def_bool n + help + Perform read protection checks on the process during I/O operations. + This mode validates task credentials when selected I/O paths run. + It is disabled by default and can be enabled for additional coverage. + Disable pointer write-only protection before enabling this read check. config IEE_SIP bool "Sensitive Instruction Protection for IEE(IEE_SIP)" @@ -31,6 +89,37 @@ config IEE_SIP such as writing system control registers. These instructions would be executated inside IEE. +config IEE_SIP + bool "Sensitive Instruction Protection for IEE(IEE_SIP)" + help + Protects critical instructions that may break the isolation of IEE, + such as writing system control registers. These instructions would + be executated inside IEE. + depends on IEE + +config PTP + bool "Page Table Protection(PTP)" + depends on IEE + help + Protect page tables by routing page-table modification paths + through the IEE gate. This keeps page table updates mediated by + the isolated execution environment instead of allowing direct + writes from normal kernel code. + +config PTP_RESERVE_ORDER + depends on PTP + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + range 9 15 + default 12 + +config PTP_S + bool "Page Table Protection - Separation (PTP_S)" + help + PTP_S (kernel/user page separation). Split PTP write paths so user page + tables can use direct writes while kernel tables stay on ptp gate paths, + with user-side safety checks for alias/integrity. + depends on PTP + config CREDP bool "Struct cred protection(CREDP)" depends on IEE @@ -40,4 +129,48 @@ config CREDP by IEE either. If unsure, say N. +config PTP + bool "Page Table Protection(PTP)" + depends on IEE + help + Protect page tables by routing page-table modification paths + through the IEE gate. This keeps page table updates mediated by + the isolated execution environment instead of allowing direct + writes from normal kernel code. + +config PTP_RESERVE_ORDER + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + depends on PTP + range 9 15 + default 12 + +config KEYP + bool "Struct Key protection(KEYP)" + depends on IEE + help + This option enables protection for the kernel's struct key. All modifications + of keys must be made and verified by IEE APIs to defend memory corruptions. + + If unsure, say N. + +config IEE_SELINUX_P + bool "SELinux Protection(IEE_SELINUX_P)" + help + Simple enhancement of the famous linux security module SELinux by putting + several critical data, like selinux policy, into the IEE region. + + If unsure, say N. + depends on SECURITY_SELINUX + depends on IEE + +config VARP + bool "Variable Protection(VARP)" + depends on IEE + help + Protect variables with IEE. At present, only modprobe_path is + covered. Writes to protected variables are routed through IEE + interfaces instead of direct kernel stores. + Enable this option when modprobe_path must be protected by the + isolated execution environment. + endmenu # HAOC diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile index 62b467471edc0..ce076e84f351d 100644 --- a/arch/arm64/kernel/haoc/Makefile +++ b/arch/arm64/kernel/haoc/Makefile @@ -1,4 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += haoc.o +obj-y += haoc.o haoc-bitmap.o obj-y += iee/ -obj-$(CONFIG_CREDP) += credp/ \ No newline at end of file +obj-$(CONFIG_CREDP) += credp/ +obj-$(CONFIG_PTP) += ptp/ +obj-$(CONFIG_KEYP) += keyp/ +obj-$(CONFIG_VARP) += varp/ + +ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ diff --git a/arch/arm64/kernel/haoc/credp/credp.c b/arch/arm64/kernel/haoc/credp/credp.c index 35b486cceb565..f54c3e05e0cbf 100644 --- a/arch/arm64/kernel/haoc/credp/credp.c +++ b/arch/arm64/kernel/haoc/credp/credp.c @@ -3,11 +3,142 @@ #include #include #include +#include +#include +#include +#ifdef CONFIG_IEE_PTRP +#include +#include +#include +#include +#include +#endif extern struct cred init_cred; +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +void iee_cycle_verify_cred(struct task_struct *current_task) +{ + struct task_struct *task = current_task; + struct task_token *token = (struct task_token *)iee_get_task_token(task); + const struct cred *old_cred; + const struct cred *token_cred; + bool valid; + unsigned int seq; + + do { + seq = read_seqcount_begin(&token->seq); + valid = token->valid; + old_cred = task->cred; + token_cred = token->curr_cred; + } while (read_seqcount_retry(&token->seq, seq)); + + if (!valid) + panic("IEE: check_all_threads (%s) Invalid Token.", __func__); + if (task != &init_task && token_cred != old_cred) + panic("IEE: check_all_threads (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx", + __func__, (u64)token_cred, (u64)old_cred); +} + +static int iee_cycle_verify_cred_in_thread(struct task_struct *current_task) +{ + struct task_struct *task = current_task; + const struct cred *old_cred = task->cred; + struct task_token *token = (struct task_token *)iee_get_task_token(task); + + if (!token->valid) + return 1; + if (task != &init_task && token->curr_cred != old_cred) + return 2; + return 0; +} + +#ifdef CONFIG_IEE_CYCLE_CHECK +static void check_all_threads(void) +{ + struct task_struct *task; + + rcu_read_lock(); + for_each_process(task) { + unsigned int seq; + int ret; + struct task_token *token = iee_get_task_token(task); + + if (!token) + continue; + + do { + seq = read_seqcount_begin(&token->seq); + ret = iee_cycle_verify_cred_in_thread(task); + } while (read_seqcount_retry(&token->seq, seq)); + + if (ret == 1) + panic("IEE: check_all_threads (%s) Invalid Token.", __func__); + if (ret == 2) + panic("IEE: check_all_threads (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx", + __func__, (u64)token->curr_cred, (u64)task->cred); + } + rcu_read_unlock(); +} + +static int checker_thread(void *data) +{ + int check_interval_ms = 500; + + while (!kthread_should_stop()) { + check_all_threads(); + msleep_interruptible(check_interval_ms); + } + pr_info("[IEE] Kernel thread exiting\n"); + return 0; +} + +static int __init thread_checker_init(void) +{ + struct task_struct *checker_task; + + if (!haoc_enabled) + return 0; + + pr_info("IEE: Initializing thread checker\n"); + checker_task = kthread_run(checker_thread, NULL, "thread_credp_cycle"); + if (IS_ERR(checker_task)) { + pr_err("IEE: Failed to create thread checker task: %ld\n", + PTR_ERR(checker_task)); + return PTR_ERR(checker_task); + } + + pr_info("IEE: Thread checker started successfully\n"); + return 0; +} + +late_initcall(thread_checker_init); +#endif +#endif + +static inline void iee_verify_cred_type(const struct cred *cred) +{ + if (!iee_init_done || !haoc_bitmap_ready) + return; + + iee_verify_type((unsigned long)cred, IEE_CRED, "cred"); +} + +static inline void iee_verify_current_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_cred(); +} + +static inline void iee_verify_new_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_update_cred(cred); +} + void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); *((struct rcu_head **)(&(cred->rcu.func))) = rcu; } @@ -15,6 +146,7 @@ void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->security = security; } @@ -22,6 +154,7 @@ void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, struct cred *cred, int flag, int nr) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); switch (flag) { case AT_ADD: { @@ -41,6 +174,7 @@ bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); atomic_long_set(&cred->usage, i); } @@ -48,6 +182,7 @@ void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->non_rcu = non_rcu; } @@ -55,6 +190,8 @@ void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, struct key *session_keyring) { + iee_verify_cred_type(cred); + iee_verify_cred(); cred = __ptr_to_iee(cred); cred->session_keyring = session_keyring; } @@ -62,6 +199,7 @@ void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, struct key *process_keyring) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->process_keyring = process_keyring; } @@ -69,6 +207,7 @@ void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, struct key *thread_keyring) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->thread_keyring = thread_keyring; } @@ -76,6 +215,7 @@ void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, struct key *request_key_auth) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->request_key_auth = request_key_auth; } @@ -83,6 +223,7 @@ void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->jit_keyring = jit_keyring; } @@ -90,6 +231,7 @@ void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, kernel_cap_t cap_inheritable) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_inheritable = cap_inheritable; } @@ -97,6 +239,7 @@ void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, kernel_cap_t cap_permitted) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_permitted = cap_permitted; } @@ -104,6 +247,7 @@ void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, kernel_cap_t cap_effective) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_effective = cap_effective; } @@ -111,6 +255,7 @@ void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_bset = cap_bset; } @@ -118,6 +263,7 @@ void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->cap_ambient = cap_ambient; } @@ -125,6 +271,7 @@ void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, void __iee_code _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, unsigned int securebits) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->securebits = securebits; } @@ -132,6 +279,7 @@ void __iee_code _iee_set_cred_securebits(unsigned long __unused, void __iee_code _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, struct group_info *group_info) { + iee_verify_current_cred_type(cred); cred = __ptr_to_iee(cred); cred->group_info = group_info; } @@ -139,6 +287,7 @@ void __iee_code _iee_set_cred_group_info(unsigned long __unused, void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->ucounts = ucounts; } @@ -146,6 +295,7 @@ void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, struct user_namespace *user_ns) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->user_ns = user_ns; } @@ -153,6 +303,7 @@ void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->user = user; } @@ -160,6 +311,7 @@ void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->fsgid = fsgid; } @@ -167,6 +319,7 @@ void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->fsuid = fsuid; } @@ -174,6 +327,7 @@ void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->egid = egid; } @@ -181,6 +335,7 @@ void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->euid = euid; } @@ -188,6 +343,7 @@ void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->sgid = sgid; } @@ -195,6 +351,7 @@ void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->suid = suid; } @@ -207,14 +364,23 @@ void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - //struct task_struct *task = current; - /* Would verify this field in commit_cred. */ - struct task_token *token = (struct task_token *)__addr_to_iee(current); + + iee_verify_cred(); + iee_verify_cred_type(old); + iee_verify_cred_type(new); if (!uid_eq(current_uid(), init_cred.uid)) panic("IEE: calling prepare_kernel_cred by unprivileged process."); - token->new_cred = new; +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -226,14 +392,26 @@ void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred void _iee_init_copied_cred(unsigned long __unused, struct task_struct *new_task, struct cred *new) { - struct task_token *old_task_token = (struct task_token *)__addr_to_iee(current); - //struct task_token *new_task_token = (struct task_token *)__addr_to_iee(new_task); - - if (old_task_token->new_cred != new) - panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", __func__, - (u64)old_task_token->new_cred, (u64)new); - /* Update token info of new task by current task token. */ - old_task_token->new_cred = NULL; + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *old_task_token = + (struct task_token *)iee_get_task_token(current); + struct task_token *new_task_token = + (struct task_token *)iee_get_task_token(new_task); + + if (!old_task_token->valid || !new_task_token->valid) + panic("IEE: (%s) Invalid Token.", __func__); + if (old_task_token->new_cred != new) + panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", + __func__, (u64)old_task_token->new_cred, (u64)new); + /* Update token info of new task by current task token. */ + old_task_token->new_cred = NULL; + new_task_token->curr_cred = new; + } +#endif new_task->cred = new_task->real_cred = new; } @@ -241,13 +419,27 @@ void _iee_init_copied_cred(unsigned long __unused, void _iee_commit_creds(unsigned long __unused, const struct cred *new) { struct task_struct *task = current; - struct task_token *token = (struct task_token *)__addr_to_iee(task); - if (token->new_cred != new) - panic("IEE: (%s) Invalid cred 0x%llx. token->new_cred 0x%llx", - __func__, (u64)new, (u64)token->new_cred); - /* task->cred shall be updated once. */ - token->new_cred = NULL; + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(task); + + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx. token->new_cred 0x%llx", + __func__, (u64)new, (u64)token->new_cred); + /* task->cred shall be updated once. */ + token->new_cred = NULL; + write_seqcount_begin(&token->seq); + token->curr_cred = new; + rcu_assign_pointer(task->real_cred, new); + rcu_assign_pointer(task->cred, new); + write_seqcount_end(&token->seq); + return; + } +#endif rcu_assign_pointer(task->real_cred, new); rcu_assign_pointer(task->cred, new); @@ -255,20 +447,78 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) void _iee_abort_cred(unsigned long __unused, const struct cred *cred) { - struct task_token *token = (struct task_token *)__addr_to_iee(current); + iee_verify_cred(); + iee_verify_cred_type(cred); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + token->new_cred = NULL; + } +#endif +} + +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old) +{ + iee_verify_cred_type(new); + iee_verify_cred_type(old); + iee_verify_cred(); - token->new_cred = NULL; +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + token->new_cred = new; + } +#endif + + new = __ptr_to_iee(new); + new->uid = old->uid; + new->euid = old->euid; + new->suid = old->suid; + new->fsuid = old->fsuid; + new->gid = old->gid; + new->egid = old->egid; + new->sgid = old->sgid; + new->fsgid = old->fsgid; + new->user = get_uid(old->user); + new->ucounts = old->ucounts; + new->user_ns = get_user_ns(old->user_ns); + new->group_info = get_group_info(old->group_info); + + new->securebits = old->securebits; + new->cap_inheritable = old->cap_inheritable; + new->cap_permitted = old->cap_permitted; + new->cap_effective = old->cap_effective; + new->cap_ambient = old->cap_ambient; + new->cap_bset = old->cap_bset; + + new->jit_keyring = old->jit_keyring; + new->thread_keyring = key_get(old->thread_keyring); + new->process_keyring = key_get(old->process_keyring); } void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - struct task_token *token = (struct task_token *)__addr_to_iee(current); /* Get old cred inside IEE is safer. */ const struct cred *old = current_cred(); - /* Would verify this field in commit_cred. */ - token->new_cred = new; + + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -279,6 +529,7 @@ void _iee_copy_cred(unsigned long __unused, struct cred *new) void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->gid = gid; } @@ -286,6 +537,47 @@ void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, void __iee_code _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid) { + iee_verify_new_cred_type(cred); cred = __ptr_to_iee(cred); cred->uid = uid; } + +void __iee_code _iee_override_creds(unsigned long __unused, const struct cred *new) +{ + iee_verify_cred(); + iee_verify_cred_type(new); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + write_seqcount_begin(&token->seq); + token->curr_cred = new; + rcu_assign_pointer(current->cred, new); + write_seqcount_end(&token->seq); + return; + } +#endif + + rcu_assign_pointer(current->cred, new); +} + +void __iee_code _iee_revert_creds(unsigned long __unused, const struct cred *old) +{ + iee_verify_cred(); + iee_verify_cred_type(old); + +#ifdef CONFIG_IEE_PTRP + if (haoc_enabled) { + struct task_token *token = (struct task_token *)iee_get_task_token(current); + + write_seqcount_begin(&token->seq); + token->curr_cred = old; + rcu_assign_pointer(current->cred, old); + write_seqcount_end(&token->seq); + return; + } +#endif + + rcu_assign_pointer(current->cred, old); +} diff --git a/arch/arm64/kernel/haoc/haoc-bitmap.c b/arch/arm64/kernel/haoc/haoc-bitmap.c new file mode 100644 index 0000000000000..c1eb4db73d44c --- /dev/null +++ b/arch/arm64/kernel/haoc/haoc-bitmap.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +bool haoc_bitmap_ready; + +static void * __init haoc_bitmap_alloc_block_zero(unsigned long size) +{ + #ifdef CONFIG_PTP + void *p = __va(early_iee_pgtable_alloc(0)); + + if (!p) + return NULL; + #else + void *p = memblock_alloc(size, size); + + if (!p) + return NULL; + memset(p, 0, size); + #endif + return p; +} + +/* Set APTable on pgd entry of bitmap addresses to prevent kernel writes. */ +static inline void bitmap_p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) +{ + p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF | PGD_APTABLE_RO; + + p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; + __p4d_populate(p4dp, __pa(pudp), p4dval); +} + +static pte_t * __init haoc_bitmap_pte_populate(pmd_t *pmd, unsigned long addr) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + + if (pte_none(ptep_get(pte))) { + pte_t entry; + void *p = __va(early_iee_data_alloc(0)); + + if (!p) + return NULL; + + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, SET_NG(PAGE_KERNEL)); + #ifdef CONFIG_PTP + write_sysreg((read_sysreg(TCR_EL1) | TCR_HPD1) & ~TCR_A1, tcr_el1); + isb(); + WRITE_ONCE(*__ptr_to_iee(pte), entry); + dsb(ishst); + write_sysreg((read_sysreg(TCR_EL1) & ~TCR_HPD1) | TCR_A1, tcr_el1); + isb(); + #else + set_pte_at(&init_mm, addr, pte, entry); + #endif + } + return pte; +} + +static pmd_t * __init haoc_bitmap_pmd_populate(pud_t *pud, unsigned long addr) +{ + pmd_t *pmd = pmd_offset(pud, addr); + void *p; + + if (pmd_none(*pmd)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); + } + return pmd; +} + +static pud_t * __init haoc_bitmap_pud_populate(p4d_t *p4d, unsigned long addr) +{ + pud_t *pud = pud_offset(p4d, addr); + void *p; + + if (pud_none(*pud)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pmd_init(p); + pud_populate(&init_mm, pud, p); + } + return pud; +} + +static p4d_t * __init haoc_bitmap_p4d_populate(pgd_t *pgd, unsigned long addr) +{ + p4d_t *p4d = p4d_offset(pgd, addr); + void *p; + + if (p4d_none(*p4d)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pud_init(p); + bitmap_p4d_populate(&init_mm, p4d, p); + } + return p4d; +} + +static pgd_t * __init haoc_bitmap_pgd_populate(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + void *p; + + if (pgd_none(*pgd)) { + p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); + } + return pgd; +} + +/* Create mappings if that address is not mapped. */ +static pte_t * __init haoc_bitmap_populate_address(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = haoc_bitmap_pgd_populate(addr); + if (!pgd) + return NULL; + p4d = haoc_bitmap_p4d_populate(pgd, addr); + if (!p4d) + return NULL; + pud = haoc_bitmap_pud_populate(p4d, addr); + if (!pud) + return NULL; + pmd = haoc_bitmap_pmd_populate(pud, addr); + if (!pmd) + return NULL; + pte = haoc_bitmap_pte_populate(pmd, addr); + if (!pte) + return NULL; + + return pte; +} + +/* Map haoc bitmap array after vmemmap region. */ +int __init haoc_bitmap_sparse_init(void) +{ + unsigned long start_pfn, end_pfn; + int i, nid; + /* Iterate through available memory blocks. */ + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + u64 bitmap_start = ALIGN_DOWN(((u64)__pfn_to_haoc_bitmap(start_pfn)), + PAGE_SIZE); + u64 bitmap_end = ALIGN(((u64)__pfn_to_haoc_bitmap(end_pfn)), + PAGE_SIZE); + u64 addr = bitmap_start; + pte_t *pte; + + for (; addr < bitmap_end; addr += PAGE_SIZE) { + pte = haoc_bitmap_populate_address(addr); + if (!pte) + pr_err("HAOC: failed on bitmap init."); + } + } + haoc_bitmap_ready = true; + return 0; +} + +void __init haoc_bitmap_setup(void) +{ + unsigned long __maybe_unused start; + unsigned long __maybe_unused end; + int __maybe_unused num_pages; + + /* Setup bitmap types of global data. */ + #ifdef CONFIG_PTP + start = (unsigned long)idmap_pg_dir; + end = (unsigned long)__iee_ptp_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_PGTABLE); + start = (unsigned long)init_pg_dir; + end = (unsigned long)init_pg_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_PGTABLE); + #endif + #ifdef CONFIG_IEE_SELINUX_P + start = (unsigned long)__iee_selinux_data_start; + end = (unsigned long)__iee_selinux_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_SELINUX); + #endif + #ifdef CONFIG_CREDP + start = (unsigned long)__iee_cred_data_start; + end = (unsigned long)__iee_cred_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_CRED); + #endif + #ifdef CONFIG_VARP + /* Map .iee.varp as RO pages, the variable need to be protected can be added */ + start = (unsigned long)__iee_varp_data_start; + end = (unsigned long)__iee_varp_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_VARP); + #endif + #ifdef CONFIG_VARP + /* Map .iee.varp as RO pages, the variable need to be protected can be added */ + start = (unsigned long)__iee_varp_data_start; + end = (unsigned long)__iee_varp_data_end; + num_pages = (end - start) / PAGE_SIZE; + iee_set_bitmap_type(start, num_pages, IEE_VARP); + #endif + + /* Mark bitmaps of early allocated iee data. */ + setup_iee_early_data_bitmap(); +} + +#pragma GCC push_options +#pragma GCC optimize("O0") +static void __iee_code _iee_bitmap_memset(void *ptr, int data, size_t n) +{ + char *_ptr; + + _ptr = (char *)ptr; + + while (n--) + *_ptr++ = data; +} +#pragma GCC pop_options + +void __iee_code _iee_set_bitmap_type(unsigned long __unused, + u64 va, enum HAOC_BITMAP_TYPE type, int num_pages) +{ + _iee_bitmap_memset(__va_to_haoc_bitmap(va), type, num_pages); +} diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c index 33bf9b57c56a4..5d9a96668509d 100644 --- a/arch/arm64/kernel/haoc/haoc.c +++ b/arch/arm64/kernel/haoc/haoc.c @@ -28,12 +28,16 @@ iee_func iee_funcs[] = { (iee_func)_iee_invalidate_token, (iee_func)_iee_validate_token, #endif + (iee_func)_iee_set_bitmap_type, #ifdef CONFIG_CREDP (iee_func)_iee_copy_cred, (iee_func)_iee_copy_kernel_cred, (iee_func)_iee_init_copied_cred, (iee_func)_iee_commit_creds, (iee_func)_iee_abort_cred, + (iee_func)_iee_fill_cred_for_session_keyring, + (iee_func)_iee_override_creds, + (iee_func)_iee_revert_creds, (iee_func)_iee_set_cred_uid, (iee_func)_iee_set_cred_gid, (iee_func)_iee_set_cred_suid, @@ -62,6 +66,60 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_cred_security, (iee_func)_iee_set_cred_rcu, (iee_func)_iee_set_cred_ucounts, +#endif +#ifdef CONFIG_PTP + (iee_func)_iee_set_static_pgd, + (iee_func)_iee_set_bm_pte, + (iee_func)_iee_set_pte, + (iee_func)_iee_set_pmd, + (iee_func)_iee_set_pud, + (iee_func)_iee_set_p4d, + (iee_func)_iee_set_swapper_pgd, + (iee_func)_iee_set_xchg_relaxed, + (iee_func)_iee_set_pmd_xchg_relaxed, + (iee_func)_iee_set_cmpxchg_relaxed, + (iee_func)_iee_set_pmd_cmpxchg_relaxed, + (iee_func)_iee_set_sensitive_pte, + (iee_func)_iee_unset_sensitive_pte, +#endif +#ifdef CONFIG_KEYP + (iee_func)_iee_set_key_union, + (iee_func)_iee_set_key_struct, + (iee_func)_iee_set_key_payload, + (iee_func)_iee_set_key_usage, + (iee_func)_iee_set_key_serial, + (iee_func)_iee_set_key_watchers, + (iee_func)_iee_set_key_user, + (iee_func)_iee_set_key_security, + (iee_func)_iee_set_key_expiry, + (iee_func)_iee_set_key_revoked_at, + (iee_func)_iee_set_key_last_used_at, + (iee_func)_iee_set_key_uid, + (iee_func)_iee_set_key_gid, + (iee_func)_iee_set_key_perm, + (iee_func)_iee_set_key_quotalen, + (iee_func)_iee_set_key_datalen, + (iee_func)_iee_set_key_state, + (iee_func)_iee_set_key_magic, + (iee_func)_iee_set_key_flags, + (iee_func)_iee_set_key_index_key, + (iee_func)_iee_set_key_hash, + (iee_func)_iee_set_key_len_desc, + (iee_func)_iee_set_key_type, + (iee_func)_iee_set_key_domain_tag, + (iee_func)_iee_set_key_description, + (iee_func)_iee_set_key_restrict_link, + (iee_func)_iee_set_key_flag_bit, +#endif +#ifdef CONFIG_IEE_SELINUX_P + (iee_func)_iee_set_selinux_status_pg, + (iee_func)_iee_set_selinux_enforcing, + (iee_func)_iee_mark_selinux_initialized, + (iee_func)_iee_set_sel_policy_cap, + (iee_func)_iee_sel_rcu_assign_policy, +#endif +#ifdef CONFIG_VARP + (iee_func)_iee_set_varp_modprobe_path, #endif NULL }; diff --git a/arch/arm64/kernel/haoc/iee/iee-func.c b/arch/arm64/kernel/haoc/iee/iee-func.c index cef63afc408be..a8458fbfadc28 100644 --- a/arch/arm64/kernel/haoc/iee/iee-func.c +++ b/arch/arm64/kernel/haoc/iee/iee-func.c @@ -8,6 +8,8 @@ */ #include +#include +#include #include #include #include @@ -49,6 +51,7 @@ static void iee_may_split_pmd(pud_t *pudp, unsigned long addr, unsigned int orde } } + #ifdef CONFIG_IEE_ALLOW_SPLIT_LM /* May split Block Descriptor. */ if (!iee_support_pmd_block(addr, order)) { struct page *page = pmd_page(*pmdp); @@ -86,6 +89,9 @@ static void iee_may_split_pmd(pud_t *pudp, unsigned long addr, unsigned int orde if (pgtable) pte_free_kernel(&init_mm, pgtable); } + #endif + dsb(ishst); + isb(); } /* @@ -218,14 +224,21 @@ void set_iee_address_invalid(unsigned long lm_addr, unsigned int order) * synchronization problems. */ static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, - int use_block_pmd) + int use_block_pmd, bool writable) { +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_SET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, + use_block_pmd, writable); +#else int i; if (use_block_pmd) { pmd_t pmd = __pmd(pte_val(__ptep_get(lm_ptep))); - pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + if (writable) + pmd = __pmd((pmd_val(pmd) & ~PMD_SECT_RDONLY) | PTE_DBM); + else + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); for (i = 0; i < (1 << order); i++) { pte_t pte = __ptep_get(iee_ptep); @@ -238,7 +251,10 @@ static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, for (i = 0; i < (1 << order); i++) { pte_t pte = __ptep_get(lm_ptep); - pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); + if (writable) + pte = __pte((pte_val(pte) & ~PTE_RDONLY) | PTE_DBM); + else + pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); WRITE_ONCE(*lm_ptep, pte); pte = __ptep_get(iee_ptep); pte = __pte(pte_val(pte) | PTE_VALID); @@ -249,10 +265,15 @@ static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, } dsb(ishst); isb(); +#endif } static void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) { +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_UNSET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, + use_block_pmd); +#else int i; if (use_block_pmd) { @@ -282,6 +303,7 @@ static void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, } dsb(ishst); isb(); +#endif } /* Only support address range smaller then one PMD block. */ @@ -333,7 +355,7 @@ void put_pages_into_iee_block(unsigned long addr, int order) pmdp = pmd_offset(pudp, iee_addr); iee_ptep = pte_offset_kernel(pmdp, iee_addr); /* Valid the IEE mappings of these pages to enable IEE access. */ - iee_set_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd); + iee_set_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd, false); flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); isb(); } @@ -366,10 +388,15 @@ void put_pages_into_iee(unsigned long addr, int order) } } +void put_pages_into_iee_rw(unsigned long addr, int order) +{ + set_iee_address_valid(addr, order); +} + /* The reverse operation of put_pages_into_iee(). * Call this function when you are returning pages back to kernel. */ -static void remove_pages_from_iee(unsigned long addr, int order) +void remove_pages_from_iee(unsigned long addr, int order) { pgd_t *pgdir = swapper_pg_dir; pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); @@ -401,15 +428,26 @@ static void remove_pages_from_iee(unsigned long addr, int order) } /* See put_pages_into_iee(). */ -void set_iee_page(unsigned long addr, int order) +void set_iee_page_type(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) +{ + set_iee_page(addr, order, type); +} + +/* See put_pages_into_iee(). */ +void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) { + if (type == IEE_USER_PGTABLE) + put_pages_into_iee_rw(addr, order); + else put_pages_into_iee(addr, order); + iee_set_bitmap_type(addr, 1UL << order, type); } /* See remove_pages_from_iee(). */ void unset_iee_page(unsigned long addr, int order) { remove_pages_from_iee(addr, order); + iee_set_bitmap_type(addr, 1UL << order, IEE_NORMAL); } unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) @@ -428,6 +466,16 @@ unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) return order; } +bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, + unsigned int order) +{ +#ifdef CONFIG_IEE_PTRP + if (s == task_struct_cachep) + iee_free_task_token_slab(s, slab, order); +#endif + return false; +} + void iee_set_min_partial(struct kmem_cache *s) { if (!haoc_enabled) { @@ -470,3 +518,22 @@ asmlinkage void notrace iee_bad_mode(struct pt_regs *regs, int reason, panic("bad mode"); } + +#ifdef CONFIG_IEE_PTRP +struct task_token *iee_get_task_token(struct task_struct *task) +{ + unsigned long slab_addr; + unsigned long task_addr; + unsigned int index; + + if (!__is_lm_address((u64)task)) + return (struct task_token *)__kimg_to_iee(task); + + slab_addr = (unsigned long)page_to_virt(virt_to_head_page(task)); + task_addr = (unsigned long)page_to_virt(virt_to_page(task)); + index = (task_addr - slab_addr) / PAGE_SIZE; + + return (struct task_token *)(__virt_to_iee(slab_addr) + + index * IEE_TOKEN_BLOCK_SIZE); +} +#endif diff --git a/arch/arm64/kernel/haoc/iee/iee-gate.S b/arch/arm64/kernel/haoc/iee/iee-gate.S index 47246ab0ec320..050844b2e87ce 100644 --- a/arch/arm64/kernel/haoc/iee/iee-gate.S +++ b/arch/arm64/kernel/haoc/iee/iee-gate.S @@ -11,7 +11,18 @@ #include #include #include + +.data +.align 3 +.global iee_gate_inst +iee_gate_inst: + .quad iee_gate_start + +.text + SYM_FUNC_START(iee_rw_gate) +iee_gate_start: + b 1f /* Rewritten to NOP when HAOC is enabled. */ /* save daif, close irq */ mrs x13, daif msr daifset, #0x2 @@ -24,6 +35,15 @@ SYM_FUNC_START(iee_rw_gate) /* restore daif */ msr daif, x13 ret +1: + /* Call IEE functions directly when HAOC stays disabled. */ + adrp x12, iee_funcs + add x12, x12, x0, lsl #3 + ldr x12, [x12, #:lo12:iee_funcs] + stp x29, x30, [sp, #-16]! + blr x12 + ldp x29, x30, [sp], #16 + ret SYM_FUNC_END(iee_rw_gate) EXPORT_SYMBOL(iee_rw_gate) diff --git a/arch/arm64/kernel/haoc/iee/iee-init.c b/arch/arm64/kernel/haoc/iee/iee-init.c index 416ca52e3e9a5..d9e4b57faade0 100644 --- a/arch/arm64/kernel/haoc/iee/iee-init.c +++ b/arch/arm64/kernel/haoc/iee/iee-init.c @@ -9,7 +9,10 @@ #include #include +#include #include +#include +#include #include #include #ifdef CONFIG_IEE_PTRP @@ -46,6 +49,22 @@ void iee_setup_asid(void) /* Load IEE ASID into ttbr0 to use it in IEE. */ write_sysreg(ttbr0, ttbr0_el1); isb(); + local_flush_tlb_all(); +} + +extern unsigned int *iee_gate_inst; + +static void iee_setup_rw_gate(void) +{ + unsigned int old_gate_inst = *iee_gate_inst; + unsigned long iee_gate_page = ALIGN_DOWN((unsigned long)&iee_rw_gate, PAGE_SIZE); + unsigned int *iee_gate_inst_lm = (unsigned int *)__va(__pa_symbol(iee_gate_inst)); + + *iee_gate_inst_lm = 0xd503201f; /* NOP */ + icache_inval_pou(iee_gate_page, iee_gate_page + PAGE_SIZE); + + pr_info("HAOC: rewrite iee gate inst 0x%p:0x%x to 0x%x", + (void *)iee_gate_inst, old_gate_inst, *iee_gate_inst); } void __init iee_init_post(void) @@ -53,21 +72,25 @@ void __init iee_init_post(void) if (!haoc_enabled) return; + iee_setup_rw_gate(); iee_setup_asid(); /* Flush tlb to enable IEE. */ flush_tlb_all(); + iee_init_done = true; + + haoc_bitmap_sparse_init(); + haoc_bitmap_setup(); + #ifdef CONFIG_IEE_PTRP iee_prepare_init_task_token(); #endif - iee_init_done = true; #ifdef CONFIG_IEE_SIP extern void iee_si_init(void); iee_si_init(); #endif } - void __init iee_stack_init(void) { if (!haoc_enabled) diff --git a/arch/arm64/kernel/haoc/iee/iee-mmu.c b/arch/arm64/kernel/haoc/iee/iee-mmu.c index c7b227907c3f0..7e47b0e6f217b 100644 --- a/arch/arm64/kernel/haoc/iee/iee-mmu.c +++ b/arch/arm64/kernel/haoc/iee/iee-mmu.c @@ -13,9 +13,13 @@ #include #include #include +#include #include #include #include +#ifdef CONFIG_PTP +#include +#endif #define IEE_EARLY_BLOCK_NR 64 @@ -43,7 +47,14 @@ static struct iee_early_alloc iee_stack = { .curr_block_nr = -1 }; -static DEFINE_MUTEX(fixmap_lock); +#ifdef CONFIG_PTP +static struct iee_early_alloc iee_pgtable = { + .name = "iee_early_pgtable", + .curr_block_nr = -1 +}; +#endif + +DEFINE_MUTEX(fixmap_lock); __aligned(PAGE_SIZE) DECLARE_PER_CPU(u64*[(PAGE_SIZE/8)], iee_cpu_stack_ptr); @@ -51,6 +62,17 @@ __aligned(PAGE_SIZE) DECLARE_PER_CPU(u64*[(PAGE_SIZE/8)], /* reserve 8 pages for iee init stack. */ __aligned(IEE_STACK_SIZE) __initdata u64 iee_init_stack[IEE_STACK_SIZE/8]; +#ifdef CONFIG_PTP +void __init iee_init_tcr(void) +{ + unsigned long ptr = (unsigned long)pte_set_fixmap_pre_init(__pa_symbol(&kernel_tcr)); + *((u64 *)ptr) = read_sysreg(tcr_el1) & IEE_TCR_MASK & ~TCR_HPD1; + pte_clear_fixmap_pre_init(); + ptr = (unsigned long)pte_set_fixmap_pre_init(__pa_symbol(&iee_tcr)); + *((u64 *)ptr) = (kernel_tcr | TCR_HPD1) & ~TCR_A1; + pte_clear_fixmap_pre_init(); +} +#else /* Setup global values used in verifications of TCR_EL1 to protect IEE switch gate. * Use fixmap functions as these globals are put inside IEE text section. */ @@ -68,6 +90,7 @@ void __init iee_init_tcr(void) *((u64 *)ptr) = (kernel_tcr | TCR_HPD1) & ~TCR_A1; clear_fixmap(FIX_PTE); } +#endif static void __init iee_setup_bootcpu_stack(void) { @@ -105,7 +128,11 @@ static phys_addr_t __init iee_mem_pool_early_alloc(struct iee_early_alloc *cache * any level of table. */ for (i = 0; i < (1 << (order)); i++) { +#ifdef CONFIG_PTP + ptr = pte_set_fixmap_pre_init(phys + i * PAGE_SIZE); +#else ptr = pte_set_fixmap(phys + i * PAGE_SIZE); +#endif memset(ptr, 0, PAGE_SIZE); @@ -113,7 +140,11 @@ static phys_addr_t __init iee_mem_pool_early_alloc(struct iee_early_alloc *cache * Implicit barriers also ensure the zeroed page is visible to the page * table walker */ +#ifdef CONFIG_PTP + pte_clear_fixmap_pre_init(); +#else pte_clear_fixmap(); +#endif } cache->begin = phys; @@ -156,11 +187,14 @@ void __init early_iee_data_cache_init(void) /* Calculate IEE stack alloc block size. */ iee_mem_pool_early_alloc(&iee_stack, IEE_DATA_ORDER); /* Calculate IEE data alloc block size. */ - iee_mem_pool_early_alloc(&iee_data, get_iee_alloc_order(1)); + iee_mem_pool_early_alloc(&iee_data, get_iee_alloc_order(0)); +#ifdef CONFIG_PTP + iee_mem_pool_early_alloc(&iee_pgtable, get_iee_alloc_order(1)); +#endif } phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, - int order) + int order, enum HAOC_BITMAP_TYPE type) { phys_addr_t phys; phys_addr_t expand_phys; @@ -178,7 +212,9 @@ phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, /* Put the expanded memory into IEE if late enough. */ block_order = cache->blocks[cache->curr_block_nr].order; - if (iee_init_done) + if (haoc_bitmap_ready) + set_iee_page((unsigned long)__va(expand_phys), block_order, type); + else if (iee_init_done) put_pages_into_iee((unsigned long)__va(expand_phys), block_order); goto redo; } @@ -193,42 +229,48 @@ phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, */ phys_addr_t __init early_iee_stack_alloc(int order) { - return iee_early_alloc(&iee_stack, order); + return iee_early_alloc(&iee_stack, order, IEE_DATA); +} + +phys_addr_t __init early_iee_data_alloc(int order) +{ + return iee_early_alloc(&iee_data, order, IEE_DATA); } -phys_addr_t __init early_iee_data_alloc(int shift) +#ifndef CONFIG_PTP +static void iee_clear_pgtable(phys_addr_t phys) { - return iee_early_alloc(&iee_data, 0); + void *table = pte_set_fixmap(phys); + + clear_page(table); + pte_clear_fixmap(); } +#endif -static phys_addr_t __init early_pgtable_alloc(int shift) +#ifdef CONFIG_PTP +phys_addr_t __init early_iee_pgtable_alloc(int shift) { + return iee_early_alloc(&iee_pgtable, 0, IEE_PGTABLE); +} +#endif + +static phys_addr_t __init iee_early_pgtable_alloc(int shift) +{ +#ifdef CONFIG_PTP + return early_iee_pgtable_alloc(shift); +#else phys_addr_t phys; - void *ptr; phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, MEMBLOCK_ALLOC_NOLEAKTRACE); if (!phys) panic("Failed to allocate page table page\n"); - /* - * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE - * slot will be free, so we can (ab)use the FIX_PTE slot to initialise - * any level of table. - */ - ptr = pte_set_fixmap(phys); - - memset(ptr, 0, PAGE_SIZE); - - /* - * Implicit barriers also ensure the zeroed page is visible to the page - * table walker - */ - pte_clear_fixmap(); - return phys; +#endif } +#ifndef CONFIG_PTP static void iee_init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot) { @@ -264,13 +306,14 @@ static void iee_alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, IEE_CHECK(pmd_sect(pmd)); if (pmd_none(pmd)) { - pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_SECT_AF; + pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF; phys_addr_t pte_phys; if (flags & NO_EXEC_MAPPINGS) pmdval |= PMD_TABLE_PXN; IEE_CHECK(!pgtable_alloc); pte_phys = pgtable_alloc(PAGE_SHIFT); + iee_clear_pgtable(pte_phys); __pmd_populate(pmdp, pte_phys, pmdval); pmd = READ_ONCE(*pmdp); } @@ -342,13 +385,16 @@ static void iee_alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, */ IEE_CHECK(pud_sect(pud)); if (pud_none(pud)) { - pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PMD_SECT_AF; + pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF; phys_addr_t pmd_phys; if (flags & NO_EXEC_MAPPINGS) pudval |= PUD_TABLE_PXN; + if ((flags & IS_IEE_MAPPINGS) && TOP_PAGE_TABLE(3)) + pudval |= PGD_APTABLE_RO; IEE_CHECK(!pgtable_alloc); pmd_phys = pgtable_alloc(PMD_SHIFT); + iee_clear_pgtable(pmd_phys); __pud_populate(pudp, pmd_phys, pudval); pud = READ_ONCE(*pudp); } @@ -381,13 +427,16 @@ static void iee_alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long en p4d_t p4d = READ_ONCE(*p4dp); if (p4d_none(p4d)) { - p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | PMD_SECT_AF; + p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF; phys_addr_t pud_phys; if (flags & NO_EXEC_MAPPINGS) p4dval |= P4D_TABLE_PXN; + if ((flags & IS_IEE_MAPPINGS) && TOP_PAGE_TABLE(4)) + p4dval |= PGD_APTABLE_RO; IEE_CHECK(!pgtable_alloc); pud_phys = pgtable_alloc(PUD_SHIFT); + iee_clear_pgtable(pud_phys); __p4d_populate(p4dp, pud_phys, p4dval); p4d = READ_ONCE(*p4dp); } @@ -409,6 +458,7 @@ static void iee_alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long en pud_clear_fixmap(); } +#endif /* This function is almost the same with __create_pgd_mapping_locked() * but not permitting block descriptors larger than pmd block to simplify @@ -436,65 +486,25 @@ void __iee_create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, do { next = pgd_addr_end(addr, end); - iee_alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, +#ifdef CONFIG_PTP + iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, flags); - phys += next - addr; - } while (pgdp++, addr = next, addr != end); -} - -/* Mark the pgd entry of IEE address ranges with APTable to setup isolation. */ -static void __init __create_pgd_mapping_for_iee_locked(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(int), - int flags) -{ - unsigned long addr, end, next; - pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); - p4d_t *p4dp; - p4d_t p4d; - - /* - * If the virtual and physical address don't have the same offset - * within a page, we cannot map the region as the caller expects. - */ - if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) - return; - - phys &= PAGE_MASK; - addr = virt & PAGE_MASK; - end = PAGE_ALIGN(virt + size); - - do { - next = pgd_addr_end(addr, end); - p4dp = p4d_offset(pgdp, addr); - p4d = READ_ONCE(*p4dp); - if (!p4d_none(p4d) && !(p4d_val(p4d) & PGD_APTABLE_RO)) { - phys += next - addr; - continue; - } +#else iee_alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, flags); - - /* Set APTable RO on pgd entries of IEE mappings to prevent kernel access - * when TCR.HPD1 == 0. - */ - p4d = READ_ONCE(*p4dp); - __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APTABLE_RO | PGD_PXNTABLE | - PGD_UXNTABLE | PUD_TYPE_TABLE)); - +#endif phys += next - addr; } while (pgdp++, addr = next, addr != end); } -static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, +static void __init __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(int), int flags) { mutex_lock(&fixmap_lock); - __create_pgd_mapping_for_iee_locked(pgdir, phys, virt, size, prot, + __iee_create_pgd_mapping_locked(pgdir, phys, virt, size, prot, pgtable_alloc, flags); mutex_unlock(&fixmap_lock); } @@ -503,7 +513,7 @@ static void __init __map_memblock_for_iee(pgd_t *pgdp, phys_addr_t start, phys_addr_t end, pgprot_t prot, int flags) { __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, - prot, early_pgtable_alloc, flags); + prot, iee_early_pgtable_alloc, flags); } /* @@ -536,7 +546,7 @@ void __init iee_init_mappings(pgd_t *pgdp) * Not allowing block or continuous mappings on IEE for faster page * attribution modification. */ - flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; + flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | IS_IEE_MAPPINGS; /* map all the memory banks non-executable but invalid on iee addresses. */ for_each_mem_range(i, &start, &end) { @@ -550,6 +560,40 @@ void __init iee_init_mappings(pgd_t *pgdp) iee_setup_bootcpu_stack(); } +static void setup_iee_data_cache_bitmap(struct iee_early_alloc *cache, + enum HAOC_BITMAP_TYPE type) +{ + int block_nr = cache->curr_block_nr + 1; + + for (int j = 0; j < block_nr; j++) { + iee_set_bitmap_type((unsigned long)__va(cache->blocks[j].start), + 1 << cache->blocks[j].order, type); + } +} + +void __init setup_iee_early_data_bitmap(void) +{ + setup_iee_data_cache_bitmap(&iee_data, IEE_DATA); + setup_iee_data_cache_bitmap(&iee_stack, IEE_DATA); +#ifdef CONFIG_PTP + setup_iee_data_cache_bitmap(&iee_pgtable, IEE_PGTABLE); +#endif +} + +#ifdef CONFIG_PTP +static void setup_iee_early_address(struct iee_early_alloc *cache) +{ + int i, j; + + for (j = 0; j < cache->curr_block_nr + 1; j++) { + for (i = 0; i < (1 << cache->blocks[j].order); i++) { + set_iee_address_pre_init(__phys_to_iee(cache->blocks[j].start + + i * PAGE_SIZE), true); + } + } +} +#endif + static void prot_iee_early_data_cache(struct iee_early_alloc *cache) { int block_nr = cache->curr_block_nr + 1; @@ -563,12 +607,89 @@ static void prot_iee_early_data_cache(struct iee_early_alloc *cache) /* Put early allocated pages into IEE. */ void __init init_early_iee_data(void) { - u64 i; + int i; + if (!haoc_enabled) return; +#ifdef CONFIG_PTP + /* Setup iee mappings of early allocated IEE objects to enable IEE. */ + for (i = 0; ((unsigned long)idmap_pg_dir + i * PAGE_SIZE) < + (unsigned long)iee_init_data_end; i++) { + set_iee_address_pre_init(__phys_to_iee(__pa_symbol((unsigned long)idmap_pg_dir + + i * PAGE_SIZE)), true); + } + for (i = 0; ((unsigned long)init_pg_dir + i * PAGE_SIZE) < + (unsigned long)init_pg_end; i++) { + set_iee_address_pre_init(__phys_to_iee(__pa_symbol((unsigned long)init_pg_dir + + i * PAGE_SIZE)), true); + } + setup_iee_early_address(&iee_pgtable); + setup_iee_early_address(&iee_data); + prot_iee_early_data_cache(&iee_pgtable); +#else for (i = 0; (iee_init_data_begin + i * PAGE_SIZE) < iee_init_data_end; i++) set_iee_address(__phys_to_iee(__pa_symbol(iee_init_data_begin + i * PAGE_SIZE)), 0, true); +#endif + prot_iee_early_data_cache(&iee_data); prot_iee_early_data_cache(&iee_stack); } + +#ifdef CONFIG_PTP +int __pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + pmdval_t old_pmdval, pmdval; + pmd_t pmd = READ_ONCE(*pmdp); + + if (pmd_same(pmd, entry)) + return 0; + + /* only preserve the access flags and write permission */ + pmd_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; + + /* + * Setting the flags must be done atomically to avoid racing with the + * hardware update of the access/dirty state. The PTE_RDONLY bit must + * be set to the most permissive (lowest value) of *ptep and entry + * (calculated as: a & b == ~(~a | ~b)). + */ + pmd_val(entry) ^= PTE_RDONLY; + pmdval = pmd_val(pmd); + do { + old_pmdval = pmdval; + pmdval ^= PTE_RDONLY; + pmdval |= pmd_val(entry); + pmdval ^= PTE_RDONLY; + pmdval = iee_set_pmd_cmpxchg_relaxed(pmdp, old_pmdval, pmdval); + } while (pmdval != old_pmdval); + + /* Invalidate a stale read-only entry */ + if (dirty) + flush_tlb_page(vma, address); + return 1; +} + +void * __ref __ptp_vmemmap_alloc_block(unsigned long size, int node) +{ + int order = get_order(size); + void *p; + + if (!haoc_enabled) { + p = vmemmap_alloc_block(size, node); + if (p) + memset(p, 0, size); + return p; + } + + /* If the main allocator is up use that, fallback to bootmem. */ + if (slab_is_available()) + return iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); + + if (order != 0) + panic("PTP: Unsupport vmemmap alloc."); + return __va(early_iee_pgtable_alloc(0)); +} +#endif diff --git a/arch/arm64/kernel/haoc/iee/iee-si-gate.S b/arch/arm64/kernel/haoc/iee/iee-si-gate.S index 07fe1a0a7c8ec..52b654273949d 100644 --- a/arch/arm64/kernel/haoc/iee/iee-si-gate.S +++ b/arch/arm64/kernel/haoc/iee/iee-si-gate.S @@ -40,6 +40,9 @@ SYM_FUNC_END(iee_rwx_gate) SYM_FUNC_START(iee_rwx_gate_tramp) /* entry gate */ mrs x12, tcr_el1 +#ifdef CONFIG_IEE_DEBUG + tbnz x12, #42, 114f +#endif orr x12, x12, #TCR_HPD1 bic x12, x12, #TCR_A1 msr tcr_el1, x12 @@ -52,10 +55,12 @@ SYM_FUNC_START(iee_rwx_gate_tramp) cmp x9, x12 b.ne 3f +#ifndef CONFIG_IEE_DEBUG /* Switch to iee stack by per cpu ptr. */ ldr_this_cpu x9, iee_cpu_stack_ptr, x10 mov x14, sp mov sp, x9 +#endif 1: stp x13, x14, [sp, #-32]! stp x29, x30, [sp, #16] @@ -78,10 +83,19 @@ SYM_FUNC_START(iee_rwx_gate_tramp) cmp x9, x12 b.ne 3f +#ifndef CONFIG_IEE_DEBUG /* Switch to kernel stack */ mov sp, x14 +#endif 2: ret +#ifdef CONFIG_IEE_DEBUG + /* Close HPDS before panic. */ +114: + bic x12, x12, #TCR_HPD1 + msr tcr_el1, x12 + isb +#endif 3: mov x0, sp mov x1, #BAD_TCR_EL1 diff --git a/arch/arm64/kernel/haoc/iee/iee-si.c b/arch/arm64/kernel/haoc/iee/iee-si.c index 09ab873ad16c3..2a0925a3d2725 100644 --- a/arch/arm64/kernel/haoc/iee/iee-si.c +++ b/arch/arm64/kernel/haoc/iee/iee-si.c @@ -1,12 +1,18 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include +#include #include #include #include #include +#include #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_PTP +#include +#endif u64 __ro_after_init iee_si_reserved_pg_dir; @@ -21,7 +27,7 @@ static inline void iee_si_check_ttbr0(void) { u64 old_ttbr0 = read_sysreg(ttbr0_el1); u64 old_phys = (old_ttbr0 & PAGE_MASK) & ~TTBR_ASID_MASK; - struct task_token *token = (struct task_token *)__addr_to_iee(current); + struct task_token *token = (struct task_token *)iee_get_task_token(current); /* Phys in TTBR0 shall be the same with current->mm->pgd. */ if (!(current == &init_task) && token->pgd && old_phys != iee_si_reserved_pg_dir) { @@ -57,7 +63,8 @@ unsigned long __iee_si_code iee_si_handler(int flag, ...) if (iee_init_done) iee_si_check_ttbr0(); /* Load the reserved IEE ASID into TTBR0.*/ - ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); + if (haoc_enabled) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); write_sysreg(ttbr0, ttbr0_el1); break; case IEE_SI_CONTEXT_SWITCH: @@ -82,6 +89,8 @@ unsigned long __iee_si_code iee_si_handler(int flag, ...) if (iee_init_done) panic("IEE: Using legacy context switch after IEE init."); + if (haoc_enabled) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); break; @@ -102,6 +111,73 @@ unsigned long __iee_si_code iee_si_handler(int flag, ...) return 0; } +static phys_addr_t __init iee_si_early_pgtable_alloc(void) +{ + phys_addr_t phys; + +#ifdef CONFIG_PTP + phys = early_iee_pgtable_alloc(0); +#else + phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, + MEMBLOCK_ALLOC_NOLEAKTRACE); +#endif + if (!phys) + panic("Failed to allocate page table page\n"); + + return phys; +} + +static inline pgprot_t iee_pmd_pgprot(pmd_t pmd) +{ + unsigned long pfn = pmd_pfn(pmd); + + return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd)); +} + +/* Split PMD block mappings so SI text can carry table/page permissions. */ +static void __init iee_si_split_pmd_early(pud_t *pudp, unsigned long addr) +{ + pmd_t *pmdp = pmd_offset(pudp, addr); + struct page *origin_page; + phys_addr_t pte_phys; + + if (!pmd_leaf(*pmdp)) + return; + + if (pmd_val(*pmdp) & PTE_CONT) { + pmd_t *cont_pmdp = pmd_offset(pudp, addr & CONT_PMD_MASK); + int i; + + for (i = 0; i < CONT_PMDS; i++, cont_pmdp++) + set_pmd(cont_pmdp, __pmd(pmd_val(*cont_pmdp) & ~PTE_CONT)); + } + + origin_page = pmd_page(*pmdp); + pte_phys = iee_si_early_pgtable_alloc(); + if (!pte_phys) + panic("Alloc pgtable error.\n"); + else { + pte_t *ptep = __va(pte_phys); + pgprot_t pgprot = iee_pmd_pgprot(*pmdp); + int i; + +#ifdef CONFIG_PTP + iee_memset(ptep, 0, PAGE_SIZE); +#else + memset(ptep, 0, PAGE_SIZE); +#endif + for (i = 0; i < PTRS_PER_PMD; i++, ptep++) { + pte_t entry; + + pgprot = __pgprot(pgprot_val(pgprot) | PTE_NG | PTE_TYPE_PAGE); + entry = mk_pte(origin_page + i, pgprot); + set_pte(ptep, entry); + } + } + + __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE | PMD_TABLE_UXN); +} + static inline void iee_si_setup_data(void) { iee_si_reserved_pg_dir = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); @@ -111,20 +187,47 @@ static inline void iee_si_setup_data(void) static int __init iee_si_init_code(void) { u64 addr = (u64)__iee_si_text_start; + u64 end = (u64)__iee_si_text_end; pgd_t *pgdir = swapper_pg_dir; - pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); - p4d_t *p4dp = p4d_offset(pgdp, addr); - pud_t *pudp = pud_offset(p4dp, addr); - pmd_t *pmdp = pmd_offset(pudp, addr); - pmd_t pmd = READ_ONCE(*pmdp); - if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) { - pr_err("IEE SI: addr 0x%llx is pmd block. SI init failed.", addr); - return 0; + while (addr < end) { + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + p4d_t *p4dp = p4d_offset(pgdp, addr); + pud_t *pudp = pud_offset(p4dp, addr); + pmd_t *pmdp = pmd_offset(pudp, addr); + pmd_t pmd = READ_ONCE(*pmdp); + pte_t *ptep; + pte_t pte; + + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) { + pr_info("IEE SI: code is on a pmd block. Splitting..."); + iee_si_split_pmd_early(pudp, addr); + pmd = READ_ONCE(*pmdp); + } + + pmd = __pmd(pmd_val(pmd) | PGD_PXNTABLE); + set_pmd(pmdp, pmd); + + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE) { + ptep = pte_offset_kernel(pmdp, addr); + pte = READ_ONCE(*ptep); + + if (pte_val(pte) & PTE_CONT) { + pte_t *cont_ptep = pte_offset_kernel(pmdp, + addr & CONT_PTE_MASK); + int i; + + for (i = 0; i < CONT_PTES; i++, cont_ptep++) + set_pte(cont_ptep, + __pte(pte_val(*cont_ptep) & ~PTE_CONT)); + pte = READ_ONCE(*ptep); + } + pte = __pte(pte_val(pte) | PTE_NG); + set_pte(ptep, pte); + } + addr += PAGE_SIZE; } - pmd = __pmd(pmd_val(pmd) | PGD_PXNTABLE); - set_pmd(pmdp, pmd); flush_tlb_all(); return 1; } diff --git a/arch/arm64/kernel/haoc/iee/iee-token.c b/arch/arm64/kernel/haoc/iee/iee-token.c index 0b341f22c9134..52f03df20e698 100644 --- a/arch/arm64/kernel/haoc/iee/iee-token.c +++ b/arch/arm64/kernel/haoc/iee/iee-token.c @@ -36,10 +36,11 @@ void __init iee_prepare_init_task_token(void) | __phys_to_pte_val(init_token_page)); /* Manaully go through IEE gates to bypass PTP checks. */ #ifdef CONFIG_PTP - write_sysreg(read_sysreg(TCR_EL1) | TCR_HPD1 | TCR_A1, tcr_el1); + write_sysreg((read_sysreg(TCR_EL1) | TCR_HPD1) & ~TCR_A1, tcr_el1); isb(); WRITE_ONCE(*__ptr_to_iee(ptep), pte); - write_sysreg(read_sysreg(TCR_EL1) & ~(TCR_HPD1 | TCR_A1), tcr_el1); + dsb(ishst); + write_sysreg((read_sysreg(TCR_EL1) & ~TCR_HPD1) | TCR_A1, tcr_el1); isb(); #else __set_pte(ptep, pte); @@ -108,6 +109,18 @@ static inline void iee_set_token(unsigned long token_addr, unsigned long token_p flush_tlb_kernel_range(token_pages, (token_pages + (PAGE_SIZE << order))); } +static unsigned long iee_token_page(unsigned long token_addr) +{ + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, token_addr); + p4d_t *p4dp = p4d_offset(pgdp, token_addr); + pud_t *pudp = pud_offset(p4dp, token_addr); + pmd_t *pmdp = pmd_offset(pudp, token_addr); + pte_t *ptep = pte_offset_kernel(pmdp, token_addr); + + return (unsigned long)page_address(pte_page(READ_ONCE(*ptep))); +} + /* * iee_set_token_page_valid() - After allocated task token pages, map them to the * corresponding IEE addresses, and enforce RO protection on their linear mappings. @@ -143,8 +156,11 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, return slab; struct folio *folio = slab_folio(slab); + unsigned int task_order = order; + unsigned int token_order = IEE_TOKEN_ORDER(task_order); unsigned long token_addr = __slab_to_iee(slab); - unsigned long alloc_token = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + unsigned long alloc_token = + __get_free_pages(GFP_KERNEL | __GFP_ZERO, token_order); /* Allocation of task_struct and token pages must be done at the same time. */ if (!alloc_token) { @@ -156,15 +172,33 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, /* Make the mapping reset visible before clearing the flag */ smp_wmb(); __folio_clear_slab(folio); - __free_pages((struct page *)folio, order); + __free_pages((struct page *)folio, task_order); return NULL; } /* Map allocated token pages to token addresses. */ - iee_set_token_page_valid(token_addr, alloc_token, order); + iee_set_token_page_valid(token_addr, alloc_token, token_order); return slab; } +void iee_free_task_token_slab(struct kmem_cache *s, struct slab *slab, + unsigned int order) +{ + unsigned int token_order; + unsigned long token_addr; + unsigned long token_page; + + if (!slab || s != task_struct_cachep) + return; + + token_order = IEE_TOKEN_ORDER(order); + token_addr = __slab_to_iee(slab); + token_page = iee_token_page(token_addr); + + iee_set_token_page_invalid(token_addr, token_page, token_order); + free_pages(token_page, token_order); +} + void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk) { /* Do nothing for now. Wait for later update. */ @@ -173,24 +207,34 @@ void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk) void __iee_code _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)iee_get_task_token(tsk); token->pgd = pgd; } void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)iee_get_task_token(tsk); if (token->valid) pr_err("IEE: validate token for multiple times."); - token->valid = true; +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = tsk->cred; +#endif + seqcount_init(&token->seq); + smp_store_release(&token->valid, true); } void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)iee_get_task_token(tsk); token->pgd = NULL; +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = NULL; +#endif + seqcount_init(&token->seq); token->valid = false; } diff --git a/arch/arm64/kernel/haoc/keyp/Makefile b/arch/arm64/kernel/haoc/keyp/Makefile new file mode 100644 index 0000000000000..307cebd57ff71 --- /dev/null +++ b/arch/arm64/kernel/haoc/keyp/Makefile @@ -0,0 +1 @@ +obj-y += keyp.o \ No newline at end of file diff --git a/arch/arm64/kernel/haoc/keyp/keyp.c b/arch/arm64/kernel/haoc/keyp/keyp.c new file mode 100644 index 0000000000000..a6d4681e97b59 --- /dev/null +++ b/arch/arm64/kernel/haoc/keyp/keyp.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +struct watch_list; + +static inline void iee_verify_key_type(const struct key *key) +{ + iee_verify_type((unsigned long)key, IEE_KEY, "key"); +} + +static inline struct key *key_write_ptr(struct key *key) +{ + iee_verify_key_type(key); + return haoc_enabled ? __ptr_to_iee(key) : key; +} + +unsigned long __iee_code _iee_set_key_flag_bit(unsigned long __unused, struct key *key, + long nr, int flag) +{ + key = key_write_ptr(key); + switch (flag) { + case SET_BIT_OP: { + set_bit(nr, &key->flags); + break; + } + case TEST_AND_CLEAR_BIT: { + return test_and_clear_bit(nr, &key->flags); + } + case TEST_AND_SET_BIT: { + return test_and_set_bit(nr, &key->flags); + } + } + return 0; +} + +void __iee_code _iee_set_key_restrict_link(unsigned long __unused, + struct key *key, + struct key_restriction *restrict_link) +{ + key = key_write_ptr(key); + key->restrict_link = restrict_link; +} + +void __iee_code _iee_set_key_magic(unsigned long __unused, struct key *key, + unsigned int magic) +{ + key = key_write_ptr(key); +#ifdef KEY_DEBUGGING + key->magic = magic; +#endif +} + +void __iee_code _iee_set_key_flags(unsigned long __unused, struct key *key, + unsigned long flags) +{ + key = key_write_ptr(key); + key->flags = flags; +} + +void __iee_code _iee_set_key_index_key(unsigned long __unused, + struct key *key, + struct keyring_index_key *index_key) +{ + key = key_write_ptr(key); + key->index_key = *index_key; +} + +void __iee_code _iee_set_key_hash(unsigned long __unused, struct key *key, + unsigned long hash) +{ + key = key_write_ptr(key); + key->hash = hash; +} + +void __iee_code _iee_set_key_len_desc(unsigned long __unused, struct key *key, + unsigned long len_desc) +{ + key = key_write_ptr(key); + key->len_desc = len_desc; +} + +void __iee_code _iee_set_key_type(unsigned long __unused, struct key *key, + struct key_type *type) +{ + key = key_write_ptr(key); + key->type = type; +} + +void __iee_code _iee_set_key_domain_tag(unsigned long __unused, + struct key *key, + struct key_tag *domain_tag) +{ + key = key_write_ptr(key); + key->domain_tag = domain_tag; +} + +void __iee_code _iee_set_key_description(unsigned long __unused, + struct key *key, char *description) +{ + key = key_write_ptr(key); + key->description = description; +} + +void __iee_code _iee_set_key_uid(unsigned long __unused, struct key *key, + kuid_t uid) +{ + key = key_write_ptr(key); + key->uid = uid; +} + +void __iee_code _iee_set_key_gid(unsigned long __unused, struct key *key, + kgid_t gid) +{ + key = key_write_ptr(key); + key->gid = gid; +} + +void __iee_code _iee_set_key_perm(unsigned long __unused, struct key *key, + key_perm_t perm) +{ + key = key_write_ptr(key); + key->perm = perm; +} + +void __iee_code _iee_set_key_quotalen(unsigned long __unused, struct key *key, + unsigned short quotalen) +{ + key = key_write_ptr(key); + key->quotalen = quotalen; +} + +void __iee_code _iee_set_key_datalen(unsigned long __unused, struct key *key, + unsigned short datalen) +{ + key = key_write_ptr(key); + key->datalen = datalen; +} + +void __iee_code _iee_set_key_state(unsigned long __unused, struct key *key, + short state) +{ + key = key_write_ptr(key); + smp_store_release(&key->state, state); +} + +void __iee_code _iee_set_key_user(unsigned long __unused, struct key *key, + struct key_user *user) +{ + key = key_write_ptr(key); + key->user = user; +} + +void __iee_code _iee_set_key_security(unsigned long __unused, struct key *key, + void *security) +{ + key = key_write_ptr(key); + key->security = security; +} + +void __iee_code _iee_set_key_expiry(unsigned long __unused, struct key *key, + time64_t expiry) +{ + key = key_write_ptr(key); + key->expiry = expiry; +} + +void __iee_code _iee_set_key_revoked_at(unsigned long __unused, + struct key *key, time64_t revoked_at) +{ + key = key_write_ptr(key); + key->revoked_at = revoked_at; +} + +void __iee_code _iee_set_key_last_used_at(unsigned long __unused, + struct key *key, + time64_t last_used_at) +{ + key = key_write_ptr(key); + key->last_used_at = last_used_at; +} + +unsigned long __iee_code _iee_set_key_usage(unsigned long __unused, struct key *key, + int n, int flag) +{ + key = key_write_ptr(key); + switch (flag) { + case REFCOUNT_INC: { + refcount_inc(&key->usage); + break; + } + case REFCOUNT_SET: { + refcount_set(&key->usage, n); + break; + } + case REFCOUNT_DEC_AND_TEST: { + return refcount_dec_and_test(&key->usage); + } + case REFCOUNT_INC_NOT_ZERO: { + return refcount_inc_not_zero(&key->usage); + } + } + return 0; +} + +void __iee_code _iee_set_key_serial(unsigned long __unused, struct key *key, + key_serial_t serial) +{ + key = key_write_ptr(key); + key->serial = serial; +} + +void __iee_code _iee_set_key_watchers(unsigned long __unused, struct key *key, + struct watch_list *watchers) +{ + key = key_write_ptr(key); +#ifdef CONFIG_KEY_NOTIFICATIONS + key->watchers = watchers; +#endif +} + +void __iee_code _iee_set_key_union(unsigned long __unused, struct key *key, + struct key_union *key_union) +{ + key = key_write_ptr(key); + key->graveyard_link.next = (struct list_head *)key_union; +} + +void __iee_code _iee_set_key_struct(unsigned long __unused, struct key *key, + struct key_struct *key_struct) +{ + key = key_write_ptr(key); + key->name_link.prev = (struct list_head *)key_struct; +} + +void __iee_code _iee_set_key_payload(unsigned long __unused, struct key *key, + union key_payload *key_payload) +{ + key = key_write_ptr(key); + key->name_link.next = (struct list_head *)key_payload; +} diff --git a/arch/arm64/kernel/haoc/ptp/Makefile b/arch/arm64/kernel/haoc/ptp/Makefile new file mode 100644 index 0000000000000..c2887cf5c18ce --- /dev/null +++ b/arch/arm64/kernel/haoc/ptp/Makefile @@ -0,0 +1 @@ +obj-y += ptp.o iee-ptp-init.o diff --git a/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c b/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c new file mode 100644 index 0000000000000..cb971ca2e6611 --- /dev/null +++ b/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#define PTP_CHECK(condition) do { \ + if (unlikely(condition)) \ + panic("PTP check failed on %s.", __func__); \ +} while (0) + +#if CONFIG_PGTABLE_LEVELS > 3 +void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) +{ + iee_set_p4d_pre_init(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); +} +#else +void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) +{ + BUILD_BUG(); +} +#endif + +void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) +{ + iee_set_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); +} + +void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, + pmdval_t prot) +{ + iee_set_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); +} + +int iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot) +{ + pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); + + /* Only allow permission changes for now */ + if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), + pud_val(new_pud))) + return 0; + + WARN_ON_ONCE(phys & ~PUD_MASK); + iee_set_pud_pre_init(pudp, new_pud); + return 1; +} + +int iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) +{ + pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); + + /* Only allow permission changes for now */ + if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), + pmd_val(new_pmd))) + return 0; + + WARN_ON_ONCE(phys & ~PMD_MASK); + iee_set_pmd_pre_init(pmdp, new_pmd); + return 1; +} + +static inline pte_t *fixmap_pte(unsigned long addr) +{ + return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; +} + +void __iee_set_fixmap_pre_init(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *ptep; + + PTP_CHECK(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + + ptep = fixmap_pte(addr); + + if (pgprot_val(flags)) { + iee_set_pte_pre_init(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); + } else { + iee_set_pte_pre_init(ptep, __pte(0)); + flush_tlb_kernel_range(addr, addr+PAGE_SIZE); + } +} + +void iee_set_pgtable_pre_init(unsigned long *addr, unsigned long content) +{ + WRITE_ONCE(*addr, content); +} + +void set_iee_address_pre_init(unsigned long addr, bool valid) +{ + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + + p4d_t *p4dp = p4d_offset(pgdp, addr); + + pud_t *pudp = pud_offset(p4dp, addr); + + pmd_t *pmdp = pmd_offset(pudp, addr); + + pte_t *ptep = pte_offset_kernel(pmdp, addr); + pte_t pte = READ_ONCE(*ptep); + + if (valid) + pte = __pte(pte_val(pte) | PTE_VALID); + else + pte = __pte(pte_val(pte) & ~PTE_VALID); + iee_set_pgtable_pre_init((unsigned long *)ptep, (unsigned long)pte.pte); + dsb(ishst); + isb(); +} + +static void iee_init_pte_pre_init(pmd_t *pmdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot) +{ + pte_t *ptep; + + ptep = pte_set_fixmap_offset_pre_init(pmdp, addr); + do { + pte_t old_pte = __ptep_get(ptep); + + iee_set_pgtable_pre_init((unsigned long *)ptep, + (unsigned long)(pfn_pte(__phys_to_pfn(phys), prot).pte)); + + /* + * After the PTE entry has been populated once, we + * only allow updates to the permission attributes. + */ + PTP_CHECK(!pgattr_change_is_safe(pte_val(old_pte), + pte_val(__ptep_get(ptep)))); + + phys += PAGE_SIZE; + } while (ptep++, addr += PAGE_SIZE, addr != end); + + pte_clear_fixmap_pre_init(); +} + +static void iee_alloc_init_cont_pte_pre_init(pmd_t *pmdp, unsigned long addr, + unsigned long end, phys_addr_t phys, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + unsigned long next; + pmd_t pmd = READ_ONCE(*pmdp); + + PTP_CHECK(pmd_sect(pmd)); + if (pmd_none(pmd)) { + pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF; + phys_addr_t pte_phys; + + if (flags & NO_EXEC_MAPPINGS) + pmdval |= PMD_TABLE_PXN; + PTP_CHECK(!pgtable_alloc); + pte_phys = pgtable_alloc(PAGE_SHIFT); + iee_set_pgtable_pre_init((unsigned long *)pmdp, + (unsigned long)(__phys_to_pmd_val(pte_phys) | pmdval)); + pmd = READ_ONCE(*pmdp); + } + PTP_CHECK(pmd_bad(pmd)); + + do { + pgprot_t __prot = prot; + + next = pte_cont_addr_end(addr, end); + + /* use a contiguous mapping if the range is suitably aligned */ + if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && + (flags & NO_CONT_MAPPINGS) == 0) + __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + + iee_init_pte_pre_init(pmdp, addr, next, phys, __prot); + + phys += next - addr; + } while (addr = next, addr != end); +} + +static void iee_init_pmd_pre_init(pud_t *pudp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), int flags) +{ + unsigned long next; + pmd_t *pmdp; + + pmdp = pmd_set_fixmap_offset_pre_init(pudp, addr); + do { + pmd_t old_pmd = READ_ONCE(*pmdp); + + next = pmd_addr_end(addr, end); + + /* try section mapping first */ + if (((addr | next | phys) & ~PMD_MASK) == 0 && + (flags & NO_BLOCK_MAPPINGS) == 0) { + iee_set_pgtable_pre_init((unsigned long *)pmdp, + (unsigned long)(pfn_pmd(__phys_to_pfn(phys), + mk_pmd_sect_prot(prot)).pmd)); + + /* + * After the PMD entry has been populated once, we + * only allow updates to the permission attributes. + */ + PTP_CHECK(!pgattr_change_is_safe(pmd_val(old_pmd), + READ_ONCE(pmd_val(*pmdp)))); + } else { + iee_alloc_init_cont_pte_pre_init(pmdp, addr, next, phys, prot, + pgtable_alloc, flags); + + PTP_CHECK(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + } + phys += next - addr; + } while (pmdp++, addr = next, addr != end); + + pmd_clear_fixmap_pre_init(); +} + +static void iee_alloc_init_cont_pmd_pre_init(pud_t *pudp, unsigned long addr, + unsigned long end, phys_addr_t phys, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), int flags) +{ + unsigned long next; + pud_t pud = READ_ONCE(*pudp); + + /* + * Check for initial section mappings in the pgd/pud. + */ + PTP_CHECK(pud_sect(pud)); + if (pud_none(pud)) { + pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF; + phys_addr_t pmd_phys; + + if (flags & NO_EXEC_MAPPINGS) + pudval |= PUD_TABLE_PXN; + PTP_CHECK(!pgtable_alloc); + pmd_phys = pgtable_alloc(PMD_SHIFT); + iee_set_pgtable_pre_init((unsigned long *)pudp, + (unsigned long)(__phys_to_pud_val(pmd_phys) | pudval)); + pud = READ_ONCE(*pudp); + } + PTP_CHECK(pud_bad(pud)); + + do { + pgprot_t __prot = prot; + + next = pmd_cont_addr_end(addr, end); + + /* use a contiguous mapping if the range is suitably aligned */ + if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && + (flags & NO_CONT_MAPPINGS) == 0) + __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + + iee_init_pmd_pre_init(pudp, addr, next, phys, __prot, pgtable_alloc, flags); + + phys += next - addr; + } while (addr = next, addr != end); +} + +void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + unsigned long next; + pud_t *pudp; + p4d_t *p4dp = p4d_offset(pgdp, addr); + p4d_t p4d = READ_ONCE(*p4dp); + + if (p4d_none(p4d)) { + p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF; + phys_addr_t pud_phys; + + if (flags & NO_EXEC_MAPPINGS) + p4dval |= P4D_TABLE_PXN; + PTP_CHECK(!pgtable_alloc); + pud_phys = pgtable_alloc(PUD_SHIFT); + iee_set_pgtable_pre_init((unsigned long *)p4dp, + (unsigned long)(__phys_to_p4d_val(pud_phys) | p4dval)); + p4d = READ_ONCE(*p4dp); + } + PTP_CHECK(p4d_bad(p4d)); + + pudp = pud_set_fixmap_offset_pre_init(p4dp, addr); + do { + pud_t old_pud = READ_ONCE(*pudp); + + next = pud_addr_end(addr, end); + + iee_alloc_init_cont_pmd_pre_init(pudp, addr, next, phys, prot, + pgtable_alloc, flags); + + PTP_CHECK(pud_val(old_pud) != 0 && + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); + phys += next - addr; + } while (pudp++, addr = next, addr != end); + + pud_clear_fixmap_pre_init(); +} + +static void __create_pgd_mapping_locked_pre_init(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + unsigned long addr, end, next; + pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); + + /* + * If the virtual and physical address don't have the same offset + * within a page, we cannot map the region as the caller expects. + */ + if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) + return; + + phys &= PAGE_MASK; + addr = virt & PAGE_MASK; + end = PAGE_ALIGN(virt + size); + + do { + next = pgd_addr_end(addr, end); + iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, + flags); + phys += next - addr; + } while (pgdp++, addr = next, addr != end); +} + +void __create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + mutex_lock(&fixmap_lock); + __create_pgd_mapping_locked_pre_init(pgdir, phys, virt, size, prot, + pgtable_alloc, flags); + mutex_unlock(&fixmap_lock); +} + +static void __init early_iounmap_after_init(void __iomem *addr, unsigned long size) +{ + unsigned long virt_addr; + unsigned long offset; + unsigned int nrpages; + enum fixed_addresses idx; + int i, slot; + + slot = -1; + for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { + if (prev_map[i] == addr) { + slot = i; + break; + } + } + + if (WARN(slot < 0, "early_iounmap(%p, %08lx) not found slot\n", + addr, size)) + return; + + if (WARN(prev_size[slot] != size, + "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n", + addr, size, slot, prev_size[slot])) + return; + + WARN(early_ioremap_debug, "early_iounmap(%p, %08lx) [%d]\n", + addr, size, slot); + + virt_addr = (unsigned long)addr; + if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN))) + return; + + offset = offset_in_page(virt_addr); + nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT; + + idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; + while (nrpages > 0) { + if (after_paging_init) + __late_clear_fixmap(idx); + else + __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); + --idx; + --nrpages; + } + prev_map[slot] = NULL; +} + +void __init efi_memmap_unmap_after_init(void) +{ + if (!efi_enabled(EFI_MEMMAP)) + return; + + if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) { + unsigned long size; + + size = efi.memmap.desc_size * efi.memmap.nr_map; + early_iounmap_after_init((__force void __iomem *)efi.memmap.map, size); + } else { + memunmap(efi.memmap.map); + } + + efi.memmap.map = NULL; + clear_bit(EFI_MEMMAP, &efi.flags); +} diff --git a/arch/arm64/kernel/haoc/ptp/ptp.c b/arch/arm64/kernel/haoc/ptp/ptp.c new file mode 100644 index 0000000000000..6bc000906c8ba --- /dev/null +++ b/arch/arm64/kernel/haoc/ptp/ptp.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_PTP_S +static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte); +static inline bool check_pte_dep(char *addr, pte_t pte); +static inline bool check_pmd_dep(char *addr, pmd_t pmd); + +bool ptp_is_user_pgtable(const void *ptp) +{ + unsigned long check_addr = (unsigned long)ptp; + + if (!(haoc_enabled && haoc_bitmap_ready && iee_init_done)) + return false; + + /* + * iee_get_bitmap_type() relies on __va_to_haoc_bitmap(). + * Normalize IEE alias addresses back to linear-map address first. + */ + if (unlikely(!__is_lm_address((u64)check_addr))) { + if (!(check_addr & IEE_OFFSET)) + return false; + + check_addr = (unsigned long)__iee_to_virt(check_addr); + if (unlikely(!__is_lm_address((u64)check_addr))) + return false; + } + + if (unlikely(!virt_addr_valid((void *)check_addr))) + return false; + + return iee_get_bitmap_type(check_addr) == IEE_USER_PGTABLE; +} + +/* + * User fast-path pre-check for PTE updates under CONFIG_PTP_S. + * Preserve alias/integrity checks before direct write. + */ +void ptp_user_check_pte_update(pte_t *ptep, pte_t pte) +{ + char *addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); + + if (!(pte_val(pte) & PTE_VALID)) + return; + + if (haoc_enabled && !is_changing_pte_prot(ptep, pte) && + check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) + panic("You are remmaping IEE page to other VA.\n"); + + // Avoid mapping a writable VA to kernel code PA. + check_pte_dep(addr, pte); + +} + +/* + * User fast-path pre-check for PMD updates under CONFIG_PTP_S. + * For leaf PMD mappings, preserve alias checking before direct write. + */ +void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd) +{ + char *addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); + + if (!(pmd_val(pmd) & PMD_SECT_VALID)) + return; + + // Check if the pte table is legally allocated. + if (haoc_enabled && (pmd_val(pmd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) + panic("You can't use non-iee-pgtable\n"); + + // Avoid mapping a huge pmd to IEE physical page. + // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) + // panic("Mapping IEE physical page to a huge pmd.\n"); + + check_pmd_dep(addr, pmd); +} + +/* + * User fast-path pre-check for PUD updates under CONFIG_PTP_S. + * For leaf PUD mappings, preserve alias checking before direct write. + */ +void ptp_user_check_pud_update(pud_t *pudp, pud_t pud) +{ + if (!(pud_val(pud) & PMD_SECT_VALID)) + return; + + if (haoc_enabled && (pud_val(pud) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) + panic("You can't use non-iee-pgtable\n"); +} + +/* + * User fast-path pre-check for P4D/PGD updates under CONFIG_PTP_S. + * A bad top-level table entry poisons later fault walks before lower-level + * checks can run, so keep the same target-page validation here too. + */ +void ptp_user_check_p4d_update(p4d_t *p4dp, p4d_t p4d) +{ + if (!(p4d_val(p4d) & PMD_SECT_VALID)) + return; + + if (haoc_enabled && (p4d_val(p4d) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) + panic("You can't use non-iee-pgtable\n"); +} +#endif + +bool check_addr_in_iee_valid(unsigned long addr) +{ + pgd_t *pgdir = swapper_pg_dir; + + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + p4d_t *p4dp = p4d_offset(pgdp, addr); + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + if (!(p4d_val(READ_ONCE(*p4dp)) & PTE_VALID)) + return false; + + pudp = pud_offset(p4dp, addr); + + if (!(pud_val(READ_ONCE(*pudp)) & PTE_VALID)) + return false; + + pmdp = pmd_offset(pudp, addr); + + if (!(pmd_val(READ_ONCE(*pmdp)) & PTE_VALID)) + return false; + + ptep = pte_offset_kernel(pmdp, addr); + + return (pte_val(READ_ONCE(*ptep)) & PTE_VALID); +} + +// Return true if it is only changing prot of a pte. +static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte) +{ + if (((pte_val(*ptep) ^ pte_val(pte)) & PTE_ADDR_MASK) == 0) + return true; + else + return false; +} + +// Return true if the modify does not break DEP. +static inline bool check_pte_dep(char *addr, pte_t pte) +{ + // DEP for kernel code and readonly data + // _text: .text start addr, __init_begin: .rodata end addr + if (addr >= _stext && addr < _etext) { + if ((PTE_WRITE & pte_val(pte)) // DBM == 1 --> writable + || !(PTE_RDONLY & pte_val(pte))) { // DBM == 0 && AP[2] = 0 --> writable + panic("Can't make kernel's text/readonly page as writable!\n" + "addr = 0x%16llx, pte_val = 0x%16llx", + (u64)addr, pte_val(pte)); + } + } + return true; +} + +// Return true if the modify does not break DEP. +static inline bool check_pmd_dep(char *addr, pmd_t pmd) +{ + // DEP for kernel code and readonly data + // _text: .text start addr, __init_begin: .rodata end addr + if (addr >= _stext && addr < _etext) { + if ((PTE_WRITE & pmd_val(pmd)) || // DBM == 1 --> writable + !(PTE_RDONLY & pmd_val(pmd))) { // DBM == 0 && AP[2] = 0 --> writable + panic("Can't make kernel's text/readonly page as writable!\n" + "addr = 0x%16llx, pmd_val = 0x%16llx", + (u64)addr, pmd_val(pmd)); + } + } + return true; +} + +void __iee_code _iee_set_static_pgd(int flag, pgd_t *pgdp, pgd_t pgd) +{ + if (haoc_enabled && (pgd_val(pgd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(pgdp), pgd); +} + +void __iee_code _iee_set_bm_pte(int flag, pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*__ptr_to_iee(ptep), pte); +} + +void __iee_code _iee_set_pte(int flag, pte_t *ptep, pte_t pte) +{ + char *addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); + + if (!(pte_val(pte) & PTE_VALID)) { + WRITE_ONCE(*__ptr_to_iee(ptep), pte); + return; + } + + // Avoid mapping a new VA to IEE PA. + if (haoc_enabled && !is_changing_pte_prot(ptep, pte) && + check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) + panic("You are remmaping IEE page to other VA.\n"); + + // Avoid mapping a writable VA to kernel code PA. + if (!check_pte_dep(addr, pte)) + return; + + WRITE_ONCE(*__ptr_to_iee(ptep), pte); +} + +void __iee_code _iee_set_pmd(int flag, pmd_t *pmdp, pmd_t pmd) +{ + char *addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); + + if (!(pmd_val(pmd) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(pmdp), pmd); + return; + } + + // Check if the pte table is legally allocated. + if (haoc_enabled && (pmd_val(pmd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) + panic("You can't use non-iee-pgtable\n"); + + // Avoid mapping a huge pmd to IEE physical page. + // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) + // panic("Mapping IEE physical page to a huge pmd.\n"); + + if (!check_pmd_dep(addr, pmd)) + return; + + WRITE_ONCE(*__ptr_to_iee(pmdp), pmd); +} + +void __iee_code _iee_set_pud(int flag, pud_t *pudp, pud_t pud) +{ + if (!(pud_val(pud) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(pudp), pud); + return; + } + + if (haoc_enabled && (pud_val(pud) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(pudp), pud); +} + +void __iee_code _iee_set_p4d(int flag, p4d_t *p4dp, p4d_t p4d) +{ + if (!(p4d_val(p4d) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(p4dp), p4d); + return; + } + + if (haoc_enabled && (p4d_val(p4d) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(p4dp), p4d); +} + +void __iee_code _iee_set_swapper_pgd(int flag, pgd_t *pgdp, pgd_t pgd) +{ + if (!(pgd_val(pgd) & PMD_SECT_VALID)) { + WRITE_ONCE(*__ptr_to_iee(pgdp), pgd); + return; + } + + if (haoc_enabled && (pgd_val(pgd) & PMD_TABLE_BIT) && + !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) + panic("You can't use non-iee-pgtable\n"); + + WRITE_ONCE(*__ptr_to_iee(pgdp), pgd); +} + +pteval_t __iee_code _iee_set_xchg_relaxed(int flag, pte_t *ptep, pteval_t pteval) +{ + pteval_t ret = xchg_relaxed((pteval_t *)__ptr_to_iee(ptep), pteval); + return ret; +} + +pmdval_t __iee_code _iee_set_pmd_xchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t pmdval) +{ + pmdval_t ret = xchg_relaxed((pmdval_t *)__ptr_to_iee(pmdp), pmdval); + return ret; +} + +pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, + pteval_t old_pteval, pteval_t new_pteval) +{ + pteval_t pteval = cmpxchg_relaxed((pteval_t *)__ptr_to_iee(ptep), + old_pteval, new_pteval); + + return pteval; +} + +pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, + pmdval_t old_pmdval, pmdval_t new_pmdval) +{ + pmdval_t pmdval = cmpxchg_relaxed((pmdval_t *)__ptr_to_iee(pmdp), + old_pmdval, new_pmdval); + + return pmdval; +} + +void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, + int order, int use_block_pmd, bool writable) +{ + int i; + + lm_ptep = __ptr_to_iee(lm_ptep); + iee_ptep = __ptr_to_iee(iee_ptep); + if (use_block_pmd) { + #ifdef CONFIG_IEE_ALLOW_SPLIT_LM + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + if (writable) + pmd = __pmd((pmd_val(pmd) & ~PMD_SECT_RDONLY) | PTE_DBM); + else + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + #else + if (order == IEE_DATA_ORDER) { + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + if (writable) + pmd = __pmd((pmd_val(pmd) & ~PMD_SECT_RDONLY) | PTE_DBM); + else + pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + } else { + /* Give up RO protection on linear mapping to avoid page table spliting. */ + // pr_err("IEE: give up RO protection to avoid page table split."); + } + #endif + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*iee_ptep); + + pte = __pte(pte_val(pte) | PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + iee_ptep++; + } + } else { + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*lm_ptep); + + if (writable) + pte = __pte((pte_val(pte) & ~PTE_RDONLY) | PTE_DBM); + else + pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); + WRITE_ONCE(*lm_ptep, pte); + pte = READ_ONCE(*iee_ptep); + pte = __pte(pte_val(pte) | PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + lm_ptep++; + iee_ptep++; + } + } +} + +void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, + int order, int use_block_pmd) +{ + int i; + + lm_ptep = __ptr_to_iee(lm_ptep); + iee_ptep = __ptr_to_iee(iee_ptep); + if (use_block_pmd) { + #ifdef CONFIG_IEE_ALLOW_SPLIT_LM + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + pmd = __pmd(pmd_val(pmd) | PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + #else + if (order == IEE_DATA_ORDER) { + pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + + pmd = __pmd(pmd_val(pmd) | PTE_DBM); + WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); + } else { + /* Give up RO protection on linear mapping to avoid page table spliting. */ + // pr_err("IEE: give up RO protection to avoid page table split."); + } + #endif + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*iee_ptep); + + pte = __pte(pte_val(pte) & ~PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + iee_ptep++; + } + } else { + for (i = 0; i < (1 << order); i++) { + pte_t pte = READ_ONCE(*lm_ptep); + + pte = __pte(pte_val(pte) | PTE_DBM); + WRITE_ONCE(*lm_ptep, pte); + pte = READ_ONCE(*iee_ptep); + pte = __pte(pte_val(pte) & ~PTE_VALID); + WRITE_ONCE(*iee_ptep, pte); + lm_ptep++; + iee_ptep++; + } + } +} diff --git a/arch/arm64/kernel/haoc/selinuxp/Makefile b/arch/arm64/kernel/haoc/selinuxp/Makefile new file mode 100644 index 0000000000000..0e5b0be4376c8 --- /dev/null +++ b/arch/arm64/kernel/haoc/selinuxp/Makefile @@ -0,0 +1,17 @@ +ccflags-y += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +ccflags-y += -I$(objtree)/security/selinux +obj-y += selinuxp.o + +ifdef CONFIG_IEE_SELINUX_P +SELINUX_GENHDRS := $(objtree)/security/selinux/flask.h \ + $(objtree)/security/selinux/av_permissions.h + +quiet_cmd_haoc_genhdrs = GEN $(SELINUX_GENHDRS) + cmd_haoc_genhdrs = mkdir -p $(objtree)/security/selinux; \ + $< $(SELINUX_GENHDRS) + +$(SELINUX_GENHDRS): scripts/selinux/genheaders/genheaders FORCE + $(call if_changed,haoc_genhdrs) + +$(obj)/selinuxp.o: $(SELINUX_GENHDRS) +endif diff --git a/arch/arm64/kernel/haoc/selinuxp/selinuxp.c b/arch/arm64/kernel/haoc/selinuxp/selinuxp.c new file mode 100644 index 0000000000000..ce4a89f8f7adf --- /dev/null +++ b/arch/arm64/kernel/haoc/selinuxp/selinuxp.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "security.h" +#include "ss/services.h" +#include + +static inline void iee_verify_selinux_type(unsigned long va) +{ + iee_verify_type(va, IEE_SELINUX, "selinux"); +} + +void __iee_code _iee_set_selinux_status_pg(unsigned long iee_offset, + struct page *new_page) +{ + *__ptr_to_iee(&selinux_state.status_page) = new_page; +} + +void __iee_code _iee_set_selinux_enforcing(unsigned long iee_offset, bool value) +{ + *__ptr_to_iee(&selinux_state.enforcing) = value; +} + +void __iee_code _iee_mark_selinux_initialized(unsigned long iee_offset) +{ + /* do a synchronized write to avoid race conditions */ + smp_store_release(__ptr_to_iee(&(selinux_state.initialized)), true); + if (haoc_enabled) + pr_info("HAOC: Mark selinux initialized."); +} + +void __iee_code _iee_set_sel_policy_cap(unsigned long iee_offset, + unsigned int idx, int cap) +{ + *__ptr_to_iee(&(selinux_state.policycap[idx])) = cap; +} + +/* + * Please make sure param iee_new_policy is from policy_jar memcache. + * Need to free new_policy after calling this func as it's only used to + * trans data from kernel. + */ +void __iee_code _iee_sel_rcu_assign_policy(unsigned long iee_offset, + struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + /* TODO: Verify information from incoming policy. */ + iee_verify_selinux_type((unsigned long)iee_new_policy); + + /* Copy data from kernel to new allocated policy struct inside iee. */ + memcpy(__ptr_to_iee(iee_new_policy), new_policy, sizeof(struct selinux_policy)); + rcu_assign_pointer(*__ptr_to_iee(&selinux_state.policy), iee_new_policy); +} diff --git a/arch/arm64/kernel/haoc/varp/Makefile b/arch/arm64/kernel/haoc/varp/Makefile new file mode 100644 index 0000000000000..775b2da3f64b1 --- /dev/null +++ b/arch/arm64/kernel/haoc/varp/Makefile @@ -0,0 +1 @@ +obj-y += varp.o \ No newline at end of file diff --git a/arch/arm64/kernel/haoc/varp/varp.c b/arch/arm64/kernel/haoc/varp/varp.c new file mode 100644 index 0000000000000..714a87571a348 --- /dev/null +++ b/arch/arm64/kernel/haoc/varp/varp.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +void __iee_code _iee_set_varp_modprobe_path(unsigned long __unused, char *data, + int maxlen, size_t len, char *buffer, + size_t *lenp) +{ + char c, *p; + + if (!data || !buffer || !lenp) + return; + data = __ptr_to_iee(data); + p = buffer; + while ((p - buffer) < *lenp && len < maxlen - 1) { + c = *(p++); + if (c == 0 || c == '\n') + break; + data[len++] = c; + } + data[len] = 0; +} diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index c6adf753e6d00..0e38fe006e9b5 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -371,6 +371,17 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) unflatten_device_tree(); bootmem_init(); +#ifdef CONFIG_PTP + /* Prepare page pool for page tables. */ + if (haoc_enabled) { + iee_cache_init(&pg_cache, 0, CONFIG_PGTABLE_LEVELS, IEE_PGTABLE, + CONFIG_PTP_RESERVE_ORDER); +#ifdef CONFIG_PTP_S + iee_cache_init(&pg_user_cache, 0, CONFIG_PGTABLE_LEVELS, IEE_USER_PGTABLE, + CONFIG_PTP_RESERVE_ORDER); +#endif + } +#endif kasan_init(); diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 4da97244a2ff9..06588d80da678 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -148,21 +148,60 @@ jiffies = jiffies_64; #define IEE_TEXT #endif +#ifdef CONFIG_PTP +#define PTP_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_ptp_data_start = .; \ + *(.iee.ptp) \ + . = ALIGN(PAGE_SIZE); \ + __iee_ptp_data_end = .; +#else +#define PTP_DATA +#endif + #ifdef CONFIG_CREDP #define CRED_DATA \ . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_start = .; \ *(.iee.cred) \ + . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_end = .; \ . = ALIGN(PAGE_SIZE); #else #define CRED_DATA #endif +#ifdef CONFIG_IEE_SELINUX_P +#define IEE_SELINUX_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_start = .; \ + *(.iee.selinux) \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_end = .; +#else +#define IEE_SELINUX_DATA +#endif + +#ifdef CONFIG_VARP +#define VARP_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_start = .; \ + *(.iee.varp) \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_end = .; +#else +#define VARP_DATA +#endif + #ifdef CONFIG_IEE #define IEE_INIT_DATA \ .iee.data : { \ . = ALIGN(PAGE_SIZE); \ iee_init_data_begin = .; \ + PTP_DATA \ CRED_DATA \ + IEE_SELINUX_DATA \ + VARP_DATA \ . = ALIGN(PAGE_SIZE); \ iee_init_data_end = .; \ } diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2edb093fe8b66..9c0f08e2bb827 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -42,6 +42,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #ifdef CONFIG_IEE_SIP #include #endif @@ -238,7 +241,11 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, pteval ^= PTE_RDONLY; pteval |= pte_val(entry); pteval ^= PTE_RDONLY; + #ifdef CONFIG_PTP + pteval = iee_set_cmpxchg_relaxed(ptep, old_pteval, pteval); + #else pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); + #endif } while (pteval != old_pteval); /* Invalidate a stale read-only entry */ @@ -314,7 +321,10 @@ static void die_kernel_fault(const char *msg, unsigned long addr, pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg, addr); - +#ifdef CONFIG_IEE + if (__is_lm_address((u64)addr)) + pr_alert("iee_get_bitmap_type(addr) : %d\n", iee_get_bitmap_type(addr)); +#endif kasan_non_canonical_hook(addr); mem_abort_decode(esr); diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index fbf904db2442f..4374f608973cc 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -16,6 +16,11 @@ #include #include +#ifdef CONFIG_PTP +#include +#include +#endif + /* ensure that the fixmap region does not grow down into the PCI I/O region */ static_assert(FIXADDR_TOT_START > PCI_IO_END); @@ -31,15 +36,35 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) +#ifdef CONFIG_PTP +pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __section(".iee.ptp") __aligned(PAGE_SIZE); +pmd_t bm_pmd[PTRS_PER_PMD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +pud_t bm_pud[PTRS_PER_PUD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +#else static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +#endif static inline pte_t *fixmap_pte(unsigned long addr) { return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; } +#ifdef CONFIG_PTP +static inline void set_bm_fixmap_pte(pte_t *ptep, pte_t pte) +{ + if (haoc_enabled && iee_init_done) { + iee_set_bm_pte(ptep, pte); + return; + } + + WRITE_ONCE(*ptep, pte); + dsb(ishst); + isb(); +} +#endif + static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) { pmd_t pmd = READ_ONCE(*pmdp); @@ -47,8 +72,12 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) if (pmd_none(pmd)) { ptep = bm_pte[BM_PTE_TABLE_IDX(addr)]; +#ifdef CONFIG_PTP + __iee_pmd_populate_pre_init(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE); +#else __pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE | PMD_TABLE_AF); +#endif } } @@ -59,9 +88,14 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, pud_t pud = READ_ONCE(*pudp); pmd_t *pmdp; - if (pud_none(pud)) + if (pud_none(pud)) { +#ifdef CONFIG_PTP + __iee_pud_populate_pre_init(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); +#else __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE | PUD_TABLE_AF); +#endif + } pmdp = pmd_offset_kimg(pudp, addr); do { @@ -87,9 +121,14 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); } - if (p4d_none(p4d)) + if (p4d_none(p4d)) { +#ifdef CONFIG_PTP + __iee_p4d_populate_pre_init(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); +#else __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE | P4D_TABLE_AF); +#endif + } pudp = pud_offset_kimg(p4dp, addr); early_fixmap_init_pmd(pudp, addr, end); @@ -107,7 +146,11 @@ void __init early_fixmap_init(void) unsigned long end = FIXADDR_TOP; pgd_t *pgdp = pgd_offset_k(addr); +#ifdef CONFIG_PTP + p4d_t *p4dp = p4d_offset_kimg(pgdp, addr); +#else p4d_t *p4dp = p4d_offset(pgdp, addr); +#endif early_fixmap_init_pud(p4dp, addr, end); } @@ -127,9 +170,17 @@ void __set_fixmap(enum fixed_addresses idx, ptep = fixmap_pte(addr); if (pgprot_val(flags)) { +#ifdef CONFIG_PTP + set_bm_fixmap_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); +#else __set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); +#endif } else { +#ifdef CONFIG_PTP + set_bm_fixmap_pte(ptep, __pte(0)); +#else __pte_clear(&init_mm, addr, ptep); +#endif flush_tlb_kernel_range(addr, addr+PAGE_SIZE); } } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index a5284c19c285e..511de939cd970 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -50,6 +50,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif /* * We need to be able to catch inadvertent references to memstart_addr @@ -549,6 +552,9 @@ void __init mem_init(void) void free_initmem(void) { + #ifdef CONFIG_IEE + if (!haoc_enabled) + #endif free_reserved_area(lm_alias(__init_begin), lm_alias(__init_end), POISON_FREE_INITMEM, "unused kernel"); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d3ca3fc09b265..e56da8dbcec5a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -42,12 +42,16 @@ #include #ifdef CONFIG_IEE #include +#include #include #include #ifdef CONFIG_IEE_SIP #include #endif #endif +#ifdef CONFIG_PTP +#include +#endif #define NO_BLOCK_MAPPINGS BIT(0) #define NO_CONT_MAPPINGS BIT(1) @@ -82,7 +86,9 @@ unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_ EXPORT_SYMBOL(empty_zero_page); static DEFINE_SPINLOCK(swapper_pgdir_lock); +#ifndef CONFIG_IEE static DEFINE_MUTEX(fixmap_lock); +#endif void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) { @@ -115,6 +121,11 @@ static phys_addr_t __init early_pgtable_alloc(int shift) { phys_addr_t phys; +#ifdef CONFIG_PTP + if (haoc_enabled) + return early_iee_pgtable_alloc(shift); +#endif + phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, MEMBLOCK_ALLOC_NOLEAKTRACE); if (!phys) @@ -421,7 +432,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, { mutex_lock(&fixmap_lock); #ifdef CONFIG_IEE - if (haoc_enabled) + if (haoc_enabled && !iee_init_done) __iee_create_pgd_mapping_locked(pgdir, phys, virt, size, prot, pgtable_alloc, flags); else @@ -443,10 +454,15 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, static phys_addr_t __pgd_pgtable_alloc(int shift) { - /* Page is zeroed by init_clear_pgtable() so don't duplicate effort. */ - void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL & ~__GFP_ZERO); - + #ifdef CONFIG_PTP + void *ptr = iee_cache_alloc(&pg_cache, GFP_PGTABLE_KERNEL); + #else + void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); + #endif BUG_ON(!ptr); + + /* Ensure the zeroed page is visible to the page table walker */ + dsb(ishst); return __pa(ptr); } @@ -484,8 +500,13 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } + #ifdef CONFIG_PTP + __create_pgd_mapping_pre_init(init_mm.pgd, phys, virt, size, prot, NULL, + NO_CONT_MAPPINGS); + #else __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, NO_CONT_MAPPINGS); + #endif } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -522,6 +543,14 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, phys_addr_t end, pgprot_t prot, int flags) { +#ifdef CONFIG_PTP + if (haoc_enabled) { + __create_pgd_mapping_pre_init(pgdp, start, __phys_to_virt(start), + end - start, prot, + early_pgtable_alloc, flags); + return; + } +#endif __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, prot, early_pgtable_alloc, flags); } @@ -534,6 +563,12 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), (unsigned long)__init_begin - (unsigned long)_stext, PAGE_KERNEL_RO); + #ifdef CONFIG_PTP + if (haoc_enabled) + update_mapping_prot(__pa_symbol(init_pg_dir), (unsigned long)lm_alias(init_pg_dir), + (unsigned long)init_pg_end - (unsigned long)init_pg_dir, + PAGE_KERNEL_RO); + #endif } #ifdef CONFIG_KFENCE @@ -595,6 +630,10 @@ static void __init map_mem(pgd_t *pgdp) static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_stext); phys_addr_t kernel_end = __pa_symbol(__init_begin); +#ifdef CONFIG_PTP + phys_addr_t init_pgtable_start = __pa_symbol(init_pg_dir); + phys_addr_t init_pgtable_end = __pa_symbol(init_pg_end); +#endif phys_addr_t start, end; phys_addr_t early_kfence_pool; int flags = NO_EXEC_MAPPINGS; @@ -621,6 +660,9 @@ static void __init map_mem(pgd_t *pgdp) * the following for-loop */ memblock_mark_nomap(kernel_start, kernel_end - kernel_start); + #ifdef CONFIG_PTP + memblock_mark_nomap(init_pgtable_start, init_pgtable_end - init_pgtable_start); + #endif /* map all the memory banks */ for_each_mem_range(i, &start, &end) { @@ -648,6 +690,11 @@ static void __init map_mem(pgd_t *pgdp) __map_memblock(pgdp, kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); + #ifdef CONFIG_PTP + __map_memblock(pgdp, init_pgtable_start, init_pgtable_end, + PAGE_KERNEL, NO_CONT_MAPPINGS); + memblock_clear_nomap(init_pgtable_start, init_pgtable_end - init_pgtable_start); + #endif arm64_kfence_map_pool(early_kfence_pool, pgdp); } @@ -662,6 +709,13 @@ void mark_rodata_ro(void) section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, section_size, PAGE_KERNEL_RO); + #ifdef CONFIG_PTP + if (haoc_enabled) { + section_size = (unsigned long)init_pg_end - (unsigned long)init_pg_dir; + update_mapping_prot(__pa_symbol(init_pg_dir), (unsigned long)init_pg_dir, + section_size, PAGE_KERNEL_RO); + } + #endif debug_checkwx(); } @@ -708,7 +762,14 @@ static int __init map_entry_trampoline(void) pgprot_val(prot) &= ~PTE_NG; /* Map only the text into the trampoline page table */ + #ifdef CONFIG_PTP + if (haoc_enabled) + iee_memset(__va(__pa_symbol(tramp_pg_dir)), 0, PGD_SIZE); + else + memset(tramp_pg_dir, 0, PGD_SIZE); + #else memset(tramp_pg_dir, 0, PGD_SIZE); + #endif __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, entry_tramp_text_size(), prot, __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); @@ -807,7 +868,8 @@ static void __init map_kernel(pgd_t *pgdp) // 只有当IEE数据段非空时才创建分离的映射 map_kernel_segment(pgdp, _data, iee_init_data_end, PAGE_KERNEL, &vmlinux_iee_init_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); - map_kernel_segment(pgdp, iee_init_data_end, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); + map_kernel_segment(pgdp, iee_init_data_end, _end, PAGE_KERNEL, + &vmlinux_data, NO_CONT_MAPPINGS, 0); } else { pr_info("IEE: No IEE init data, using standard data mapping\n"); map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); @@ -879,8 +941,11 @@ void __init paging_init(void) init_early_iee_data(); #endif - memblock_phys_free(__pa_symbol(init_pg_dir), - __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); + #ifdef CONFIG_IEE + if (!haoc_enabled) + #endif + memblock_phys_free(__pa_symbol(init_pg_dir), + __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); memblock_allow_resize(); @@ -905,7 +970,11 @@ static void free_hotplug_page_range(struct page *page, size_t size, static void free_hotplug_pgtable_page(struct page *page) { +#ifdef CONFIG_PTP + pagetable_free(virt_to_ptdesc(page_to_virt(page))); +#else free_hotplug_page_range(page, PAGE_SIZE, NULL); +#endif } static bool pgtable_range_aligned(unsigned long start, unsigned long end, diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index 4a64089e5771c..356eae166d31d 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -15,29 +15,55 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + static struct kmem_cache *pgd_cache __ro_after_init; +static bool pgdir_is_page_size(void) +{ + if (PGD_SIZE == PAGE_SIZE) + return true; + if (CONFIG_PGTABLE_LEVELS == 4) + return !pgtable_l4_enabled(); + if (CONFIG_PGTABLE_LEVELS == 5) + return !pgtable_l5_enabled(); + return false; +} + pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; - if (PGD_SIZE == PAGE_SIZE) + if (pgdir_is_page_size()) { +#ifdef CONFIG_PTP + return (pgd_t *)iee_cache_alloc(&pg_cache, gfp); +#else return (pgd_t *)__get_free_page(gfp); - else - return kmem_cache_alloc(pgd_cache, gfp); +#endif + } + + return kmem_cache_alloc(pgd_cache, gfp); } void pgd_free(struct mm_struct *mm, pgd_t *pgd) { - if (PGD_SIZE == PAGE_SIZE) + if (pgdir_is_page_size()) { +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pgd); +#else free_page((unsigned long)pgd); - else - kmem_cache_free(pgd_cache, pgd); +#endif + return; + } + + kmem_cache_free(pgd_cache, pgd); } void __init pgtable_cache_init(void) { - if (PGD_SIZE == PAGE_SIZE) + if (pgdir_is_page_size()) return; #ifdef CONFIG_ARM64_PA_BITS_52 diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c index 5139a28130c08..c7a356adb51ba 100644 --- a/arch/arm64/mm/trans_pgd.c +++ b/arch/arm64/mm/trans_pgd.c @@ -252,7 +252,11 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, level_mask = GENMASK_ULL(level_msb, level_lsb); index = (dst_addr & level_mask) >> level_lsb; + #ifdef CONFIG_PTP + set_pte((pte_t *)(levels[this_level] + index), __pte(prev_level_entry)); + #else *(levels[this_level] + index) = prev_level_entry; + #endif pfn = virt_to_pfn(levels[this_level]); prev_level_entry = pte_val(pfn_pte(pfn, diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 91685de954096..47a7a68927973 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1556,14 +1556,36 @@ config IEE Could isolate kernel critical data and enforce all write access made and verified in IEE APIs. +config PTP + bool "Page Table Protection(PTP)" + depends on IEE + help + Protect page tables by routing page-table modification paths + through the IEE gate. This keeps page table updates mediated by + the isolated execution environment instead of allowing direct + writes from normal kernel code. + +config PTP_RESERVE_ORDER + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + depends on PTP + range 9 15 + config IEE_PTRP bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE help - Provide IEE matadata for each process called task_token to allow - verifing pointers inside task_struct, like struct cred* that determines - the capabilities of a process. - Could be a enhancement of other sub-module of HAOC. + Enable task_token pointer protection for IEE. When the write-only + sub-option IEE_PTRP_W is disabled, both read and write checks are + active. When IEE_PTRP_W is enabled, only write protection is active. + +config IEE_PTRP_W + bool "Pointer write-only protection (IEE_PTRP_W)" + depends on IEE_PTRP + help + When enabled, only write protection is performed, without read-side + verification. Disable this option to enable full read and write checks + under IEE_PTRP. This mode keeps pointer writes mediated by IEE while + avoiding read-side verification overhead. config IEE_SIP bool "Sensitive Instruction Protection for IEE(IEE_SIP)" @@ -1572,7 +1594,6 @@ config IEE_SIP Protects critical instructions that may break the isolation of IEE, such as writing system control registers. These instructions would be executated inside IEE. - config CREDP bool "Struct cred protection(CREDP)" depends on IEE @@ -1582,6 +1603,50 @@ config CREDP by IEE either. If unsure, say N. +config KEYP + bool "Struct Key protection(KEYP)" + depends on IEE + help + This option enables protection for the kernel's struct key. All modifications + of keys must be made and verified by IEE APIs to defend memory corruptions. + + If unsure, say N. + +config IEE_SELINUX_P + bool "SELinux Protection(IEE_SELINUX_P)" + help + Simple enhancement of the famous linux security module SELinux by putting + several critical data, like selinux policy, into the IEE region. + + If unsure, say N. + depends on SECURITY_SELINUX + depends on IEE + +config PTP_S + bool "Page Table Protection - Separation (PTP_S)" + depends on PTP + help + PTP_S (kernel/user page separation). Split PTP write paths so user page + tables can use direct writes while kernel tables stay on ptp gate paths, + with user-side safety checks for alias/integrity. + Enable this option when user page-table writes should avoid the PTP + gate while kernel page-table writes remain protected by PTP. + +config VARP + bool "variable protection for IEE" + depends on IEE + help + Protect variables with IEE. At present, only modprobe_path is + covered. Writes to protected variables are routed through IEE + interfaces instead of direct kernel stores. + Enable this option when modprobe_path must be protected by the + isolated execution environment. + +config PTP_RESERVE_ORDER + depends on PTP + int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" + range 9 15 + # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 848a958c639f8..1494e396768af 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -23,7 +23,11 @@ /* Use the static base for this part of the boot process */ #undef __PAGE_OFFSET #define __PAGE_OFFSET __PAGE_OFFSET_BASE +#ifdef CONFIG_PTP +#include "../../mm/ptp_ident_map.c" +#else #include "../../mm/ident_map.c" +#endif #define _SETUP #include /* For COMMAND_LINE_SIZE */ @@ -101,9 +105,15 @@ void kernel_add_identity_map(unsigned long start, unsigned long end) return; /* Build the mapping. */ +#ifdef CONFIG_PTP + ret = ptp_kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); + if (ret) + error("Error: kernel_ident_mapping_init_for_iee() failed\n"); +#else ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); if (ret) error("Error: kernel_ident_mapping_init() failed\n"); +#endif } /* Locates and clears a region for a new top level page table. */ @@ -180,14 +190,7 @@ void initialize_identity_maps(void *rmode) sev_prep_identity_maps(top_level_pgt); /* Load the new page-table. */ - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early(top_level_pgt); - else - write_cr3(top_level_pgt); - #else write_cr3(top_level_pgt); - #endif /* * Now that the required page table mappings are established and a @@ -215,7 +218,11 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, /* Populate the PTEs */ for (i = 0; i < PTRS_PER_PMD; i++) { +#ifdef CONFIG_PTP + ptp_set_pte_pre_init(&pte[i], __pte(address | page_flags)); +#else set_pte(&pte[i], __pte(address | page_flags)); +#endif address += PAGE_SIZE; } @@ -229,16 +236,13 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, * of a TLB multihit. */ pmd = __pmd((unsigned long)pte | info->kernpg_flag); +#ifdef CONFIG_PTP + ptp_set_pmd_pre_init(pmdp, pmd); +#else set_pmd(pmdp, pmd); + #endif /* Flush TLB to establish the new PMD */ - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early(top_level_pgt); - else - write_cr3(top_level_pgt); - #else write_cr3(top_level_pgt); - #endif return pte + pte_index(__address); } @@ -328,7 +332,11 @@ static int set_clr_page_flags(struct x86_mapping_info *info, pte = *ptep; pte = pte_set_flags(pte, set); pte = pte_clear_flags(pte, clr); +#ifdef CONFIG_PTP + ptp_set_pte_pre_init(ptep, pte); +#else set_pte(ptep, pte); +#endif /* * If the encryption attribute is being set, then change the page state to @@ -339,14 +347,7 @@ static int set_clr_page_flags(struct x86_mapping_info *info, snp_set_page_private(__pa(address & PAGE_MASK)); /* Flush TLB after changing encryption attribute */ - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early(top_level_pgt); - else - write_cr3(top_level_pgt); - #else write_cr3(top_level_pgt); - #endif return 0; } diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index ab495a0b16f22..dc21b0d89d667 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -200,14 +200,7 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) * Move the top level page table out of trampoline memory. */ memcpy(pgtable, trampoline_32bit, PAGE_SIZE); - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_write_cr3_early((unsigned long)pgtable); - else - native_write_cr3((unsigned long)pgtable); - #else native_write_cr3((unsigned long)pgtable); - #endif /* Restore trampoline memory */ memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 81f8e1d106164..d60bb6a82a98e 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -89,9 +89,16 @@ CONFIG_X86_MSR=m CONFIG_X86_CPUID=m CONFIG_AMD_MEM_ENCRYPT=y CONFIG_IEE=y +CONFIG_PTP=y +CONFIG_PTP_RESERVE_ORDER=12 CONFIG_IEE_PTRP=y +CONFIG_IEE_PTRP_W=y CONFIG_IEE_SIP=y CONFIG_CREDP=y +CONFIG_KEYP=y +CONFIG_IEE_SELINUX_P=y +CONFIG_PTP_S=y +CONFIG_VARP=y CONFIG_NUMA=y CONFIG_ARCH_MEMORY_PROBE=y CONFIG_X86_PMEM_LEGACY=m diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index 4a90cf3c78dc7..94a8674872fd8 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -13,7 +13,7 @@ #include #include -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) #include extern bool haoc_enabled; #endif @@ -215,7 +215,7 @@ static inline void native_load_gdt(const struct desc_ptr *dtr) asm volatile("lgdt %0"::"m" (*dtr)); } -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) static __always_inline void iee_load_idt_early(const struct desc_ptr *dtr) { asm volatile("lidt %0"::"m" (*dtr)); @@ -224,7 +224,7 @@ static __always_inline void iee_load_idt_early(const struct desc_ptr *dtr) static __always_inline void native_load_idt(const struct desc_ptr *dtr) { - #ifdef CONFIG_IEE_SIP + #if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) if(haoc_enabled) iee_load_idt((void *)dtr); else @@ -261,7 +261,7 @@ static inline void native_idt_invalidate(void) .size = 0 }; -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) // The native_idt_invalidate() is only called by machine_kexec(). // In the kdump path, IEE_SIP should not be used, so we directly // call iee_load_idt_early(), which is the original version of diff --git a/arch/x86/include/asm/haoc/haoc-bitmap.h b/arch/x86/include/asm/haoc/haoc-bitmap.h new file mode 100644 index 0000000000000..9d6090b20a8b2 --- /dev/null +++ b/arch/x86/include/asm/haoc/haoc-bitmap.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HAOC_BITMAP_H +#define _LINUX_HAOC_BITMAP_H + +#include +#include +#include +extern unsigned long iee_bitmap_base; +#define __VIEEBITMAP_BASE_L4 0xffffeb0000000000UL +#define __VIEEBITMAP_BASE_L5 0xffd6000000000000UL +#define __pfn_to_iee_bitmap(pfn) ((uint8_t *)iee_bitmap_base + (pfn)) + +enum HAOC_BITMAP_TYPE { + IEE_NORMAL = 0, + IEE_PGTABLE, + IEE_USER_PGTABLE, // modifying IEE_USER_PGTABLE needn't use ptp_rw_gate + IEE_FIXED_PGTABLE, + IEE_POLICY, + IEE_KEY, + IEE_SIP_DATA, + IEE_BITMAP, + IEE_VARP, + IEE_CRED, +}; + +extern void iee_init_bitmap(void); +#define IEE_PHYS_BIT_PTR(phys_addr) ((uint8_t *)((uint8_t *)iee_bitmap_base + PHYS_PFN(phys_addr))) + +extern unsigned long long iee_rw_gate(int flag, ...); + +static inline void iee_set_bitmap_type(phys_addr_t phys_addr, int num_pages, + enum HAOC_BITMAP_TYPE type) +{ + uint8_t *bitmap_ptr = IEE_PHYS_BIT_PTR(phys_addr); + + iee_rw_gate(IEE_OP_SET_BITMAP_TYPE, bitmap_ptr, type, num_pages); +} + +static inline enum HAOC_BITMAP_TYPE iee_get_bitmap_type(phys_addr_t phys_addr) +{ + return *IEE_PHYS_BIT_PTR(phys_addr); +} + +static inline void iee_verify_type(phys_addr_t phys_addr, + enum HAOC_BITMAP_TYPE type, + const char *name) +{ + uint8_t bit_type; + + if (unlikely(!haoc_init_done)) + return; + + bit_type = iee_get_bitmap_type(phys_addr); + + if (unlikely(bit_type != type)) + panic("IEE detected type: %d, fake %s: va(0x%lx)", + bit_type, name, (unsigned long)__va(phys_addr)); +} + +#endif diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index f8139664ca0be..45fee9457444e 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -20,12 +20,16 @@ enum { IEE_OP_INVALIDATE_TOKEN, IEE_OP_VALIDATE_TOKEN, #endif + IEE_OP_SET_BITMAP_TYPE, #ifdef CONFIG_CREDP IEE_OP_COPY_CRED, IEE_OP_COPY_KERNEL_CRED, IEE_OP_INIT_COPIED_CRED, IEE_OP_COMMIT_CRED, IEE_OP_ABORT_CRED, + IEE_OP_FILL_SESSION_KEYRING_CRED, + IEE_OP_OVERRIDE_CRED, + IEE_OP_REVERT_CRED, IEE_OP_SET_CRED_UID, IEE_OP_SET_CRED_GID, IEE_OP_SET_CRED_SUID, @@ -54,6 +58,60 @@ enum { IEE_OP_SET_CRED_SECURITY, IEE_OP_SET_CRED_RCU, IEE_OP_SET_CRED_UCOUNTS, +#endif +#ifdef CONFIG_KEYP + IEE_OP_SET_KEY_UNION, + IEE_OP_SET_KEY_STRUCT, + IEE_OP_SET_KEY_PAYLOAD, + IEE_OP_SET_KEY_USAGE, + IEE_OP_SET_KEY_SERIAL, + IEE_OP_SET_KEY_WATCHERS, + IEE_OP_SET_KEY_USERS, + IEE_OP_SET_KEY_SECURITY, + IEE_OP_SET_KEY_EXPIRY, + IEE_OP_SET_KEY_REVOKED_AT, + IEE_OP_SET_KEY_LAST_USED_AT, + IEE_OP_SET_KEY_UID, + IEE_OP_SET_KEY_GID, + IEE_OP_SET_KEY_PERM, + IEE_OP_SET_KEY_QUOTALEN, + IEE_OP_SET_KEY_DATALEN, + IEE_OP_SET_KEY_STATE, + IEE_OP_SET_KEY_MAGIC, + IEE_OP_SET_KEY_FLAGS, + IEE_OP_SET_KEY_INDEX_KEY, + IEE_OP_SET_KEY_HASH, + IEE_OP_SET_KEY_LEN_DESC, + IEE_OP_SET_KEY_TYPE, + IEE_OP_SET_KEY_TAG, + IEE_OP_SET_KEY_DESCRIPTION, + IEE_OP_SET_KEY_RESTRICT_LINK, + IEE_OP_SET_KEY_FLAG_BIT, +#endif +#ifdef CONFIG_IEE_SELINUX_P + IEE_SEL_SET_STATUS_PG, + IEE_SEL_SET_ENFORCING, + IEE_SEL_SET_INITIALIZED, + IEE_SEL_SET_POLICY_CAP, + IEE_SEL_RCU_ASSIGN_POLICY, +#endif +#ifdef CONFIG_PTP +#ifdef CONFIG_IEE_PTRP + IEE_OP_UNSET_TOKEN, + IEE_OP_SET_TOKEN, +#endif + IEE_OP_SET_PTE, + IEE_OP_SET_PMD, + IEE_OP_SET_PUD, + IEE_OP_SET_P4D, + IEE_OP_SET_PGD, + IEE_OP_SET_PTE_TEXT_POKE, + IEE_OP_SET_PMD_IDENT, + IEE_OP_SET_PUD_IDENT, + IEE_OP_SET_SX_PTE, +#endif +#ifdef CONFIG_VARP + IEE_OP_SET_VARP_MODPROBE_PATH, #endif IEE_FLAG_END }; @@ -71,4 +129,15 @@ enum { #define AT_INC_NOT_ZERO 2 #define AT_SUB_AND_TEST 3 #endif + +#ifdef CONFIG_KEYP +#define REFCOUNT_INC 1 +#define REFCOUNT_SET 2 +#define REFCOUNT_DEC_AND_TEST 3 +#define REFCOUNT_INC_NOT_ZERO 4 + +#define SET_BIT_OP 1 +#define TEST_AND_CLEAR_BIT 2 +#define TEST_AND_SET_BIT 3 +#endif #endif diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index ef665161259de..55c1cb0f79b05 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -10,6 +10,7 @@ #ifndef _LINUX_HAOC_H #define _LINUX_HAOC_H +#include #include #include @@ -24,6 +25,36 @@ void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); #endif +void _iee_set_bitmap_type(unsigned long __unused, uint8_t *bitmap_ptr, + enum HAOC_BITMAP_TYPE type, int num_pages); + +#ifdef CONFIG_IEE_SELINUX_P +#include +void _iee_set_selinux_status_pg(unsigned long iee_offset, struct page *new_page); +void _iee_set_selinux_enforcing(unsigned long iee_offset, bool value); +void _iee_mark_selinux_initialized(unsigned long iee_offset); +void _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap); +void _iee_sel_rcu_assign_policy(unsigned long iee_offset, + struct selinux_policy *new_policy, struct selinux_policy *iee_new_policy); +#endif + +#ifdef CONFIG_PTP +#ifdef CONFIG_IEE_PTRP +void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token, unsigned int order); +void _iee_set_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token_page, unsigned int order); +#endif +void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte); +void _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd); +void _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud); +void _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d); +void _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd); +void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte); +void _iee_set_pmd_ident(unsigned long __unused, pmd_t *pmdp, pmd_t pmd); +void _iee_set_pud_ident(unsigned long __unused, pud_t *pudp, pud_t pud); +void _iee_set_sx_pte(unsigned long __unused, pte_t *ptep, pte_t pte); +#endif #ifdef CONFIG_CREDP #include @@ -31,9 +62,13 @@ void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); void _iee_copy_cred(unsigned long __unused, struct cred *new); void _iee_copy_kernel_cred(unsigned long iee_offset, const struct cred *old, struct cred *new); void _iee_abort_cred(unsigned long iee_offset, const struct cred *cred); +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old); void _iee_init_copied_cred(unsigned long iee_offset, struct task_struct *new_task, struct cred *new); void _iee_commit_creds(unsigned long iee_offset, const struct cred *new); +void _iee_override_creds(unsigned long iee_offset, const struct cred *new); +void _iee_revert_creds(unsigned long iee_offset, const struct cred *old); void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); @@ -75,4 +110,53 @@ void _iee_set_cred_security(unsigned long __unused, struct cred *cred,void *secu void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu); void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); #endif + +#ifdef CONFIG_VARP +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, int maxlen, size_t len, + char *buffer, size_t *lenp); +#endif + +#ifdef CONFIG_VARP +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, int maxlen, size_t len, + char *buffer, size_t *lenp); +#endif + +#ifdef CONFIG_KEYP +#include +struct watch_list; + +void _iee_set_key_union(unsigned long __unused, struct key *key, struct key_union *key_union); +void _iee_set_key_struct(unsigned long __unused, struct key *key, struct key_struct *key_struct); +void _iee_set_key_payload(unsigned long __unused, struct key *key, union key_payload *key_payload); +unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, int n, int flag); +void _iee_set_key_serial(unsigned long __unused, struct key *key, key_serial_t serial); +void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers); +void _iee_set_key_user(unsigned long __unused, struct key *key, struct key_user *user); +void _iee_set_key_security(unsigned long __unused, struct key *key, void *security); +void _iee_set_key_expiry(unsigned long __unused, struct key *key, time64_t expiry); +void _iee_set_key_revoked_at(unsigned long __unused, struct key *key, time64_t revoked_at); +void _iee_set_key_last_used_at(unsigned long __unused, struct key *key, time64_t last_used_at); +void _iee_set_key_uid(unsigned long __unused, struct key *key, kuid_t uid); +void _iee_set_key_gid(unsigned long __unused, struct key *key, kgid_t gid); +void _iee_set_key_perm(unsigned long __unused, struct key *key, key_perm_t perm); +void _iee_set_key_quotalen(unsigned long __unused, struct key *key, unsigned short quotalen); +void _iee_set_key_datalen(unsigned long __unused, struct key *key, unsigned short datalen); +void _iee_set_key_state(unsigned long __unused, struct key *key, short state); +void _iee_set_key_magic(unsigned long __unused, struct key *key, unsigned int magic); +void _iee_set_key_flags(unsigned long __unused, struct key *key, unsigned long flags); +void _iee_set_key_index_key(unsigned long __unused, struct key *key, + struct keyring_index_key *index_key); +void _iee_set_key_hash(unsigned long __unused, struct key *key, unsigned long hash); +void _iee_set_key_len_desc(unsigned long __unused, struct key *key, unsigned long len_desc); +void _iee_set_key_type(unsigned long __unused, struct key *key, struct key_type *type); +void _iee_set_key_domain_tag(unsigned long __unused, struct key *key, struct key_tag *domain_tag); +void _iee_set_key_description(unsigned long __unused, struct key *key, char *description); +void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, + struct key_restriction *restrict_link); +unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); +#endif + +unsigned long iee_dispatch(unsigned long flag, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5); #endif diff --git a/arch/x86/include/asm/haoc/iee-cred.h b/arch/x86/include/asm/haoc/iee-cred.h index 58b4971d711b3..1bb1b84de4c33 100644 --- a/arch/x86/include/asm/haoc/iee-cred.h +++ b/arch/x86/include/asm/haoc/iee-cred.h @@ -2,11 +2,73 @@ #ifndef _LINUX_IEE_CRED_H #define _LINUX_IEE_CRED_H -#include #include +#include +#include +#include +#ifdef CONFIG_IEE_PTRP +#include +#endif extern unsigned long long iee_rw_gate(int flag, ...); +static inline void iee_detect_cred_uaf(struct cred *cred, const char *caller) +{ + if (unlikely(!haoc_init_done)) + return; + + if (atomic_long_read(&cred->usage) != 0) + panic("IEE: (%s) Cred usage %ld.", caller, + atomic_long_read(&cred->usage)); +} + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +static inline void __iee_verify_cred(void) +{ + struct task_struct *curr_task; + struct task_token *token; + const struct cred *curr_cred; + + if (unlikely(!haoc_init_done)) + return; + + curr_task = current; + if (curr_task == &init_task) + return; + + token = (struct task_token *)__addr_to_token(curr_task); + curr_cred = current_cred(); + if (!token->valid) + panic("IEE: (%s) Invalid Token.", __func__); + if (token->curr_cred != curr_cred) + panic("IEE: (%s) Task cred corrupted.", __func__); +} + +static inline void iee_verify_cred(void) +{ + __iee_verify_cred(); +} + +static inline void iee_verify_update_cred(struct cred *new) +{ + struct task_token *token; + + if (unlikely(!haoc_init_done)) + return; + + __iee_verify_cred(); + if (current == &init_task) + return; + + token = (struct task_token *)__addr_to_token(current); + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); +} +#else /* !CONFIG_IEE_PTRP || CONFIG_IEE_PTRP_W */ +static inline void iee_verify_cred(void) { } +static inline void iee_verify_update_cred(struct cred *new) { } +#endif /* CONFIG_IEE_PTRP && !CONFIG_IEE_PTRP_W */ + static void __maybe_unused iee_copy_cred(struct cred *new) { iee_rw_gate(IEE_OP_COPY_CRED, new); @@ -29,11 +91,27 @@ static void __maybe_unused iee_abort_creds(struct cred *cred) iee_rw_gate(IEE_OP_ABORT_CRED, cred); } +static void __maybe_unused iee_fill_cred_for_session_keyring(struct cred *new, + const struct cred *old) +{ + iee_rw_gate(IEE_OP_FILL_SESSION_KEYRING_CRED, new, old); +} + static void __maybe_unused iee_commit_creds(const struct cred *new) { iee_rw_gate(IEE_OP_COMMIT_CRED, new); } +static void __maybe_unused iee_override_creds(const struct cred *new) +{ + iee_rw_gate(IEE_OP_OVERRIDE_CRED, new); +} + +static void __maybe_unused iee_revert_creds(const struct cred *old) +{ + iee_rw_gate(IEE_OP_REVERT_CRED, old); +} + static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) { if(!haoc_enabled) @@ -299,4 +377,9 @@ static void __maybe_unused iee_set_cred_security(struct cred *cred, void *securi } #endif -#endif \ No newline at end of file +extern void iee_free_cred_slab(struct work_struct *work); + +extern unsigned long __iee_cred_data_start[]; +extern unsigned long __iee_cred_data_end[]; + +#endif diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h index ffad94ecb8b63..791423c74d45c 100644 --- a/arch/x86/include/asm/haoc/iee-func.h +++ b/arch/x86/include/asm/haoc/iee-func.h @@ -12,9 +12,14 @@ #define HUGE_PMD_ORDER 9 +#include #include -extern void set_iee_page(unsigned long addr, unsigned int order); -extern void unset_iee_page(unsigned long addr, unsigned int order); +extern void set_iee_pages(unsigned long addr, int num_pages, + enum HAOC_BITMAP_TYPE type); +extern void unset_iee_pages(unsigned long addr, int num_pages); +extern void set_iee_page(unsigned long addr, int num_pages, + enum HAOC_BITMAP_TYPE type); +extern void unset_iee_page(unsigned long addr, int num_pages); extern bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order); extern unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order); diff --git a/arch/x86/include/asm/haoc/iee-key.h b/arch/x86/include/asm/haoc/iee-key.h new file mode 100644 index 0000000000000..86d499d9aff64 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-key.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_KEY_H +#define _LINUX_IEE_KEY_H + +#include +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_set_key_union(struct key *key, struct key_union *key_union) +{ + iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); +} + +static void __maybe_unused iee_set_key_struct(struct key *key, struct key_struct *key_struct) +{ + iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); +} + +static void __maybe_unused iee_set_key_payload(struct key *key, union key_payload *key_payload) +{ + iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); +} + +extern bool iee_set_key_usage(struct key *key, int n, int flag); + +static void __maybe_unused iee_set_key_serial(struct key *key, key_serial_t serial) +{ + iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); +} + +#ifdef CONFIG_KEY_NOTIFICATIONS +static void __maybe_unused iee_set_key_watchers(struct key *key, struct watch_list *watchers) +{ + iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); +} +#endif + +static void __maybe_unused iee_set_key_user(struct key *key, struct key_user *user) +{ + iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); +} + +static void __maybe_unused iee_set_key_security(struct key *key, void *security) +{ + iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); +} + +static void __maybe_unused iee_set_key_expiry(struct key *key, time64_t expiry) +{ + iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); +} + +static void __maybe_unused iee_set_key_revoked_at(struct key *key, time64_t revoked_at) +{ + iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); +} + +static void __maybe_unused iee_set_key_last_used_at(struct key *key, time64_t last_used_at) +{ + iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); +} + +static void __maybe_unused iee_set_key_uid(struct key *key, kuid_t uid) +{ + iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); +} + +static void __maybe_unused iee_set_key_gid(struct key *key, kgid_t gid) +{ + iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); +} + +static void __maybe_unused iee_set_key_perm(struct key *key, key_perm_t perm) +{ + iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); +} + +static void __maybe_unused iee_set_key_quotalen(struct key *key, unsigned short quotalen) +{ + iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); +} + +static void __maybe_unused iee_set_key_datalen(struct key *key, unsigned short datalen) +{ + iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); +} + +static void __maybe_unused iee_set_key_state(struct key *key, short state) +{ + iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); +} + +#ifdef KEY_DEBUGGING +static void __maybe_unused iee_set_key_magic(struct key *key, unsigned int magic) +{ + iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); +} +#endif + +static void __maybe_unused iee_set_key_flags(struct key *key, unsigned long flags) +{ + iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); +} + +static void __maybe_unused iee_set_key_index_key(struct key *key, + struct keyring_index_key *index_key) +{ + iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); +} + +static void __maybe_unused iee_set_key_hash(struct key *key, unsigned long hash) +{ + iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); +} + +static void __maybe_unused iee_set_key_len_desc(struct key *key, unsigned long len_desc) +{ + iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); +} + +static void __maybe_unused iee_set_key_type(struct key *key, struct key_type *type) +{ + iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); +} + +static void __maybe_unused iee_set_key_domain_tag(struct key *key, struct key_tag *domain_tag) +{ + iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); +} + +static void __maybe_unused iee_set_key_description(struct key *key, char *description) +{ + iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); +} + +static void __maybe_unused iee_set_key_restrict_link(struct key *key, + struct key_restriction *restrict_link) +{ + iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); +} + +static bool __maybe_unused iee_set_key_flag_bit(struct key *key, long nr, int flag) +{ + bool ret; + + ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); + return ret; +} + +#endif diff --git a/arch/x86/include/asm/haoc/iee-selinux.h b/arch/x86/include/asm/haoc/iee-selinux.h new file mode 100644 index 0000000000000..413802b7c1273 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-selinux.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_SELINUX_P_H +#define _LINUX_IEE_SELINUX_P_H + +#include +#include +#include "security.h" +#include "ss/services.h" + +extern struct kmem_cache *policy_jar; +extern unsigned long long iee_rw_gate(int flag, ...); +extern unsigned long __iee_selinux_data_start[]; +extern unsigned long __iee_selinux_data_end[]; + +static inline struct mutex *iee_get_selinux_policy_lock(void) +{ + return (struct mutex *)(selinux_state.policy_mutex.owner.counter); +} + +static inline struct mutex *iee_get_selinux_status_lock(void) +{ + return (struct mutex *)(selinux_state.status_lock.owner.counter); +} + +static inline void iee_set_selinux_status_pg(struct page *new_page) +{ + iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); +} + +static inline void enforcing_set(bool value) +{ + iee_rw_gate(IEE_SEL_SET_ENFORCING, value); +} + +static inline void selinux_mark_initialized(void) +{ + iee_rw_gate(IEE_SEL_SET_INITIALIZED); +} + +static inline void iee_set_sel_policy_cap(unsigned int idx, int cap) +{ + iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); +} + +/* + * Please make sure param iee_new_policy is from policy_jar memcache. + * Need to free new_policy after calling this func as it's only used to + * trans data from kernel. + */ +static inline void iee_sel_rcu_assign_policy(struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); +} + +#endif diff --git a/arch/x86/include/asm/haoc/iee-si.h b/arch/x86/include/asm/haoc/iee-si.h index 3baf1813f19d3..b20966778c6cd 100644 --- a/arch/x86/include/asm/haoc/iee-si.h +++ b/arch/x86/include/asm/haoc/iee-si.h @@ -7,7 +7,7 @@ #define __iee_si_code __section(".iee.si_text") #define __iee_si_data __section(".iee.si_data") -extern unsigned long cr4_pinned_mask; +extern const unsigned long cr4_pinned_mask; extern struct static_key_false cr_pinning; extern unsigned long cr4_pinned_bits; @@ -39,5 +39,11 @@ static inline void iee_load_idt(void *ptr) { iee_rwx_gate(IEE_LOAD_IDT, ptr); } +extern bool iee_init_done; -#endif \ No newline at end of file +#ifdef CONFIG_IEE_SELINUX_P +extern unsigned long __iee_selinux_data_start[]; +extern unsigned long __iee_selinux_data_end[]; +#endif + +#endif diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h index 53e75396577d3..21dc25f816a02 100644 --- a/arch/x86/include/asm/haoc/iee-token.h +++ b/arch/x86/include/asm/haoc/iee-token.h @@ -3,15 +3,20 @@ #define _LINUX_IEE_TOKEN_H #include #include +#include +#include #include #include +#include extern unsigned long long iee_rw_gate(int flag, ...); +#ifdef CONFIG_IEE_PTRP extern struct kmem_cache *task_struct_cachep; +extern void __init iee_prepare_init_task_token(void); extern void iee_set_token_page_valid(unsigned long token, unsigned long new, - unsigned int order); + unsigned int order); extern void iee_set_token_page_invalid(unsigned long token_addr, unsigned long __unused, unsigned int order); @@ -19,6 +24,16 @@ extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, unsigned int order); +#ifndef IEE_TOKEN_BLOCK_SIZE +#define IEE_TOKEN_BLOCK_SIZE 64 +#endif + +#ifndef IEE_TOKEN_ORDER +#define IEE_TOKEN_ORDER(task_order) \ + order_base_2(((1U << (task_order)) * IEE_TOKEN_BLOCK_SIZE) / PAGE_SIZE) +#endif +#endif /* CONFIG_IEE_PTRP */ + #ifdef CONFIG_IEE struct task_token { pgd_t *pgd; /* Logical VA */ @@ -27,24 +42,16 @@ struct task_token { bool valid; void *kernel_stack; /* VA */ #ifdef CONFIG_CREDP - struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *new_cred; /* The valid target for commit_creds. */ + const struct cred *curr_cred; /* The current subjective credentials. */ +#endif +#ifdef CONFIG_IEE_PTRP + seqcount_t seq; #endif }; #endif /* CONFIG_IEE */ -static inline void iee_verify_token_pgd(struct task_struct *tsk) -{ - struct task_token *token; - - if (tsk == &init_task) - return; - - token = (struct task_token *)__addr_to_iee(tsk); - if (token->pgd != tsk->mm->pgd) - panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", - (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); -} - +#ifdef CONFIG_IEE_PTRP static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) { iee_rw_gate(IEE_OP_SET_TOKEN_PGD, tsk, pgd); @@ -60,4 +67,11 @@ static inline void iee_validate_token(struct task_struct *tsk) iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); } -#endif \ No newline at end of file +#if !defined(CONFIG_IEE_PTRP_W) +void iee_verify_pgd(struct task_struct *next); +void iee_verify_token(struct task_struct *tsk); +#endif + +#endif /* CONFIG_IEE_PTRP */ + +#endif /* _LINUX_IEE_TOKEN_H */ diff --git a/arch/x86/include/asm/haoc/iee-varp.h b/arch/x86/include/asm/haoc/iee-varp.h new file mode 100644 index 0000000000000..2c5566d2c3755 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-varp.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_VARP_H +#define _LINUX_IEE_VARP_H + +#include + +#ifdef CONFIG_VARP +#define __iee_varp_data __section(".iee.varp") +#endif + +extern unsigned long long iee_rw_gate(int flag, ...); + +extern unsigned long __iee_varp_data_start[]; +extern unsigned long __iee_varp_data_end[]; + +static void __maybe_unused iee_set_varp_modprobe_path(char *data, int maxlen, + size_t len, char *buffer, + size_t *lenp) +{ + iee_rw_gate(IEE_OP_SET_VARP_MODPROBE_PATH, data, maxlen, len, buffer, + lenp); +} +#endif diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h index bb9bb11875006..25cfba69d4c3f 100644 --- a/arch/x86/include/asm/haoc/iee.h +++ b/arch/x86/include/asm/haoc/iee.h @@ -13,6 +13,8 @@ #include extern unsigned long IEE_OFFSET; +extern bool haoc_enabled; +extern bool haoc_init_done; #define __iee_pa(x) (__pa(x - IEE_OFFSET)) #define __phys_to_iee(x) ((void *)(__va(x) + IEE_OFFSET)) #define __page_to_phys(x) (page_to_pfn(x) << PAGE_SHIFT) @@ -38,6 +40,9 @@ extern unsigned long IEE_OFFSET; __val = (u64)__kimg_to_iee((u64)x); \ __val; \ }) +#ifdef CONFIG_IEE_PTRP +#define __addr_to_token(x) iee_get_task_token(x) +#endif #define IEE_DATA_ORDER (PMD_SHIFT - PAGE_SHIFT) #define IEE_STACK_ORDER 0 @@ -48,7 +53,13 @@ struct iee_stack { DECLARE_PER_CPU(struct iee_stack, iee_stacks); extern void *alloc_low_pages(unsigned int num); +extern void iee_early_init(void); extern void iee_init(void); -extern bool haoc_enabled; +extern void iee_post_init(void); extern bool iee_init_done; +#ifdef CONFIG_IEE_PTRP +struct task_struct; +struct task_token; +extern struct task_token *iee_get_task_token(struct task_struct *task); +#endif #endif diff --git a/arch/x86/include/asm/haoc/ptp.h b/arch/x86/include/asm/haoc/ptp.h new file mode 100644 index 0000000000000..1f8d51b89166d --- /dev/null +++ b/arch/x86/include/asm/haoc/ptp.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PTP_H +#define _LINUX_PTP_H + +#include +#include +#include +#ifdef CONFIG_PTP_S +#include +#endif + +extern unsigned long long ptp_rw_gate(int flag, ...); + +#ifdef CONFIG_PTP_S +extern bool ptp_is_user_pgtable(const void *ptp); +extern void ptp_user_check_pte_update(pte_t *ptep, pte_t pte); +extern void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd); +extern void ptp_user_check_pud_update(pud_t *pudp, pud_t pud); +#endif + +static inline void ptp_set_pte(pte_t *ptep, pte_t pte) +{ +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(ptep)) { + ptp_user_check_pte_update(ptep, pte); + WRITE_ONCE(*ptep, pte); + return; + } +#endif + ptp_rw_gate(IEE_OP_SET_PTE, ptep, pte); +} + +static inline void ptp_set_pmd(pmd_t *pmdp, pmd_t pmd) +{ +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, pmd); + WRITE_ONCE(*pmdp, pmd); + return; + } +#endif + ptp_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); +} + +static inline void ptp_set_pud(pud_t *pudp, pud_t pud) +{ +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pudp)) { + ptp_user_check_pud_update(pudp, pud); + WRITE_ONCE(*pudp, pud); + return; + } +#endif + ptp_rw_gate(IEE_OP_SET_PUD, pudp, pud); +} + +static inline void ptp_set_p4d(p4d_t *p4dp, p4d_t p4d) +{ +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(p4dp)) { + WRITE_ONCE(*p4dp, p4d); + return; + } +#endif + ptp_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); +} + +static inline void ptp_set_pgd(pgd_t *pgdp, pgd_t pgd) +{ +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pgdp)) { + WRITE_ONCE(*pgdp, pgd); + return; + } +#endif + ptp_rw_gate(IEE_OP_SET_PGD, pgdp, pgd); +} + +static inline void ptp_set_pte_text_poke(pte_t *ptep, pte_t pte) +{ + ptp_rw_gate(IEE_OP_SET_PTE_TEXT_POKE, ptep, pte); +} + +static inline void ptp_set_pmd_ident(pmd_t *pmdp, pmd_t pmd) +{ + ptp_rw_gate(IEE_OP_SET_PMD_IDENT, pmdp, pmd); +} + +static inline void ptp_set_pud_ident(pud_t *pudp, pud_t pud) +{ + ptp_rw_gate(IEE_OP_SET_PUD_IDENT, pudp, pud); +} + +static inline void ptp_set_sx_pte(pte_t *ptep, pte_t pte) +{ + ptp_rw_gate(IEE_OP_SET_SX_PTE, ptep, pte); +} + +extern pgprotval_t ptp_xchg(pgprotval_t *pgprotp, pgprotval_t pgprotval); +extern pgprotval_t ptp_try_cmpxchg(pgprotval_t *pgprotp, + pgprotval_t old_pgprot, pgprotval_t new_pgprotval); +extern void haoc_ptp_init(void); +extern struct iee_cache pgd_cache; + +#include +struct iee_cr0 { + /* Writable but considered safe to expose */ + unsigned long wp_disabled_cnt; +}; + +DECLARE_PER_CPU(struct iee_cr0, iee_cr0s); +#endif diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index c7ec5bb88334e..eeebd7345fa82 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -9,6 +9,9 @@ #define __HAVE_ARCH_PTE_ALLOC_ONE #define __HAVE_ARCH_PGD_FREE #include +#ifdef CONFIG_PTP +#include +#endif static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } @@ -153,7 +156,11 @@ static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) if (mm == &init_mm) gfp &= ~__GFP_ACCOUNT; +#ifdef CONFIG_PTP + return (p4d_t *)iee_cache_alloc(&pg_cache, gfp | __GFP_ZERO); +#else return (p4d_t *)get_zeroed_page(gfp); +#endif } static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) @@ -162,7 +169,11 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) return; BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, p4d); +#else free_page((unsigned long)p4d); +#endif } extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 993d49cd379a2..b3fd5f9f5165b 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -23,6 +23,13 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#include +#ifdef CONFIG_PTP_S +#include +#endif +#endif extern pgd_t early_top_pgt[PTRS_PER_PGD]; bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); @@ -1301,9 +1308,29 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, pte_t old_pte, new_pte; old_pte = READ_ONCE(*ptep); +#ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + do { + new_pte = pte_wrprotect(old_pte); + if (ptp_is_user_pgtable(ptep)) { + if (try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, + *(long *)&new_pte)) + break; + } else if (ptp_try_cmpxchg((long *)ptep, pte_val(old_pte), + pte_val(new_pte))) { + break; + } + } while (1); +#else /* CONFIG_PTP && !CONFIG_PTP_S */ + do { + new_pte = pte_wrprotect(old_pte); + } while (!ptp_try_cmpxchg((long *)ptep, pte_val(old_pte), pte_val(new_pte))); +#endif /* CONFIG_PTP_S */ +#else /* !CONFIG_PTP */ do { new_pte = pte_wrprotect(old_pte); } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte)); +#endif } #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) @@ -1363,9 +1390,29 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, pmd_t old_pmd, new_pmd; old_pmd = READ_ONCE(*pmdp); +#ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + do { + new_pmd = pmd_wrprotect(old_pmd); + if (ptp_is_user_pgtable(pmdp)) { + if (try_cmpxchg((long *)pmdp, (long *)&old_pmd, + *(long *)&new_pmd)) + break; + } else if ((ptp_try_cmpxchg((long *)pmdp, pmd_val(old_pmd), + pmd_val(new_pmd)))) { + break; + } + } while (1); +#else /* CONFIG_PTP && !CONFIG_PTP_S */ + do { + new_pmd = pmd_wrprotect(old_pmd); + } while (!ptp_try_cmpxchg((long *)pmdp, pmd_val(old_pmd), pmd_val(new_pmd))); +#endif /* CONFIG_PTP_S */ +#else do { new_pmd = pmd_wrprotect(old_pmd); } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd)); +#endif } #ifndef pmdp_establish @@ -1375,10 +1422,24 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, { page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); if (IS_ENABLED(CONFIG_SMP)) { +#ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(pmdp)) { + ptp_user_check_pmd_update(pmdp, pmd); + return xchg(pmdp, pmd); + } +#endif + return native_make_pmd(ptp_xchg((pgprotval_t *)pmdp, pmd_val(pmd))); +#else return xchg(pmdp, pmd); +#endif } else { pmd_t old = *pmdp; +#ifdef CONFIG_PTP + set_pmd(pmdp, pmd); +#else WRITE_ONCE(*pmdp, pmd); +#endif return old; } } @@ -1466,14 +1527,23 @@ static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) */ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) { +#ifdef CONFIG_PTP + iee_memcpy(dst, src, count * sizeof(pgd_t)); +#else memcpy(dst, src, count * sizeof(pgd_t)); +#endif #ifdef CONFIG_PAGE_TABLE_ISOLATION if (!static_cpu_has(X86_FEATURE_PTI)) return; /* Clone the user space pgd as well */ +#ifdef CONFIG_PTP + iee_memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), + count * sizeof(pgd_t)); +#else memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), count * sizeof(pgd_t)); #endif +#endif } #define PTE_SHIFT ilog2(PTRS_PER_PTE) diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index a629b1b9f65a6..a8f04cfca8d8d 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -15,6 +15,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif extern p4d_t level4_kernel_pgt[512]; extern p4d_t level4_ident_pgt[512]; @@ -64,7 +67,12 @@ void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte); static inline void native_set_pte(pte_t *ptep, pte_t pte) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*ptep); + ptp_set_pte(ptep, pte); +#else WRITE_ONCE(*ptep, pte); +#endif } static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, @@ -80,7 +88,12 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*pmdp); + ptp_set_pmd(pmdp, pmd); +#else WRITE_ONCE(*pmdp, pmd); +#endif } static inline void native_pmd_clear(pmd_t *pmd) @@ -91,7 +104,15 @@ static inline void native_pmd_clear(pmd_t *pmd) static inline pte_t native_ptep_get_and_clear(pte_t *xp) { #ifdef CONFIG_SMP +#ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(xp)) + return native_make_pte(xchg(&xp->pte, 0)); +#endif + return native_make_pte(ptp_xchg((pgprotval_t *)xp, 0)); +#else return native_make_pte(xchg(&xp->pte, 0)); +#endif #else /* native_local_ptep_get_and_clear, but duplicated because of cyclic dependency */ @@ -104,7 +125,15 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) { #ifdef CONFIG_SMP +#ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(xp)) + return native_make_pmd(xchg(&xp->pmd, 0)); +#endif + return native_make_pmd(ptp_xchg((pgprotval_t *)xp, 0)); +#else return native_make_pmd(xchg(&xp->pmd, 0)); +#endif #else /* native_local_pmdp_get_and_clear, but duplicated because of cyclic dependency */ @@ -116,7 +145,12 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) static inline void native_set_pud(pud_t *pudp, pud_t pud) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*pudp); + ptp_set_pud(pudp, pud); +#else WRITE_ONCE(*pudp, pud); +#endif } static inline void native_pud_clear(pud_t *pud) @@ -127,7 +161,15 @@ static inline void native_pud_clear(pud_t *pud) static inline pud_t native_pudp_get_and_clear(pud_t *xp) { #ifdef CONFIG_SMP +#ifdef CONFIG_PTP +#ifdef CONFIG_PTP_S + if (ptp_is_user_pgtable(xp)) + return native_make_pud(xchg(&xp->pud, 0)); +#endif + return native_make_pud(ptp_xchg((pgprotval_t *)xp, 0)); +#else return native_make_pud(xchg(&xp->pud, 0)); +#endif #else /* native_local_pudp_get_and_clear, * but duplicated because of cyclic dependency @@ -144,13 +186,23 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) pgd_t pgd; if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*p4dp); + ptp_set_p4d(p4dp, p4d); +#else WRITE_ONCE(*p4dp, p4d); +#endif return; } pgd = native_make_pgd(native_p4d_val(p4d)); pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*p4dp); + ptp_set_p4d(p4dp, native_make_p4d(native_pgd_val(pgd))); +#else WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); +#endif } static inline void native_p4d_clear(p4d_t *p4d) @@ -160,7 +212,12 @@ static inline void native_p4d_clear(p4d_t *p4d) static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) { +#ifdef CONFIG_PTP + compiletime_assert_rwonce_type(*pgdp); + ptp_set_pgd(pgdp, pti_set_user_pgtbl(pgdp, pgd)); +#else WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); +#endif } static inline void native_pgd_clear(pgd_t *pgd) diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index 1ab08117dea8c..0a3737ba7a7c9 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -9,7 +9,7 @@ #include #include #include -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) #include extern bool haoc_enabled; #endif @@ -53,7 +53,7 @@ static inline unsigned long __native_read_cr3(void) return val; } -#ifdef CONFIG_IEE_SIP +#if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) static inline void iee_write_cr3_early(unsigned long val) { asm volatile("mov %0,%%cr3" : : "r" (val) : "memory"); @@ -62,7 +62,7 @@ static inline void iee_write_cr3_early(unsigned long val) static inline void native_write_cr3(unsigned long val) { - #ifdef CONFIG_IEE_SIP + #if defined(CONFIG_IEE_SIP) && !defined(__DISABLE_EXPORTS) if(haoc_enabled) iee_write_cr3(val); else diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h index 4d3c9d00d6b6b..4a08a69b172ba 100644 --- a/arch/x86/include/asm/tlb.h +++ b/arch/x86/include/asm/tlb.h @@ -38,4 +38,13 @@ static inline void invlpg(unsigned long addr) { asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); } + +#ifdef CONFIG_PTP +#include +static inline void __ptp_tlb_remove_table(void *table) +{ + iee_cache_free(&pg_cache, page_to_virt((struct page *)table)); +} +#endif + #endif /* _ASM_X86_TLB_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 18ce2b759dbb8..4e9689479bfce 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1992,6 +1992,25 @@ static void text_poke_memset(void *dst, const void *src, size_t len) typedef void text_poke_f(void *dst, const void *src, size_t len); +#ifdef CONFIG_PTP +static inline void set_ptes_text_poke(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + page_table_check_ptes_set(mm, ptep, pte, nr); + + arch_enter_lazy_mmu_mode(); + for (;;) { + compiletime_assert_rwonce_type(*ptep); + ptp_set_pte_text_poke(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_next_pfn(pte); + } + arch_leave_lazy_mmu_mode(); +} +#endif + static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) { bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; @@ -2043,11 +2062,19 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l local_irq_save(flags); pte = mk_pte(pages[0], pgprot); +#ifdef CONFIG_PTP + set_ptes_text_poke(poking_mm, poking_addr, ptep, pte, 1); +#else set_pte_at(poking_mm, poking_addr, ptep, pte); +#endif if (cross_page_boundary) { pte = mk_pte(pages[1], pgprot); +#ifdef CONFIG_PTP + set_ptes_text_poke(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte, 1); +#else set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); +#endif } /* diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index d9e596eae225c..3d57a3cc0b3b1 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -406,7 +406,7 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c) /* These bits should not change their value after CPU init is finished. */ #ifdef CONFIG_IEE_SIP -unsigned long cr4_pinned_mask = +const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE | X86_CR4_CET; DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); @@ -2644,8 +2644,4 @@ void __init arch_cpu_finalize_init(void) * hypercalls work when the SWIOTLB bounce buffers are decrypted. */ mem_encrypt_init(); - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - iee_sip_init(); - #endif } diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index 367af97ba37ea..826c316116f99 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -1,3 +1,8 @@ -obj-y += haoc.o +obj-y += haoc.o bitmap.o obj-y += iee/ -obj-$(CONFIG_CREDP) += credp/ \ No newline at end of file +obj-$(CONFIG_PTP) += ptp/ +obj-$(CONFIG_CREDP) += credp/ +obj-$(CONFIG_KEYP) += keyp/ +ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ +obj-$(CONFIG_VARP) += varp/ diff --git a/arch/x86/kernel/haoc/bitmap.c b/arch/x86/kernel/haoc/bitmap.c new file mode 100644 index 0000000000000..57fc0ff87cf19 --- /dev/null +++ b/arch/x86/kernel/haoc/bitmap.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +unsigned long iee_bitmap_base __ro_after_init = __VIEEBITMAP_BASE_L4; +EXPORT_SYMBOL(iee_bitmap_base); + +static pte_t * __init iee_bitmap_pte_populate(pmd_t *pmd, unsigned long addr) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + + if (pte_none(*pte)) { + void *p = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 0); + + if (!p) + return NULL; + pte_t entry = pfn_pte(__pa(p) >> PAGE_SHIFT, + __pgprot(pgprot_val(PAGE_KERNEL) & (~__RW) & (~___D))); + + set_pte_at(&init_mm, addr, pte, entry); + } + return pte; +} + +static pmd_t * __init iee_bitmap_pmd_populate(pud_t *pud, unsigned long addr) +{ + pmd_t *pmd = pmd_offset(pud, addr); + + if (pmd_none(*pmd)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); + } + return pmd; +} + +static pud_t * __init iee_bitmap_pud_populate(p4d_t *p4d, unsigned long addr) +{ + pud_t *pud = pud_offset(p4d, addr); + + if (pud_none(*pud)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + pud_populate(&init_mm, pud, p); + } + return pud; +} + +static p4d_t * __init iee_bitmap_p4d_populate(pgd_t *pgd, unsigned long addr) +{ + p4d_t *p4d = p4d_offset(pgd, addr); + + if (p4d_none(*p4d)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + p4d_populate(&init_mm, p4d, p); + } + return p4d; +} + +static pgd_t * __init iee_bitmap_pgd_populate(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + + if (pgd_none(*pgd)) { +#ifdef CONFIG_PTP + void *p = iee_cache_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); +#else + void *p = (void *)get_zeroed_page(GFP_KERNEL); +#endif + + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); + } + return pgd; +} + +static pte_t * __init iee_bitmap_populate(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = iee_bitmap_pgd_populate(addr); + if (!pgd) + return NULL; + p4d = iee_bitmap_p4d_populate(pgd, addr); + if (!p4d) + return NULL; + pud = iee_bitmap_pud_populate(p4d, addr); + if (!pud) + return NULL; + pmd = iee_bitmap_pmd_populate(pud, addr); + if (!pmd) + return NULL; + pte = iee_bitmap_pte_populate(pmd, addr); + return pte; +} + +static void __init iee_set_bitmap_type_init(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + phys_addr_t phys_addr; + + pgd = pgd_offset_k(addr); + p4d = p4d_offset(pgd, addr); + pud = pud_offset(p4d, addr); + pmd = pmd_offset(pud, addr); + pte = pte_offset_kernel(pmd, addr); + phys_addr = pte_pfn(*pte) << PAGE_SHIFT; + iee_set_bitmap_type(phys_addr, 1, IEE_BITMAP); +} + +void __init iee_init_bitmap(void) +{ + unsigned long addr, start, end; + pte_t *pte; + + start = PAGE_ALIGN_DOWN((unsigned long)(__pfn_to_iee_bitmap(0))); + end = PAGE_ALIGN((unsigned long)(__pfn_to_iee_bitmap(max_pfn))); + addr = start; + for (; addr < end; addr += PAGE_SIZE) { + pte = iee_bitmap_populate(addr); + if (!pte) + panic("No mem for iee bitmap init"); + } + addr = start; + for (; addr < end; addr += PAGE_SIZE) + iee_set_bitmap_type_init(addr); + pr_info("IEE bitmap: [0x%lx, 0x%lx]!", start, end); +} + +static inline void verify_bitmap_ptr(uint8_t *bitmap_ptr) +{ + unsigned long addr = (unsigned long)bitmap_ptr; + unsigned long start = PAGE_ALIGN_DOWN((unsigned long)(__pfn_to_iee_bitmap(0))); + unsigned long end = PAGE_ALIGN((unsigned long)(__pfn_to_iee_bitmap(max_pfn))); + + if (addr < start || addr >= end) + panic("IEE detected non-bitmap ptr: %lx", addr); +} + +static uint8_t *iee_bitmap_rw_alias(uint8_t *bitmap_ptr) +{ + unsigned long addr = (unsigned long)bitmap_ptr; + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); + pmd_t *pmd = pmd_offset(pud, addr); + pte_t *pte = pte_offset_kernel(pmd, addr); + phys_addr_t phys = (pte_pfn(*pte) << PAGE_SHIFT) | (addr & ~PAGE_MASK); + + return __va(phys); +} + +void _iee_set_bitmap_type(unsigned long __unused, + uint8_t *bitmap_ptr, enum HAOC_BITMAP_TYPE type, int num_pages) +{ + verify_bitmap_ptr(bitmap_ptr); + if (!haoc_enabled) + bitmap_ptr = iee_bitmap_rw_alias(bitmap_ptr); + memset(bitmap_ptr, type, num_pages); +} diff --git a/arch/x86/kernel/haoc/credp/credp.c b/arch/x86/kernel/haoc/credp/credp.c index 2ecc48a9fe152..cfc8cf115ae7b 100644 --- a/arch/x86/kernel/haoc/credp/credp.c +++ b/arch/x86/kernel/haoc/credp/credp.c @@ -2,27 +2,52 @@ #include #include #include -#include +#include +#include #include +#include #include #include +#ifdef CONFIG_IEE_PTRP +#include +#endif #include "slab.h" extern struct cred init_cred; +static inline void iee_verify_cred_type(const struct cred *cred) +{ + iee_verify_type(__pa(cred), IEE_CRED, "cred"); +} + +static inline void iee_verify_current_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_cred(); +} + +static inline void iee_verify_new_cred_type(struct cred *cred) +{ + iee_verify_cred_type(cred); + iee_verify_cred(); +} + void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) { + iee_verify_current_cred_type(cred); *((struct rcu_head **)(&(cred->rcu.func))) = rcu; } void _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security) { + iee_verify_current_cred_type(cred); cred->security = security; } unsigned long _iee_set_cred_atomic_op_usage(unsigned long __unused, struct cred *cred, int flag, int nr) { + iee_verify_current_cred_type(cred); switch (flag) { case AT_ADD: { atomic_long_add(nr, &cred->usage); @@ -40,126 +65,149 @@ unsigned long _iee_set_cred_atomic_op_usage(unsigned long __unused, void _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i) { + iee_verify_current_cred_type(cred); atomic_long_set(&cred->usage, i); } void _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu) { + iee_verify_current_cred_type(cred); cred->non_rcu = non_rcu; } void _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, struct key *session_keyring) { + iee_verify_new_cred_type(cred); cred->session_keyring = session_keyring; } void _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, struct key *process_keyring) { + iee_verify_new_cred_type(cred); cred->process_keyring = process_keyring; } void _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, struct key *thread_keyring) { + iee_verify_current_cred_type(cred); cred->thread_keyring = thread_keyring; } void _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, struct key *request_key_auth) { + iee_verify_new_cred_type(cred); cred->request_key_auth = request_key_auth; } void _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring) { + iee_verify_new_cred_type(cred); cred->jit_keyring = jit_keyring; } void _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, kernel_cap_t cap_inheritable) { + iee_verify_new_cred_type(cred); cred->cap_inheritable = cap_inheritable; } void _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, kernel_cap_t cap_permitted) { + iee_verify_new_cred_type(cred); cred->cap_permitted = cap_permitted; } void _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, kernel_cap_t cap_effective) { + iee_verify_new_cred_type(cred); cred->cap_effective = cap_effective; } void _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset) { + iee_verify_new_cred_type(cred); cred->cap_bset = cap_bset; } void _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient) { + iee_verify_new_cred_type(cred); cred->cap_ambient = cap_ambient; } void _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, unsigned int securebits) { + iee_verify_new_cred_type(cred); cred->securebits = securebits; } void _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, struct group_info *group_info) { + iee_verify_current_cred_type(cred); cred->group_info = group_info; } void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts) { + iee_verify_new_cred_type(cred); cred->ucounts = ucounts; } void _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, struct user_namespace *user_ns) { + iee_verify_new_cred_type(cred); cred->user_ns = user_ns; } void _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user) { + iee_verify_new_cred_type(cred); cred->user = user; } void _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid) { + iee_verify_new_cred_type(cred); cred->fsgid = fsgid; } void _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid) { + iee_verify_new_cred_type(cred); cred->fsuid = fsuid; } void _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid) { + iee_verify_new_cred_type(cred); cred->egid = egid; } void _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid) { + iee_verify_new_cred_type(cred); cred->euid = euid; } void _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid) { + iee_verify_new_cred_type(cred); cred->sgid = sgid; } void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid) { + iee_verify_new_cred_type(cred); cred->suid = suid; } @@ -169,11 +217,21 @@ void _iee_copy_cred(unsigned long __unused, struct cred *new) { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - struct task_token *token = (struct task_token *)__addr_to_iee(current); /* Get old cred inside IEE is safer. */ const struct cred *old = current_cred(); - /* Would verify this field in commit_cred. */ - token->new_cred = new; + + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -187,13 +245,23 @@ void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, { struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); struct cred *_new = __ptr_to_iee(new); - /* Would verify this field in commit_cred. */ - struct task_token *token = (struct task_token *)__addr_to_iee(current); + + iee_verify_cred_type(old); + iee_verify_cred_type(new); + iee_verify_cred(); if (!uid_eq(current_uid(), init_cred.uid)) panic("IEE: calling prepare_kernel_cred by unprivileged process."); - token->new_cred = new; +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + /* Would verify this field in commit_creds. */ + token->new_cred = new; + } +#endif + _iee_memcpy(0, new, (struct cred *)old, sizeof(struct cred)); _new->non_rcu = 0; atomic_long_set(&_new->usage, 1); @@ -205,13 +273,24 @@ void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, void _iee_init_copied_cred(unsigned long __unused, struct task_struct *new_task, struct cred *new) { - struct task_token *old_task_token = (struct task_token *)__addr_to_iee(current); - - if (old_task_token->new_cred != new) - panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", __func__, - (u64)old_task_token->new_cred, (u64)new); - /* Update token info of new task by current task token. */ - old_task_token->new_cred = NULL; + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *old_task_token = + (struct task_token *)__addr_to_token(current); + struct task_token *new_task_token = + (struct task_token *)__addr_to_token(new_task); + + if (old_task_token->new_cred != new) + panic("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", + __func__, (u64)old_task_token->new_cred, (u64)new); + /* Update token info of new task by current task token. */ + old_task_token->new_cred = NULL; + new_task_token->curr_cred = new; + } +#endif new_task->cred = new_task->real_cred = new; } @@ -219,12 +298,26 @@ void _iee_init_copied_cred(unsigned long __unused, void _iee_commit_creds(unsigned long __unused, const struct cred *new) { struct task_struct *task = current; - struct task_token *token = (struct task_token *)__addr_to_iee(task); - if (token->new_cred != new) - panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); - /* task->cred shall be updated once. */ - token->new_cred = NULL; + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(task); + + if (token->new_cred != new) + panic("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); + /* task->cred shall be updated once. */ + token->new_cred = NULL; + write_seqcount_begin(&token->seq); + token->curr_cred = new; + rcu_assign_pointer(task->real_cred, new); + rcu_assign_pointer(task->cred, new); + write_seqcount_end(&token->seq); + return; + } +#endif rcu_assign_pointer(task->real_cred, new); rcu_assign_pointer(task->cred, new); @@ -232,21 +325,110 @@ void _iee_commit_creds(unsigned long __unused, const struct cred *new) void _iee_abort_cred(unsigned long __unused, const struct cred *cred) { - struct task_token *token = (struct task_token *)__addr_to_iee(current); + iee_verify_cred_type(cred); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); - token->new_cred = NULL; + token->new_cred = NULL; + } +#endif +} + +void _iee_fill_cred_for_session_keyring(unsigned long __unused, struct cred *new, + const struct cred *old) +{ + iee_verify_cred_type(new); + iee_verify_cred_type(old); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + token->new_cred = new; + } +#endif + + new->uid = old->uid; + new->euid = old->euid; + new->suid = old->suid; + new->fsuid = old->fsuid; + new->gid = old->gid; + new->egid = old->egid; + new->sgid = old->sgid; + new->fsgid = old->fsgid; + new->user = get_uid(old->user); + new->ucounts = old->ucounts; + new->user_ns = get_user_ns(old->user_ns); + new->group_info = get_group_info(old->group_info); + + new->securebits = old->securebits; + new->cap_inheritable = old->cap_inheritable; + new->cap_permitted = old->cap_permitted; + new->cap_effective = old->cap_effective; + new->cap_ambient = old->cap_ambient; + new->cap_bset = old->cap_bset; + + new->jit_keyring = old->jit_keyring; + new->thread_keyring = key_get(old->thread_keyring); + new->process_keyring = key_get(old->process_keyring); } void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) { + iee_verify_new_cred_type(cred); cred->gid = gid; } void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid) { + iee_verify_new_cred_type(cred); cred->uid = uid; } +void _iee_override_creds(unsigned long __unused, const struct cred *new) +{ + iee_verify_cred_type(new); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + write_seqcount_begin(&token->seq); + token->curr_cred = new; + rcu_assign_pointer(current->cred, new); + write_seqcount_end(&token->seq); + return; + } +#endif + + rcu_assign_pointer(current->cred, new); +} + +void _iee_revert_creds(unsigned long __unused, const struct cred *old) +{ + iee_verify_cred_type(old); + iee_verify_cred(); + +#ifdef CONFIG_IEE_PTRP + if (haoc_init_done) { + struct task_token *token = (struct task_token *)__addr_to_token(current); + + write_seqcount_begin(&token->seq); + token->curr_cred = old; + rcu_assign_pointer(current->cred, old); + write_seqcount_end(&token->seq); + return; + } +#endif + + rcu_assign_pointer(current->cred, old); +} + struct iee_free_slab_work { struct work_struct work; struct kmem_cache *s; @@ -261,7 +443,19 @@ void iee_free_cred_slab(struct work_struct *work) struct folio *folio = slab_folio(slab); int order = folio_order(folio); - unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); + unset_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << order); __free_pages(&folio->page, order); kfree(iee_free_slab_work); } + +void __init credp_init(void) +{ + unsigned long start, end; + int num_pages; + + start = (unsigned long)__iee_cred_data_start; + end = (unsigned long)__iee_cred_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_pages(start, num_pages, IEE_CRED); +} diff --git a/arch/x86/kernel/haoc/gate.h b/arch/x86/kernel/haoc/gate.h new file mode 100644 index 0000000000000..c3a8f4f24cf3c --- /dev/null +++ b/arch/x86/kernel/haoc/gate.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) + +/* + * scratch_reg would be changed, caller should dertimine if scratch_reg + * should be saved and restored. + */ +.macro DISABLE_WP scratch_reg:req + /* Disable write protection */ + movq %cr0, %\scratch_reg + andq $(~X86_CR0_WP), %\scratch_reg + movq %\scratch_reg, %cr0 +.endm + +.macro ENABLE_WP scratch_reg:req + /* Enable write protection */ + movq %cr0, %\scratch_reg +1: + orq $X86_CR0_WP, %\scratch_reg + movq %\scratch_reg, %cr0 + testq $X86_CR0_WP, %\scratch_reg + je 1b +.endm diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index b9576af3fc913..014f8d2f624ab 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -7,53 +7,350 @@ * Hu Bing */ +#include +#include #include +#include +#include -typedef void (*iee_func)(void); -iee_func iee_funcs[] = { - (iee_func)_iee_memcpy, - (iee_func)_iee_memset, - (iee_func)_iee_set_freeptr, - (iee_func)_iee_test_and_clear_bit, +notrace unsigned long iee_dispatch(unsigned long flag, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5) +{ + switch (flag) { + case IEE_OP_MEMCPY: + _iee_memcpy(flag, (void *)arg1, (void *)arg2, (size_t)arg3); + break; + case IEE_OP_MEMSET: + _iee_memset(flag, (void *)arg1, (int)arg2, (size_t)arg3); + break; + case IEE_OP_SET_FREEPTR: + _iee_set_freeptr(flag, (void **)arg1, (void *)arg2); + break; + case IEE_OP_TEST_CLEAR_BIT: + return _iee_test_and_clear_bit(flag, (long)arg1, + (unsigned long *)arg2); #ifdef CONFIG_IEE_PTRP - (iee_func)_iee_set_token_pgd, - (iee_func)_iee_invalidate_token, - (iee_func)_iee_validate_token, + case IEE_OP_SET_TOKEN_PGD: + _iee_set_token_pgd(flag, (struct task_struct *)arg1, + (pgd_t *)arg2); + break; + case IEE_OP_INVALIDATE_TOKEN: + _iee_invalidate_token(flag, (struct task_struct *)arg1); + break; + case IEE_OP_VALIDATE_TOKEN: + _iee_validate_token(flag, (struct task_struct *)arg1); + break; #endif + case IEE_OP_SET_BITMAP_TYPE: + _iee_set_bitmap_type(flag, (uint8_t *)arg1, + (enum HAOC_BITMAP_TYPE)arg2, (int)arg3); + break; #ifdef CONFIG_CREDP - (iee_func)_iee_copy_cred, - (iee_func)_iee_copy_kernel_cred, - (iee_func)_iee_init_copied_cred, - (iee_func)_iee_commit_creds, - (iee_func)_iee_abort_cred, - (iee_func)_iee_set_cred_uid, - (iee_func)_iee_set_cred_gid, - (iee_func)_iee_set_cred_suid, - (iee_func)_iee_set_cred_sgid, - (iee_func)_iee_set_cred_euid, - (iee_func)_iee_set_cred_egid, - (iee_func)_iee_set_cred_fsuid, - (iee_func)_iee_set_cred_fsgid, - (iee_func)_iee_set_cred_user, - (iee_func)_iee_set_cred_user_ns, - (iee_func)_iee_set_cred_group_info, - (iee_func)_iee_set_cred_securebits, - (iee_func)_iee_set_cred_cap_inheritable, - (iee_func)_iee_set_cred_cap_permitted, - (iee_func)_iee_set_cred_cap_effective, - (iee_func)_iee_set_cred_cap_bset, - (iee_func)_iee_set_cred_cap_ambient, - (iee_func)_iee_set_cred_jit_keyring, - (iee_func)_iee_set_cred_session_keyring, - (iee_func)_iee_set_cred_process_keyring, - (iee_func)_iee_set_cred_thread_keyring, - (iee_func)_iee_set_cred_request_key_auth, - (iee_func)_iee_set_cred_non_rcu, - (iee_func)_iee_set_cred_atomic_set_usage, - (iee_func)_iee_set_cred_atomic_op_usage, - (iee_func)_iee_set_cred_security, - (iee_func)_iee_set_cred_rcu, - (iee_func)_iee_set_cred_ucounts, + case IEE_OP_COPY_CRED: + _iee_copy_cred(flag, (struct cred *)arg1); + break; + case IEE_OP_COPY_KERNEL_CRED: + _iee_copy_kernel_cred(flag, (const struct cred *)arg1, + (struct cred *)arg2); + break; + case IEE_OP_INIT_COPIED_CRED: + _iee_init_copied_cred(flag, (struct task_struct *)arg1, + (struct cred *)arg2); + break; + case IEE_OP_COMMIT_CRED: + _iee_commit_creds(flag, (const struct cred *)arg1); + break; + case IEE_OP_ABORT_CRED: + _iee_abort_cred(flag, (const struct cred *)arg1); + break; + case IEE_OP_FILL_SESSION_KEYRING_CRED: + _iee_fill_cred_for_session_keyring(flag, (struct cred *)arg1, + (const struct cred *)arg2); + break; + case IEE_OP_OVERRIDE_CRED: + _iee_override_creds(flag, (const struct cred *)arg1); + break; + case IEE_OP_REVERT_CRED: + _iee_revert_creds(flag, (const struct cred *)arg1); + break; + case IEE_OP_SET_CRED_UID: + _iee_set_cred_uid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_GID: + _iee_set_cred_gid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_SUID: + _iee_set_cred_suid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_SGID: + _iee_set_cred_sgid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_EUID: + _iee_set_cred_euid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_EGID: + _iee_set_cred_egid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_FSUID: + _iee_set_cred_fsuid(flag, (struct cred *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_CRED_FSGID: + _iee_set_cred_fsgid(flag, (struct cred *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_CRED_USER: + _iee_set_cred_user(flag, (struct cred *)arg1, + (struct user_struct *)arg2); + break; + case IEE_OP_SET_CRED_USER_NS: + _iee_set_cred_user_ns(flag, (struct cred *)arg1, + (struct user_namespace *)arg2); + break; + case IEE_OP_SET_CRED_GROUP_INFO: + _iee_set_cred_group_info(flag, (struct cred *)arg1, + (struct group_info *)arg2); + break; + case IEE_OP_SET_CRED_SECUREBITS: + _iee_set_cred_securebits(flag, (struct cred *)arg1, + (unsigned int)arg2); + break; + case IEE_OP_SET_CRED_CAP_INHER: + _iee_set_cred_cap_inheritable(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_PERM: + _iee_set_cred_cap_permitted(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_EFFECT: + _iee_set_cred_cap_effective(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_BSET: + _iee_set_cred_cap_bset(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_CAP_AMBIENT: + _iee_set_cred_cap_ambient(flag, (struct cred *)arg1, + (kernel_cap_t){ .val = arg2 }); + break; + case IEE_OP_SET_CRED_JIT_KEYRING: + _iee_set_cred_jit_keyring(flag, (struct cred *)arg1, + (unsigned char)arg2); + break; + case IEE_OP_SET_CRED_SESS_KEYRING: + _iee_set_cred_session_keyring(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_PROC_KEYRING: + _iee_set_cred_process_keyring(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_THREAD_KEYRING: + _iee_set_cred_thread_keyring(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_REQ_KEYRING: + _iee_set_cred_request_key_auth(flag, (struct cred *)arg1, + (struct key *)arg2); + break; + case IEE_OP_SET_CRED_NON_RCU: + _iee_set_cred_non_rcu(flag, (struct cred *)arg1, (int)arg2); + break; + case IEE_OP_SET_CRED_ATSET_USAGE: + _iee_set_cred_atomic_set_usage(flag, (struct cred *)arg1, + (int)arg2); + break; + case IEE_OP_SET_CRED_ATOP_USAGE: + return _iee_set_cred_atomic_op_usage(flag, (struct cred *)arg1, + (int)arg2, (int)arg3); + case IEE_OP_SET_CRED_SECURITY: + _iee_set_cred_security(flag, (struct cred *)arg1, (void *)arg2); + break; + case IEE_OP_SET_CRED_RCU: + _iee_set_cred_rcu(flag, (struct cred *)arg1, + (struct rcu_head *)arg2); + break; + case IEE_OP_SET_CRED_UCOUNTS: + _iee_set_cred_ucounts(flag, (struct cred *)arg1, + (struct ucounts *)arg2); + break; #endif - NULL -}; +#ifdef CONFIG_KEYP + case IEE_OP_SET_KEY_UNION: + _iee_set_key_union(flag, (struct key *)arg1, + (struct key_union *)arg2); + break; + case IEE_OP_SET_KEY_STRUCT: + _iee_set_key_struct(flag, (struct key *)arg1, + (struct key_struct *)arg2); + break; + case IEE_OP_SET_KEY_PAYLOAD: + _iee_set_key_payload(flag, (struct key *)arg1, + (union key_payload *)arg2); + break; + case IEE_OP_SET_KEY_USAGE: + return _iee_set_key_usage(flag, (struct key *)arg1, (int)arg2, + (int)arg3); + case IEE_OP_SET_KEY_SERIAL: + _iee_set_key_serial(flag, (struct key *)arg1, + (key_serial_t)arg2); + break; + case IEE_OP_SET_KEY_WATCHERS: + _iee_set_key_watchers(flag, (struct key *)arg1, + (struct watch_list *)arg2); + break; + case IEE_OP_SET_KEY_USERS: + _iee_set_key_user(flag, (struct key *)arg1, + (struct key_user *)arg2); + break; + case IEE_OP_SET_KEY_SECURITY: + _iee_set_key_security(flag, (struct key *)arg1, (void *)arg2); + break; + case IEE_OP_SET_KEY_EXPIRY: + _iee_set_key_expiry(flag, (struct key *)arg1, (time64_t)arg2); + break; + case IEE_OP_SET_KEY_REVOKED_AT: + _iee_set_key_revoked_at(flag, (struct key *)arg1, + (time64_t)arg2); + break; + case IEE_OP_SET_KEY_LAST_USED_AT: + _iee_set_key_last_used_at(flag, (struct key *)arg1, + (time64_t)arg2); + break; + case IEE_OP_SET_KEY_UID: + _iee_set_key_uid(flag, (struct key *)arg1, + KUIDT_INIT((uid_t)arg2)); + break; + case IEE_OP_SET_KEY_GID: + _iee_set_key_gid(flag, (struct key *)arg1, + KGIDT_INIT((gid_t)arg2)); + break; + case IEE_OP_SET_KEY_PERM: + _iee_set_key_perm(flag, (struct key *)arg1, (key_perm_t)arg2); + break; + case IEE_OP_SET_KEY_QUOTALEN: + _iee_set_key_quotalen(flag, (struct key *)arg1, + (unsigned short)arg2); + break; + case IEE_OP_SET_KEY_DATALEN: + _iee_set_key_datalen(flag, (struct key *)arg1, + (unsigned short)arg2); + break; + case IEE_OP_SET_KEY_STATE: + _iee_set_key_state(flag, (struct key *)arg1, (short)arg2); + break; + case IEE_OP_SET_KEY_MAGIC: + _iee_set_key_magic(flag, (struct key *)arg1, + (unsigned int)arg2); + break; + case IEE_OP_SET_KEY_FLAGS: + _iee_set_key_flags(flag, (struct key *)arg1, arg2); + break; + case IEE_OP_SET_KEY_INDEX_KEY: + _iee_set_key_index_key(flag, (struct key *)arg1, + (struct keyring_index_key *)arg2); + break; + case IEE_OP_SET_KEY_HASH: + _iee_set_key_hash(flag, (struct key *)arg1, arg2); + break; + case IEE_OP_SET_KEY_LEN_DESC: + _iee_set_key_len_desc(flag, (struct key *)arg1, arg2); + break; + case IEE_OP_SET_KEY_TYPE: + _iee_set_key_type(flag, (struct key *)arg1, + (struct key_type *)arg2); + break; + case IEE_OP_SET_KEY_TAG: + _iee_set_key_domain_tag(flag, (struct key *)arg1, + (struct key_tag *)arg2); + break; + case IEE_OP_SET_KEY_DESCRIPTION: + _iee_set_key_description(flag, (struct key *)arg1, (char *)arg2); + break; + case IEE_OP_SET_KEY_RESTRICT_LINK: + _iee_set_key_restrict_link(flag, (struct key *)arg1, + (struct key_restriction *)arg2); + break; + case IEE_OP_SET_KEY_FLAG_BIT: + return _iee_set_key_flag_bit(flag, (struct key *)arg1, (long)arg2, + (int)arg3); +#endif +#ifdef CONFIG_IEE_SELINUX_P + case IEE_SEL_SET_STATUS_PG: + _iee_set_selinux_status_pg(flag, (struct page *)arg1); + break; + case IEE_SEL_SET_ENFORCING: + _iee_set_selinux_enforcing(flag, (bool)arg1); + break; + case IEE_SEL_SET_INITIALIZED: + _iee_mark_selinux_initialized(flag); + break; + case IEE_SEL_SET_POLICY_CAP: + _iee_set_sel_policy_cap(flag, (unsigned int)arg1, (int)arg2); + break; + case IEE_SEL_RCU_ASSIGN_POLICY: + _iee_sel_rcu_assign_policy(flag, (struct selinux_policy *)arg1, + (struct selinux_policy *)arg2); + break; +#endif +#ifdef CONFIG_PTP +#ifdef CONFIG_IEE_PTRP + case IEE_OP_UNSET_TOKEN: + _iee_unset_token(flag, (pte_t *)arg1, (pte_t *)arg2, arg3, + (unsigned int)arg4); + break; + case IEE_OP_SET_TOKEN: + _iee_set_token(flag, (pte_t *)arg1, (pte_t *)arg2, arg3, + (unsigned int)arg4); + break; +#endif + case IEE_OP_SET_PTE: + _iee_set_pte(flag, (pte_t *)arg1, __pte(arg2)); + break; + case IEE_OP_SET_PMD: + _iee_set_pmd(flag, (pmd_t *)arg1, __pmd(arg2)); + break; + case IEE_OP_SET_PUD: + _iee_set_pud(flag, (pud_t *)arg1, __pud(arg2)); + break; + case IEE_OP_SET_P4D: + _iee_set_p4d(flag, (p4d_t *)arg1, __p4d(arg2)); + break; + case IEE_OP_SET_PGD: + _iee_set_pgd(flag, (pgd_t *)arg1, __pgd(arg2)); + break; + case IEE_OP_SET_PTE_TEXT_POKE: + _iee_set_pte_text_poke(flag, (pte_t *)arg1, __pte(arg2)); + break; + case IEE_OP_SET_PMD_IDENT: + _iee_set_pmd_ident(flag, (pmd_t *)arg1, __pmd(arg2)); + break; + case IEE_OP_SET_PUD_IDENT: + _iee_set_pud_ident(flag, (pud_t *)arg1, __pud(arg2)); + break; + case IEE_OP_SET_SX_PTE: + _iee_set_sx_pte(flag, (pte_t *)arg1, __pte(arg2)); + break; +#endif +#ifdef CONFIG_VARP + case IEE_OP_SET_VARP_MODPROBE_PATH: + _iee_set_varp_modprobe_path(flag, (char *)arg1, (int)arg2, + (size_t)arg3, (char *)arg4, + (size_t *)arg5); + break; +#endif + default: + panic("iee_dispatch: invalid flag %lu\n", flag); + } + + return 0; +} diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index 0054c1b3d6169..33d4f024c0921 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -15,14 +15,31 @@ #endif extern bool haoc_enabled; -void set_iee_page(unsigned long addr, unsigned int order) +void set_iee_page(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) { - set_memory_ro(addr, 1 << order); +#ifdef CONFIG_PTP_S + if (type == IEE_USER_PGTABLE) + set_memory_rw(addr, num_pages); + else +#endif + set_memory_ro(addr, num_pages); + iee_set_bitmap_type(__pa(addr), num_pages, type); +} + +void unset_iee_page(unsigned long addr, int num_pages) +{ + iee_set_bitmap_type(__pa(addr), num_pages, IEE_NORMAL); + set_memory_rw(addr, num_pages); } -void unset_iee_page(unsigned long addr, unsigned int order) +void set_iee_pages(unsigned long addr, int num_pages, enum HAOC_BITMAP_TYPE type) { - set_memory_rw(addr, 1 << order); + set_iee_page(addr, num_pages, type); +} + +void unset_iee_pages(unsigned long addr, int num_pages) +{ + unset_iee_page(addr, num_pages); } struct iee_free_slab_work { @@ -34,11 +51,14 @@ struct iee_free_slab_work { void iee_free_slab(struct kmem_cache *s, struct slab *slab, void (*do_free_slab)(struct work_struct *work)) { - if(haoc_enabled) - return; struct iee_free_slab_work *iee_free_slab_work = kmalloc(sizeof(struct iee_free_slab_work), GFP_ATOMIC); + if (!iee_free_slab_work) { + pr_warn("HAOC: failed to allocate deferred slab free work\n"); + return; + } + iee_free_slab_work->s = s; iee_free_slab_work->slab = slab; INIT_WORK(&iee_free_slab_work->work, do_free_slab); @@ -52,11 +72,12 @@ static void iee_free_task_struct_slab(struct work_struct *work) container_of(work, struct iee_free_slab_work, work); struct slab *slab = iee_free_slab_work->slab; struct folio *folio = slab_folio(slab); - unsigned int order = folio_order(folio); + unsigned int task_order = folio_order(folio); + unsigned int token_order = IEE_TOKEN_ORDER(task_order); unsigned long token = __slab_to_iee(slab); - // Free token. - iee_set_token_page_invalid(token, 0, order); - __free_pages(&folio->page, order); + + iee_set_token_page_invalid(token, 0, token_order); + __free_pages(&folio->page, task_order); kfree(iee_free_slab_work); } #endif @@ -88,4 +109,4 @@ unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) order = IEE_DATA_ORDER; #endif return order; -} \ No newline at end of file +} diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index 5497c3788b026..7ebb5df3a31a2 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -1,47 +1,19 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * HAOC feature support - * - * Copyright (C) 2025 ZGCLAB - * Authors: Shu Hang - * Hu Bing - */ - -#include -#include -#include -#include -#include - -#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) - -/* - * scratch_reg would be changed, - * caller should dertimine if scratch_reg should be saved and restored. - */ -.macro DISABLE_WP scratch_reg:req - /* Disable write protection*/ - movq %cr0, %\scratch_reg - andq $(~X86_CR0_WP), %\scratch_reg - movq %\scratch_reg, %cr0 -.endm - -.macro ENABLE_WP scratch_reg:req - /* Enable write protection */ - movq %cr0, %\scratch_reg -1: - orq $X86_CR0_WP, %\scratch_reg - movq %\scratch_reg, %cr0 - testq $X86_CR0_WP, %\scratch_reg - je 1b -.endm - -/* - * IEE memory access gate. - * Kernel calls the gate to modify IEE-protected memory. - */ +#include "../gate.h" SYM_FUNC_START(iee_rw_gate) + pushq %rbp + movq %rsp, %rbp + + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_rw + +.L_haoc_disabled_rw: + call iee_dispatch + popq %rbp + jmp __x86_return_thunk + +.L_haoc_enabled_rw: /* save Interrupt flag */ pushfq /* close irq*/ @@ -49,18 +21,24 @@ SYM_FUNC_START(iee_rw_gate) pushq %r12 - DISABLE_WP r12 +#ifdef CONFIG_PTP + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz rw_wp_enabled - /* switch to iee stack */ - movq %rsp, %r12 - movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp + call iee_dispatch - /* call iee func */ - leaq iee_funcs(%rip), %rax - call *(%rax, %rdi, 8) + popq %r12 + popfq + popq %rbp + jmp __x86_return_thunk - /* switch to kernel stack */ - movq %r12, %rsp +rw_wp_enabled: +#endif + DISABLE_WP r12 + + /* call iee func */ + call iee_dispatch ENABLE_WP r12 @@ -69,40 +47,70 @@ SYM_FUNC_START(iee_rw_gate) /* restore irq*/ popfq - jmp __x86_return_thunk /* ret */ + popq %rbp + jmp __x86_return_thunk SYM_FUNC_END(iee_rw_gate) +#if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) EXPORT_SYMBOL(iee_rw_gate) +#endif #ifdef CONFIG_IEE_SIP SYM_FUNC_START(iee_rwx_gate) + pushq %rbp + movq %rsp, %rbp + + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_rwx + +.L_haoc_disabled_rwx: + call _iee_si_handler + popq %rbp + jmp __x86_return_thunk + +.L_haoc_enabled_rwx: pushq %r12 - + /* save Interrupt flag*/ pushfq /* close irq */ cli - + +#ifdef CONFIG_PTP + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz rwx_wp_enabled + + movq %cr4, %rax /* rax -> cr4 */ + andq $(~X86_CR4_SMEP), %rax + movq %rax, %cr4 + + call _iee_si_handler + + movq %cr4, %rax /* rax -> cr4 */ +3: orq $X86_CR4_SMEP_SMAP, %rax + movq %rax, %cr4 + andq $X86_CR4_SMEP_SMAP, %rax + cmpq $X86_CR4_SMEP_SMAP, %rax + jnz 3 + + popfq + popq %r12 + popq %rbp + jmp __x86_return_thunk /* ret */ + +rwx_wp_enabled: +#endif /* set SMEP=0 to enable supervisor-mode exec user-mode insn */ movq %cr4, %rax /* rax -> cr4 */ andq $(~X86_CR4_SMEP), %rax movq %rax, %cr4 - + DISABLE_WP r12 - - movq %rsp, %r12 - /* If iee hasn't been initialized, skip stack switch. */ - cmpb $0, iee_init_done(%rip) - jz 2f - - /* switch to iee stack */ - movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp - + 2: call _iee_si_handler - /* switch to kernel stack. If iee hasn't been initialized, skip switch*/ - movq %r12, %rsp - + ENABLE_WP r12 - + /* set SMEP=1 to disable supervisor-mode exec user-mode insn */ movq %cr4, %rax /* rax -> cr4 */ 1: orq $X86_CR4_SMEP_SMAP, %rax @@ -110,11 +118,12 @@ SYM_FUNC_START(iee_rwx_gate) andq $X86_CR4_SMEP_SMAP, %rax cmpq $X86_CR4_SMEP_SMAP, %rax jnz 1 - + /* restore irq*/ popfq - + popq %r12 + popq %rbp jmp __x86_return_thunk /* ret */ SYM_FUNC_END(iee_rwx_gate) -#endif \ No newline at end of file +#endif diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 7ac4704769f4d..ebe34a511d728 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -7,19 +7,44 @@ * Hu Bing */ -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include #include #include +#include +#include #ifdef CONFIG_IEE_SIP #include #endif +#ifdef CONFIG_IEE_PTRP +#include +#endif +#ifdef CONFIG_PTP +#include +#include +#endif +#ifdef CONFIG_CREDP +extern void credp_init(void); +#endif + +#ifdef CONFIG_IEE_SELINUX_P +extern void iee_selinuxp_init(void); +#endif +#ifdef CONFIG_PTP +extern void haoc_ptp_init(void); +#endif +#ifdef CONFIG_VARP +extern void varp_init(void); +#endif /* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ unsigned long IEE_OFFSET = 0x200000000000; + #ifdef CONFIG_IEE_SIP bool iee_init_done __iee_si_data; #else @@ -27,7 +52,24 @@ bool iee_init_done; #endif DEFINE_PER_CPU(struct iee_stack, iee_stacks); -static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, unsigned long end) +bool __ro_after_init haoc_enabled; +EXPORT_SYMBOL(haoc_enabled); + +bool __ro_after_init haoc_init_done; + +/* + * Deepin keeps HAOC disabled by default. The boot parameter enables the + * three-stage IEE initialization when set to a true value, for example haoc=on. + */ +static bool cmd_haoc_enabled; +static int __init parse_cmd_haoc_enabled(char *str) +{ + return kstrtobool(str, &cmd_haoc_enabled); +} +early_param("haoc", parse_cmd_haoc_enabled); + +static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, + unsigned long end) { void *p; pmd_t *pmd; @@ -37,10 +79,10 @@ static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, uns addr = ALIGN_DOWN(addr, PMD_SIZE); phys = __iee_pa(addr); - pgprot_shadow_pmd = __pgprot(pgprot_val(PAGE_KERNEL_LARGE) & (~__RW) & (~___D)); + pgprot_shadow_pmd = __pgprot(pgprot_val(PAGE_KERNEL_LARGE) & ~__RW & ~___D); if (pud_none(*pud)) { - p = alloc_low_pages(1); + p = ptdesc_to_virt(pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0)); pud_populate(&init_mm, pud, p); } @@ -52,14 +94,15 @@ static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, uns } while (pmd++, addr = pmd_next, addr != end); } -static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, unsigned long end) +static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, + unsigned long end) { void *p; pud_t *pud; unsigned long pud_next; if (p4d_none(*p4d)) { - p = alloc_low_pages(1); + p = ptdesc_to_virt(pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0)); p4d_populate(&init_mm, p4d, p); } @@ -72,14 +115,15 @@ static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, uns } while (pud++, addr = pud_next, addr != end); } -static void __init _iee_mapping_populate_pgd(pgd_t *pgd, unsigned long addr, unsigned long end) +static void __init _iee_mapping_populate_pgd(pgd_t *pgd, unsigned long addr, + unsigned long end) { void *p; p4d_t *p4d; unsigned long p4d_next; if (pgd_none(*pgd)) { - p = alloc_low_pages(1); + p = ptdesc_to_virt(pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0)); pgd_populate(&init_mm, pgd, p); } @@ -97,7 +141,6 @@ static void __init _iee_init_mapping(phys_addr_t start_paddr, phys_addr_t end_pa unsigned long addr = (unsigned long)__phys_to_iee(start_paddr); unsigned long end = (unsigned long)__phys_to_iee(end_paddr); unsigned long pgd_next; - pgd_t *pgd = pgd_offset_k(addr); spin_lock(&pgd_lock); @@ -110,12 +153,17 @@ static void __init _iee_init_mapping(phys_addr_t start_paddr, phys_addr_t end_pa spin_unlock(&pgd_lock); } -static void __init _iee_mapping_init(void) +static void __init iee_init_offset(void) +{ + if (pgtable_l5_enabled()) + IEE_OFFSET = 0x40000000000000; +} + +static void __init iee_init_mapping(void) { struct memblock_region *r; unsigned long start_pfn, end_pfn; phys_addr_t start_paddr, end_paddr; - unsigned long nr_pages = 0; for_each_mem_region(r) { start_pfn = memblock_region_memory_base_pfn(r); @@ -124,8 +172,6 @@ static void __init _iee_mapping_init(void) start_paddr = PFN_PHYS(start_pfn); end_paddr = PFN_PHYS(end_pfn); - nr_pages += end_pfn - start_pfn; - pr_info("IEE: mapping iee mapping [mem %#010lx-%#010lx]\n", (unsigned long)start_paddr, (unsigned long)end_paddr); @@ -134,52 +180,115 @@ static void __init _iee_mapping_init(void) pr_info("IEE: IEE shadow mapping init done"); } -static void __init _iee_stack_init(void) +static void __init iee_init_stack(void) { int cpu; struct iee_stack *iee_stack; - void *stack_base; + unsigned long stack_base; struct page *page; for_each_possible_cpu(cpu) { - stack_base = (void *)page_address(alloc_pages(GFP_KERNEL, IEE_STACK_ORDER)); iee_stack = per_cpu_ptr(&iee_stacks, cpu); page = alloc_pages(GFP_KERNEL, IEE_STACK_ORDER); - iee_stack->stack = (void *)page_address(page) + PAGE_SIZE * (1 << IEE_STACK_ORDER); - pr_info("IEE: cpu %d, iee_stack 0x%lx", cpu, (unsigned long)iee_stack->stack); - set_memory_ro((unsigned long)stack_base, (1 << IEE_STACK_ORDER)); + if (!page) + panic("IEE: no memory for iee stack"); + + stack_base = (unsigned long)page_address(page); + iee_stack->stack = (void *)stack_base + PAGE_SIZE * (1 << IEE_STACK_ORDER); + pr_info("IEE: cpu %d, iee_stack 0x%lx", cpu, + (unsigned long)iee_stack->stack); } } -static void __init _iee_offset_init(void) +static void __init iee_stack_set_ro(void) { + int cpu; + struct iee_stack *iee_stack; + unsigned long stack_base; + + for_each_possible_cpu(cpu) { + iee_stack = per_cpu_ptr(&iee_stacks, cpu); + stack_base = (unsigned long)iee_stack->stack - PAGE_SIZE * (1 << IEE_STACK_ORDER); + set_iee_pages(stack_base, 1 << IEE_STACK_ORDER, IEE_SIP_DATA); + } +} + +static bool __init check_haoc_hardware_support(void) +{ + return cpu_feature_enabled(X86_FEATURE_SMAP) && + cpu_feature_enabled(X86_FEATURE_SMEP); +} + +void __init iee_early_init(void) +{ + if (!cmd_haoc_enabled) + return; + + if (!check_haoc_hardware_support()) { + pr_info("HAOC disabled because SMAP or SMEP is not available"); + return; + } + +#ifdef CONFIG_PTP + iee_cache_init(&pgd_cache, PGD_ALLOCATION_ORDER, 1, IEE_PGTABLE, + CONFIG_PTP_RESERVE_ORDER); if (pgtable_l5_enabled()) - IEE_OFFSET = 0x40000000000000; + iee_cache_init(&pg_cache, 0, 4, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); + else + iee_cache_init(&pg_cache, 0, 3, IEE_PGTABLE, CONFIG_PTP_RESERVE_ORDER); +#ifdef CONFIG_PTP_S + iee_cache_init(&pg_user_cache, 0, 3, IEE_USER_PGTABLE, CONFIG_PTP_RESERVE_ORDER); +#endif /* CONFIG_PTP_S */ +#endif } void __init iee_init(void) { - _iee_offset_init(); - _iee_mapping_init(); - _iee_stack_init(); + if (!cmd_haoc_enabled) + return; + + if (!check_haoc_hardware_support()) + return; + + iee_init_offset(); + iee_init_mapping(); + iee_init_bitmap(); + iee_init_stack(); + + /* + * HAOC temporarily clears CR0.WP in the write gate. Disable CET while + * HAOC is active to avoid the CR4.CET and CR0.WP architectural conflict. + */ + cet_disable(); + + haoc_enabled = true; +#ifdef CONFIG_IEE_PTRP + iee_prepare_init_task_token(); +#endif + pr_info("HAOC initialized"); } -bool __ro_after_init haoc_enabled; -EXPORT_SYMBOL(haoc_enabled); +void __init iee_post_init(void) +{ + if (!cmd_haoc_enabled || !haoc_enabled) + return; + + iee_stack_set_ro(); +#ifdef CONFIG_PTP + haoc_ptp_init(); +#endif #ifdef CONFIG_IEE_SIP -extern unsigned long cr4_pinned_mask; + iee_sip_init(); #endif -static int __init parse_haoc_enabled(char *str) -{ - int ret = kstrtobool(str, &haoc_enabled); - #ifdef CONFIG_IEE_SIP - if(haoc_enabled) - { - cr4_pinned_mask = - X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET; - } - #endif - return ret; +#ifdef CONFIG_CREDP + credp_init(); +#endif +#ifdef CONFIG_IEE_SELINUX_P + iee_selinuxp_init(); +#endif + +#ifdef CONFIG_VARP + varp_init(); +#endif + haoc_init_done = true; } -early_param("haoc", parse_haoc_enabled); diff --git a/arch/x86/kernel/haoc/iee/iee-si.c b/arch/x86/kernel/haoc/iee/iee-si.c index 214c7617cfed0..393f683ae9a80 100644 --- a/arch/x86/kernel/haoc/iee/iee-si.c +++ b/arch/x86/kernel/haoc/iee/iee-si.c @@ -4,6 +4,7 @@ #include #include #include +#include unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) { @@ -36,9 +37,25 @@ unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) break; } case IEE_WRITE_CR4: { + unsigned long bits_changed = 0; + const unsigned long check_mask = + cr4_pinned_mask & ~X86_CR4_SMEP; + const unsigned long check_bits = + cr4_pinned_bits & ~X86_CR4_SMEP; + val = va_arg(pArgs, u64); - val &= ~(X86_CR4_SMEP); + val &= ~X86_CR4_SMEP; + if (static_branch_likely(&cr_pinning)) { + if (unlikely((val & check_mask) != check_bits)) { + bits_changed = (val & check_mask) ^ check_bits; + val = (val & ~check_mask) | check_bits; + } + } asm volatile("mov %0,%%cr4" : "+r" (val) : : "memory"); + if (static_branch_likely(&cr_pinning)) + WARN_ONCE(bits_changed, + "pinned CR4 bits changed: 0x%lx!?\n", + bits_changed); break; } case IEE_LOAD_IDT: { @@ -104,12 +121,18 @@ void __init iee_sip_init(void) _iee_set_kernel_upage((unsigned long)__va(__pa(addr))); } iee_init_done = true; - /* Map .iee.data as RO pages */ + /* Map .iee.si_data as IEE SIP data pages. */ start = (unsigned long)__iee_si_data_start; end = (unsigned long)__iee_si_data_end; num_pages = (end - start) / PAGE_SIZE; - set_memory_ro(start, num_pages); + set_iee_pages(start, num_pages, IEE_SIP_DATA); +#ifdef CONFIG_IEE_SELINUX_P + start = (unsigned long)__iee_selinux_data_start; + end = (unsigned long)__iee_selinux_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_pages(start, num_pages, IEE_SIP_DATA); +#endif /* All initialization is done. Do some simple tests. */ pr_info("IEE: testing iee_exec_entry si_test..."); iee_sip_test(); -} \ No newline at end of file +} diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c index b39ba03d9c565..9a2ca697ca0c1 100644 --- a/arch/x86/kernel/haoc/iee/iee-token.c +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -1,10 +1,40 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include +#include +#include +#include +#include +#include #include "slab.h" +/* + * Slab-allocated task_struct objects use dedicated token pages, but init_task + * is a kernel image symbol whose IEE alias initially maps its own page. Give + * init_task a dedicated token page before its first credential operation. + */ +void __init iee_prepare_init_task_token(void) +{ + unsigned long token = (unsigned long)__kimg_to_iee(&init_task); + unsigned long token_page; + unsigned int order = 0; + + if (ALIGN(token + sizeof(struct task_token), PAGE_SIZE) != + ALIGN(token + 1, PAGE_SIZE)) + order = 1; + + token_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!token_page) + panic("IEE: failed to allocate token page for init_task\n"); + + iee_set_token_page_valid(ALIGN_DOWN(token, PAGE_SIZE), token_page, + order); + iee_validate_token(&init_task); +} + void iee_set_token_page_valid(unsigned long token, unsigned long token_page, - unsigned int order) + unsigned int order) { set_memory_4k(token, 1 << order); set_memory_4k(token_page, 1 << order); @@ -25,6 +55,9 @@ void iee_set_token_page_valid(unsigned long token, unsigned long token_page, pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_SET_TOKEN, token_ptep, token_page_ptep, token_page, order); +#else for (int i = 0; i < (0x1 << order); i++) { pte_t pte = READ_ONCE(*token_ptep); @@ -38,6 +71,7 @@ void iee_set_token_page_valid(unsigned long token, unsigned long token_page, token_ptep++; token_page_ptep++; } +#endif flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); flush_tlb_kernel_range(token_page, @@ -65,6 +99,9 @@ void iee_set_token_page_invalid(unsigned long token, unsigned long __unused, pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); +#ifdef CONFIG_PTP + iee_rw_gate(IEE_OP_UNSET_TOKEN, token_ptep, token_page_ptep, token, order); +#else for (int i = 0; i < (0x1 << order); i++) { pte_t pte = READ_ONCE(*token_ptep); @@ -78,6 +115,7 @@ void iee_set_token_page_invalid(unsigned long token, unsigned long __unused, token_ptep++; token_page_ptep++; } +#endif free_pages(token_page, order); flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); flush_tlb_kernel_range(token_page, @@ -91,9 +129,11 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, return slab; struct folio *folio = slab_folio(slab); + unsigned int task_order = order; + unsigned int token_order = IEE_TOKEN_ORDER(task_order); unsigned long token_addr = __slab_to_iee(slab); unsigned long alloc_token = - __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + __get_free_pages(GFP_KERNEL | __GFP_ZERO, token_order); /* Allocation of task_struct and token pages must be done at the same time. */ if (!alloc_token) { @@ -105,36 +145,183 @@ struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, /* Make the mapping reset visible before clearing the flag */ smp_wmb(); __folio_clear_slab(folio); - __free_pages((struct page *)folio, order); + __free_pages((struct page *)folio, task_order); return NULL; } /* Map allocated token pages to token addresses. */ - iee_set_token_page_valid(token_addr, alloc_token, order); + iee_set_token_page_valid(token_addr, alloc_token, token_order); return slab; } void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)__addr_to_token(tsk); token->pgd = pgd; } void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)__addr_to_token(tsk); token->pgd = NULL; +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = NULL; +#endif + seqcount_init(&token->seq); token->valid = false; } void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) { - struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + struct task_token *token = (struct task_token *)__addr_to_token(tsk); if (token->valid) pr_err("IEE: validate token for multiple times."); +#ifdef CONFIG_CREDP + token->new_cred = NULL; + token->curr_cred = tsk->cred; +#endif + seqcount_init(&token->seq); token->valid = true; -} \ No newline at end of file +} + +#ifdef CONFIG_PTP +void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token, unsigned int order) +{ + token_ptep = (pte_t *)__addr_to_iee(token_ptep); + token_page_ptep = (pte_t *)__addr_to_iee(token_page_ptep); + + for (int i = 0; i < (0x1 << order); i++) { + pte_t pte = READ_ONCE(*token_ptep); + + pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | + (__phys_to_pfn(__iee_pa(token + i * PAGE_SIZE)) << PAGE_SHIFT)); + WRITE_ONCE(*token_ptep, pte); + pte = READ_ONCE(*token_page_ptep); + pte = __pte(pte_val(pte) | ___D | __RW); + WRITE_ONCE(*token_page_ptep, pte); + token_ptep++; + token_page_ptep++; + } +} + +void _iee_set_token(unsigned long __unused, pte_t *token_ptep, + pte_t *token_page_ptep, unsigned long token_page, unsigned int order) +{ + token_ptep = (pte_t *)__addr_to_iee(token_ptep); + token_page_ptep = (pte_t *)__addr_to_iee(token_page_ptep); + + for (int i = 0; i < (0x1 << order); i++) { + pte_t pte = READ_ONCE(*token_ptep); + + pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK)) | + (__phys_to_pfn(__pa(token_page + i * PAGE_SIZE)) << PAGE_SHIFT)); + WRITE_ONCE(*token_ptep, pte); + pte = READ_ONCE(*token_page_ptep); + pte = __pte((pte_val(pte) & ~__RW) & ~___D); + WRITE_ONCE(*token_page_ptep, pte); + token_ptep++; + token_page_ptep++; + } +} +#endif + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +void iee_verify_token(struct task_struct *tsk) +{ + struct task_token *token; +#ifdef CONFIG_CREDP + const struct cred *old_cred; + const struct cred *token_cred; +#endif + bool valid; + unsigned int seq; + + if (unlikely(!haoc_init_done)) + return; + + if (!haoc_enabled) + return; + + if (tsk == &init_task) + return; + + token = (struct task_token *)__addr_to_token(tsk); + do { + seq = read_seqcount_begin(&token->seq); + valid = token->valid; +#ifdef CONFIG_CREDP + old_cred = tsk->cred; + token_cred = token->curr_cred; +#endif + } while (read_seqcount_retry(&token->seq, seq)); + + if (!valid) + panic("IEE: (%s) Invalid Token.", __func__); +#ifdef CONFIG_CREDP + if (token_cred != old_cred) + panic("IEE: (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx", + __func__, (u64)token_cred, (u64)old_cred); +#endif +} + +static void check_all_threads(void) +{ + struct task_struct *task; + + rcu_read_lock(); + for_each_process(task) + iee_verify_token(task); + rcu_read_unlock(); +} + +static int checker_thread(void *data) +{ + int check_interval_ms = 500; + + while (!kthread_should_stop()) { + check_all_threads(); + msleep_interruptible(check_interval_ms); + } + pr_info("[IEE] Kernel thread exiting\n"); + return 0; +} + +static int __init thread_checker_init(void) +{ + struct task_struct *checker_task; + + if (!haoc_enabled) + return 0; + + pr_info("IEE: Initializing thread checker\n"); + checker_task = kthread_run(checker_thread, NULL, "thread_credp_cycle"); + if (IS_ERR(checker_task)) { + pr_err("IEE: Failed to create thread checker task: %ld\n", + PTR_ERR(checker_task)); + return PTR_ERR(checker_task); + } + + pr_info("IEE: Thread checker started successfully\n"); + return 0; +} + +late_initcall(thread_checker_init); + +void iee_verify_pgd(struct task_struct *next) +{ + if (haoc_enabled && next != &init_task) { + struct task_token *token; + + token = (struct task_token *)__addr_to_token(next); + if (token->pgd != next->mm->pgd) + panic("IEE Pgd Error: next_pgd: 0x%lx, token_pgd: 0x%lx", + (unsigned long)next->mm->pgd, (unsigned long)token->pgd); + } +} +#endif /* CONFIG_IEE_PTRP && !CONFIG_IEE_PTRP_W */ diff --git a/arch/x86/kernel/haoc/iee/iee.c b/arch/x86/kernel/haoc/iee/iee.c index 35c54c3352ac3..55e50c0b0cb79 100644 --- a/arch/x86/kernel/haoc/iee/iee.c +++ b/arch/x86/kernel/haoc/iee/iee.c @@ -8,6 +8,10 @@ */ #include +#ifdef CONFIG_IEE_PTRP +#include +#endif +#include void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n) { @@ -30,3 +34,19 @@ unsigned long _iee_test_and_clear_bit(unsigned long __unused, long nr, unsigned instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); return arch_test_and_clear_bit(nr, addr); } + +#ifdef CONFIG_IEE_PTRP +struct task_token *iee_get_task_token(struct task_struct *task) +{ + unsigned long slab_addr; + unsigned long task_addr; + unsigned int index; + + slab_addr = (unsigned long)page_to_virt(virt_to_head_page(task)); + task_addr = (unsigned long)page_to_virt(virt_to_page(task)); + index = (task_addr - slab_addr) / PAGE_SIZE; + + return (struct task_token *)((unsigned long)__phys_to_iee(__pa(slab_addr)) + + index * IEE_TOKEN_BLOCK_SIZE); +} +#endif diff --git a/arch/x86/kernel/haoc/keyp/Makefile b/arch/x86/kernel/haoc/keyp/Makefile new file mode 100644 index 0000000000000..307cebd57ff71 --- /dev/null +++ b/arch/x86/kernel/haoc/keyp/Makefile @@ -0,0 +1 @@ +obj-y += keyp.o \ No newline at end of file diff --git a/arch/x86/kernel/haoc/keyp/keyp.c b/arch/x86/kernel/haoc/keyp/keyp.c new file mode 100644 index 0000000000000..b371548d08633 --- /dev/null +++ b/arch/x86/kernel/haoc/keyp/keyp.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +struct watch_list; + +static inline void iee_verify_key_type(const struct key *key) +{ + iee_verify_type(__pa(key), IEE_KEY, "key"); +} + +unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, + long nr, int flag) +{ + iee_verify_key_type(key); + switch (flag) { + case SET_BIT_OP: { + set_bit(nr, &key->flags); + break; + } + case TEST_AND_CLEAR_BIT: { + return test_and_clear_bit(nr, &key->flags); + } + case TEST_AND_SET_BIT: { + return test_and_set_bit(nr, &key->flags); + } + } + return 0; +} + +void _iee_set_key_restrict_link(unsigned long __unused, + struct key *key, + struct key_restriction *restrict_link) +{ + iee_verify_key_type(key); + key->restrict_link = restrict_link; +} + +void _iee_set_key_magic(unsigned long __unused, struct key *key, + unsigned int magic) +{ +#ifdef KEY_DEBUGGING + iee_verify_key_type(key); + key->magic = magic; +#endif +} + +void _iee_set_key_flags(unsigned long __unused, struct key *key, + unsigned long flags) +{ + iee_verify_key_type(key); + key->flags = flags; +} + +void _iee_set_key_index_key(unsigned long __unused, + struct key *key, + struct keyring_index_key *index_key) +{ + iee_verify_key_type(key); + key->index_key = *index_key; +} + +void _iee_set_key_hash(unsigned long __unused, struct key *key, + unsigned long hash) +{ + iee_verify_key_type(key); + key->hash = hash; +} + +void _iee_set_key_len_desc(unsigned long __unused, struct key *key, + unsigned long len_desc) +{ + iee_verify_key_type(key); + key->len_desc = len_desc; +} + +void _iee_set_key_type(unsigned long __unused, struct key *key, + struct key_type *type) +{ + iee_verify_key_type(key); + key->type = type; +} + +void _iee_set_key_domain_tag(unsigned long __unused, + struct key *key, + struct key_tag *domain_tag) +{ + iee_verify_key_type(key); + key->domain_tag = domain_tag; +} + +void _iee_set_key_description(unsigned long __unused, + struct key *key, char *description) +{ + iee_verify_key_type(key); + key->description = description; +} + +void _iee_set_key_uid(unsigned long __unused, struct key *key, + kuid_t uid) +{ + iee_verify_key_type(key); + key->uid = uid; +} + +void _iee_set_key_gid(unsigned long __unused, struct key *key, + kgid_t gid) +{ + iee_verify_key_type(key); + key->gid = gid; +} + +void _iee_set_key_perm(unsigned long __unused, struct key *key, + key_perm_t perm) +{ + iee_verify_key_type(key); + key->perm = perm; +} + +void _iee_set_key_quotalen(unsigned long __unused, struct key *key, + unsigned short quotalen) +{ + iee_verify_key_type(key); + key->quotalen = quotalen; +} + +void _iee_set_key_datalen(unsigned long __unused, struct key *key, + unsigned short datalen) +{ + iee_verify_key_type(key); + key->datalen = datalen; +} + +void _iee_set_key_state(unsigned long __unused, struct key *key, + short state) +{ + iee_verify_key_type(key); + smp_store_release(&key->state, state); +} + +void _iee_set_key_user(unsigned long __unused, struct key *key, + struct key_user *user) +{ + iee_verify_key_type(key); + key->user = user; +} + +void _iee_set_key_security(unsigned long __unused, struct key *key, + void *security) +{ + iee_verify_key_type(key); + key->security = security; +} + +void _iee_set_key_expiry(unsigned long __unused, struct key *key, + time64_t expiry) +{ + iee_verify_key_type(key); + key->expiry = expiry; +} + +void _iee_set_key_revoked_at(unsigned long __unused, + struct key *key, time64_t revoked_at) +{ + iee_verify_key_type(key); + key->revoked_at = revoked_at; +} + +void _iee_set_key_last_used_at(unsigned long __unused, + struct key *key, + time64_t last_used_at) +{ + iee_verify_key_type(key); + key->last_used_at = last_used_at; +} + +unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, + int n, int flag) +{ + iee_verify_key_type(key); + switch (flag) { + case REFCOUNT_INC: { + refcount_inc(&key->usage); + break; + } + case REFCOUNT_SET: { + refcount_set(&key->usage, n); + break; + } + case REFCOUNT_DEC_AND_TEST: { + return refcount_dec_and_test(&key->usage); + } + case REFCOUNT_INC_NOT_ZERO: { + return refcount_inc_not_zero(&key->usage); + } + } + return 0; +} + +void _iee_set_key_serial(unsigned long __unused, struct key *key, + key_serial_t serial) +{ + iee_verify_key_type(key); + key->serial = serial; +} + +void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers) +{ +#ifdef CONFIG_KEY_NOTIFICATIONS + iee_verify_key_type(key); + key->watchers = watchers; +#endif +} + +void _iee_set_key_union(unsigned long __unused, struct key *key, + struct key_union *key_union) +{ + iee_verify_key_type(key); + key->graveyard_link.next = (struct list_head *)key_union; +} + +void _iee_set_key_struct(unsigned long __unused, struct key *key, + struct key_struct *key_struct) +{ + iee_verify_key_type(key); + key->name_link.prev = (struct list_head *)key_struct; +} + +void _iee_set_key_payload(unsigned long __unused, struct key *key, + union key_payload *key_payload) +{ + iee_verify_key_type(key); + key->name_link.next = (struct list_head *)key_payload; +} diff --git a/arch/x86/kernel/haoc/ptp/Makefile b/arch/x86/kernel/haoc/ptp/Makefile new file mode 100644 index 0000000000000..ab30a58e0e739 --- /dev/null +++ b/arch/x86/kernel/haoc/ptp/Makefile @@ -0,0 +1 @@ +obj-y += ptp.o ptp-gate.o diff --git a/arch/x86/kernel/haoc/ptp/ptp-gate.S b/arch/x86/kernel/haoc/ptp/ptp-gate.S new file mode 100644 index 0000000000000..1655465b175e6 --- /dev/null +++ b/arch/x86/kernel/haoc/ptp/ptp-gate.S @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include "../gate.h" + +SYM_FUNC_START(ptp_xchg) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_xchg + +.L_haoc_disabled_xchg: + xchg %rsi, (%rdi) + movq %rsi, %rax + jmp __x86_return_thunk + +.L_haoc_enabled_xchg: + /* save RFLAGS, close irq */ + pushfq + cli + + pushq %r12 + + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz xchg_wp_enabled + + xchg %rsi, (%rdi) + movq %rsi, %rax + + popq %r12 + popfq + jmp __x86_return_thunk + +xchg_wp_enabled: + DISABLE_WP r12 + + xchg %rsi, (%rdi) + movq %rsi, %rax + + ENABLE_WP r12 + + popq %r12 + + /* restore RFLAGS*/ + popfq + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(ptp_xchg) + +SYM_FUNC_START(ptp_try_cmpxchg) + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_cmpxchg + +.L_haoc_disabled_cmpxchg: + movq %rsi, %rax + lock cmpxchgq %rdx, (%rdi) + jmp __x86_return_thunk + +.L_haoc_enabled_cmpxchg: + /* save RFLAGS, close irq */ + pushfq + cli + + pushq %r12 + + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz cmpxchg_wp_enabled + + movq %rsi, %rax + lock cmpxchgq %rdx, (%rdi) + + popq %r12 + popfq + jmp __x86_return_thunk + +cmpxchg_wp_enabled: + DISABLE_WP r12 + + movq %rsi, %rax + lock cmpxchgq %rdx, (%rdi) + + ENABLE_WP r12 + + popq %r12 + /* restore RFLAGS*/ + popfq + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(ptp_try_cmpxchg) + +SYM_FUNC_START(ptp_rw_gate) + pushq %rbp + movq %rsp, %rbp + + cmpb $0, haoc_enabled(%rip) + jnz .L_haoc_enabled_rw + +.L_haoc_disabled_rw: + call iee_dispatch + popq %rbp + jmp __x86_return_thunk + +.L_haoc_enabled_rw: + /* save Interrupt flag */ + pushfq + /* close irq*/ + cli + + pushq %r12 + + movq PER_CPU_VAR(iee_cr0s), %r12 + testq %r12, %r12 + jz ptprw_wp_enabled + + call iee_dispatch + + popq %r12 + popfq + popq %rbp + jmp __x86_return_thunk + +ptprw_wp_enabled: + DISABLE_WP r12 + + /* call iee func */ + call iee_dispatch + + ENABLE_WP r12 + + popq %r12 + + /* restore irq*/ + popfq + + popq %rbp + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(ptp_rw_gate) +EXPORT_SYMBOL(ptp_rw_gate) diff --git a/arch/x86/kernel/haoc/ptp/ptp.c b/arch/x86/kernel/haoc/ptp/ptp.c new file mode 100644 index 0000000000000..e042d185ae22f --- /dev/null +++ b/arch/x86/kernel/haoc/ptp/ptp.c @@ -0,0 +1,611 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define __pte_to_phys(pte) (pte_pfn(pte) << PAGE_SHIFT) +#define __pmd_to_phys(pmd) (__pte_to_phys(__pte(pmd_val(pmd)))) +#define __pud_to_phys(pud) (__pte_to_phys(__pte(pud_val(pud)))) +#define __p4d_to_phys(p4d) (__pte_to_phys(__pte(p4d_val(p4d)))) +#define __pgd_to_phys(pgd) (__pte_to_phys(__pte(pgd_val(pgd)))) + +static void __init _ptp_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end) +{ + pmd_t pmd = READ_ONCE(*pmdp); + unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); +} + +static void __init _ptp_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end) +{ + unsigned long next; + pud_t pud = READ_ONCE(*pudp); + pmd_t *pmdp; + pmd_t pmd; + unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); + pmdp = pmd_offset(pudp, addr); + do { + next = pmd_addr_end(addr, end); + pmd = READ_ONCE(*pmdp); + if (pmd_val(pmd) & _PSE) + continue; + else + _ptp_set_pte_table_ro(pmdp, addr, next); + } while (pmdp++, addr = next, addr != end); +} + +static void __init _ptp_set_pud_table_ro(p4d_t *p4dp, unsigned long addr, unsigned long end) +{ + unsigned long next; + p4d_t p4d = READ_ONCE(*p4dp); + pud_t *pudp; + pud_t pud; + unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); + pudp = pud_offset(p4dp, addr); + do { + next = pud_addr_end(addr, end); + pud = READ_ONCE(*pudp); + if (pud_val(pud) & _PSE) { + // _PSE = 1 means a page, not a table + continue; + } else { + _ptp_set_pmd_table_ro(pudp, addr, next); + } + } while (pudp++, addr = next, addr != end); +} + +static void __init _ptp_set_p4d_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end) +{ + unsigned long next; + pgd_t pgd = READ_ONCE(*pgdp); + p4d_t *p4dp; + p4d_t p4d; + unsigned long logical_addr = (unsigned long)__va(__pgd_to_phys(pgd)); + + set_iee_page(logical_addr, 1, IEE_FIXED_PGTABLE); + p4dp = p4d_offset(pgdp, addr); + do { + next = p4d_addr_end(addr, end); + p4d = READ_ONCE(*p4dp); + /* No 512 GiB huge pages yet */ + _ptp_set_pud_table_ro(p4dp, addr, next); + } while (p4dp++, addr = next, addr != end); +} + +static void __init _ptp_mark_iee_pgtable_for_one_region_ro(pgd_t *pgdir, + unsigned long va_start, unsigned long va_end) +{ + unsigned long addr, end, next; + pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); + + addr = va_start & PAGE_MASK; + end = PAGE_ALIGN(va_end); + + do { + next = pgd_addr_end(addr, end); + _ptp_set_p4d_table_ro(pgdp, addr, next); + } while (pgdp++, addr = next, addr != end); +} + +static void __init ptp_mark_all_pgtable_ro(void) +{ + unsigned long logical_addr; + phys_addr_t start, end; + u64 i; + pgd_t *pgdp; + + // handing 1-level page table swapper_pg_dir + pgdp = swapper_pg_dir; + set_iee_page((unsigned long)swapper_pg_dir, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + + // handling 2/3/4/5-level page table for kernel + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)_text, (unsigned long)_etext); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__start_rodata, (unsigned long)__end_rodata); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)_sdata, (unsigned long)_edata); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__bss_start, (unsigned long)__bss_stop); + + // handling 2/3/4/5-level statically allocated page table + #ifdef CONFIG_X86_5LEVEL + set_iee_page((unsigned long)level4_kernel_pgt, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level4_kernel_pgt)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + #endif + + set_iee_page((unsigned long)level3_kernel_pgt, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level3_kernel_pgt)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + + set_iee_page((unsigned long)level2_kernel_pgt, 1, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level2_kernel_pgt)); + set_iee_page(logical_addr, 1, IEE_PGTABLE); + + set_iee_page((unsigned long)level2_fixmap_pgt, FIXMAP_PMD_NUM, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level2_fixmap_pgt)); + set_iee_page(logical_addr, FIXMAP_PMD_NUM, IEE_PGTABLE); + + set_iee_page((unsigned long)level1_fixmap_pgt, FIXMAP_PMD_NUM, IEE_PGTABLE); + logical_addr = (unsigned long)__va(__pa_symbol(level1_fixmap_pgt)); + set_iee_page(logical_addr, FIXMAP_PMD_NUM, IEE_PGTABLE); + + // handling 2/3/4-level page table for logical mem and iee + for_each_mem_range(i, &start, &end) { + if (start >= end) + break; + /* + * The linear map must allow allocation tags reading/writing + * if MTE is present. Otherwise, it has the same attributes as + * PAGE_KERNEL. + */ + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__va(start), (unsigned long)__va(end)); + _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, + (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); + } + pr_info("PTP: mark all kernel page tables ro done"); +} + +static inline void check_addr_range_ro(unsigned long addr_start, unsigned long addr_end, + unsigned long start, unsigned long end, pte_t pte, const char *msg) +{ + if (unlikely(!haoc_init_done)) + return; + + if (unlikely((addr_start >= start && addr_start < end) + || (addr_end > start && addr_end < end))) + if (unlikely(pte_write(pte))) + panic("IEE Error: Are you trying to write %s: [0x%lx, 0x%lx]?", + msg, addr_start, addr_end); +} + +static inline void check_dep_and_ro(unsigned long image_addr, unsigned long size, pte_t pte) +{ + if (unlikely(!haoc_init_done)) + return; + + if (unlikely(!kernel_set_to_readonly)) + return; + + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)_stext, (unsigned long)_etext, pte, "text"); + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); +#ifdef CONFIG_IEE_SIP + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_si_text_start, (unsigned long)__iee_si_text_end, pte, "iee_si_text"); + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_si_data_start, (unsigned long)__iee_si_data_end, pte, "iee_si_data"); +#endif +#ifdef CONFIG_IEE_SELINUX_P + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_selinux_data_start, (unsigned long)__iee_selinux_data_end, pte, "iee_si_data"); +#endif +} + +static inline bool preserves_existing_writable_mapping(pte_t *ptep, pte_t pte) +{ + pte_t old_pte = READ_ONCE(*ptep); + + return pte_present(old_pte) && pte_write(old_pte) && + pte_pfn(old_pte) == pte_pfn(pte); +} + +static inline bool +pmd_preserves_existing_writable_mapping(pmd_t *pmdp, pmd_t pmd) +{ + pmd_t old_pmd = READ_ONCE(*pmdp); + + return pmd_present(old_pmd) && pmd_leaf(old_pmd) && + pmd_write(old_pmd) && pmd_pfn(old_pmd) == pmd_pfn(pmd); +} + +static inline bool +pud_preserves_existing_writable_mapping(pud_t *pudp, pud_t pud) +{ + pud_t old_pud = READ_ONCE(*pudp); + + return pud_present(old_pud) && pud_leaf(old_pud) && + pud_write(old_pud) && pud_pfn(old_pud) == pud_pfn(pud); +} + +static inline void check_alias_mapping(phys_addr_t start, phys_addr_t end, + bool writable, + bool existing_writable_mapping) +{ + uint8_t type; + phys_addr_t addr; + + if (unlikely(!haoc_init_done)) + return; + + if (unlikely(PHYS_PFN(end) >= max_pfn)) + return; + + addr = start; + while (addr < end) { + type = iee_get_bitmap_type(addr); + if (unlikely(type != IEE_NORMAL && writable && + !existing_writable_mapping)) + panic("Detected Alias mapping of IEE data"); + addr += PAGE_SIZE; + } +} + +static inline int pte_smode(pte_t pte) +{ + return !(pte_flags(pte) & _PAGE_USER); +} + +static inline void check_sx(pte_t *ptep, pte_t pte) +{ + pte_t old_pte; + pteval_t diff; + + if (unlikely(!haoc_init_done)) + return; + old_pte = READ_ONCE(*ptep); + if (unlikely(!pte_present(old_pte))) + return; + + diff = pte_flags(old_pte) ^ pte_flags(pte); + if (pte_exec(pte) && pte_smode(pte) && + ((diff & _PAGE_USER) || (diff & _PAGE_NX))) { + pr_err("PTP: old_pte=%lx, pte=%lx\n", pte_flags(old_pte), pte_flags(pte)); + pr_err("PTP: diff=%lx\n", diff); + panic("Detected new S-page with execute permission"); + } + if (pte_exec(pte) && pte_smode(pte) && pte_write(pte)) + panic("Detected SX-page with write permission"); +} + +static inline void check_pt_integrity(pte_t *ptep, pte_t pte) +{ + pte_t old_pte; + + if (unlikely(!haoc_init_done)) + return; + if (iee_get_bitmap_type(__pa(ptep)) != IEE_FIXED_PGTABLE) + return; + old_pte = READ_ONCE(*ptep); + if (pte_pfn(old_pte) != pte_pfn(pte)) + panic("PTP: Are you trying to break pt integrity? old_pte: 0x%lx, new_pte: 0x%lx", + old_pte.pte, pte.pte); +} + +#ifdef CONFIG_PTP_S +bool ptp_is_user_pgtable(const void *ptp) +{ + if (!haoc_enabled) + return false; + + if (unlikely(!haoc_init_done)) + return false; + + if (unlikely(!virt_addr_valid(ptp))) + return false; + + return iee_get_bitmap_type(__pa((unsigned long)ptp)) == IEE_USER_PGTABLE; +} +EXPORT_SYMBOL(ptp_is_user_pgtable); + +void ptp_user_check_pte_update(pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) + return; + + check_alias_mapping(__pte_to_phys(pte), __pte_to_phys(pte) + PAGE_SIZE, + pte_write(pte), preserves_existing_writable_mapping(ptep, pte)); + check_pt_integrity(ptep, pte); +} +EXPORT_SYMBOL(ptp_user_check_pte_update); + +void ptp_user_check_pmd_update(pmd_t *pmdp, pmd_t pmd) +{ + if (!(pmd_val(pmd) & _PAGE_PRESENT)) + return; + + if (pmd_leaf(pmd)) + check_alias_mapping(__pmd_to_phys(pmd), __pmd_to_phys(pmd) + PMD_SIZE, + pmd_write(pmd), + pmd_preserves_existing_writable_mapping(pmdp, pmd)); +} + +void ptp_user_check_pud_update(pud_t *pudp, pud_t pud) +{ + if (!(pud_val(pud) & _PAGE_PRESENT)) + return; + + if (pud_leaf(pud)) + check_alias_mapping(__pud_to_phys(pud), __pud_to_phys(pud) + PUD_SIZE, + pud_write(pud), + pud_preserves_existing_writable_mapping(pudp, pud)); +} +#endif + +void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) { + WRITE_ONCE(*ptep, pte); + return; + } + + phys_addr_t phys_addr = __pte_to_phys(pte); + unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; + + check_dep_and_ro(image_addr, PAGE_SIZE, pte); + check_alias_mapping(phys_addr, phys_addr + PAGE_SIZE, pte_write(pte), + preserves_existing_writable_mapping(ptep, pte)); + check_sx(ptep, pte); + check_pt_integrity(ptep, pte); + WRITE_ONCE(*ptep, pte); +} + +void _iee_set_sx_pte(unsigned long __unused, pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) { + WRITE_ONCE(*ptep, pte); + return; + } + + phys_addr_t phys_addr = __pte_to_phys(pte); + unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; + + check_dep_and_ro(image_addr, PAGE_SIZE, pte); + + // FIXME: Writing the PTE during page split may create temporary alias mappings, + // so we cannot perform the check here. + // A different approach is needed in the future. + // check_alias_mapping(phys_addr, phys_addr + PAGE_SIZE, ptep, pte); + WRITE_ONCE(*ptep, pte); +} + +void _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pmd_val(pmd) & _PAGE_PRESENT)) { + WRITE_ONCE(*pmdp, pmd); + return; + } + if (pmd_leaf(pmd)) { + phys_addr = __pmd_to_phys(pmd); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PMD_SIZE, __pte(pmd_val(pmd))); + check_alias_mapping(phys_addr, phys_addr + PMD_SIZE, + pmd_write(pmd), + pmd_preserves_existing_writable_mapping(pmdp, pmd)); + check_sx((pte_t *)pmdp, __pte(pmd_val(pmd))); + } + + WRITE_ONCE(*pmdp, pmd); +} + +void _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pud_val(pud) & _PAGE_PRESENT)) { + WRITE_ONCE(*pudp, pud); + return; + } + if (pud_leaf(pud)) { + phys_addr = __pud_to_phys(pud); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PUD_SIZE, __pte(pud_val(pud))); + check_alias_mapping(phys_addr, phys_addr + PUD_SIZE, + pud_write(pud), + pud_preserves_existing_writable_mapping(pudp, pud)); + check_sx((pte_t *)pudp, __pte(pud_val(pud))); + } + + WRITE_ONCE(*pudp, pud); +} + +void _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d) +{ + WRITE_ONCE(*p4dp, p4d); +} + +void _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd) +{ + WRITE_ONCE(*pgdp, pgd); +} + +static inline void check_text_poke_ro(unsigned long image_addr, unsigned long size, pte_t pte) +{ + if (unlikely(!kernel_set_to_readonly)) + return; + + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); +#ifdef CONFIG_IEE_SIP + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_si_data_start, (unsigned long)__iee_si_data_end, pte, "iee_si_data"); +#endif +#ifdef CONFIG_IEE_SELINUX_P + check_addr_range_ro(image_addr, image_addr + size, +(unsigned long)__iee_selinux_data_start, (unsigned long)__iee_selinux_data_end, pte, "iee_si_data"); +#endif +} + +void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte) +{ + if (!(pte_val(pte) & _PAGE_PRESENT)) { + WRITE_ONCE(*ptep, pte); + return; + } + + phys_addr_t phys_addr = __pte_to_phys(pte); + unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; + + check_text_poke_ro(image_addr, PAGE_SIZE, pte); + check_alias_mapping(phys_addr, phys_addr + PAGE_SIZE, pte_write(pte), + preserves_existing_writable_mapping(ptep, pte)); + WRITE_ONCE(*ptep, pte); +} + +void _iee_set_pmd_ident(unsigned long __unused, pmd_t *pmdp, pmd_t pmd) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pmd_val(pmd) & _PAGE_PRESENT)) { + WRITE_ONCE(*pmdp, pmd); + return; + } + if (pmd_leaf(pmd)) { + phys_addr = __pmd_to_phys(pmd); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PMD_SIZE, __pte(pmd_val(pmd))); + } + + WRITE_ONCE(*pmdp, pmd); +} + +void _iee_set_pud_ident(unsigned long __unused, pud_t *pudp, pud_t pud) +{ + phys_addr_t phys_addr; + unsigned long image_addr; + + if (!(pud_val(pud) & _PAGE_PRESENT)) { + WRITE_ONCE(*pudp, pud); + return; + } + if (pud_leaf(pud)) { + phys_addr = __pud_to_phys(pud); + image_addr = phys_addr + __START_KERNEL_map - phys_base; + check_dep_and_ro(image_addr, PUD_SIZE, __pte(pud_val(pud))); + } + + WRITE_ONCE(*pudp, pud); +} + +DEFINE_PER_CPU(struct iee_cr0, iee_cr0s); + +void __init haoc_ptp_init(void) +{ + int cpu; + struct iee_cr0 *iee_cr0; + + set_iee_page(pg_cache.reserve_start_addr, + (1 << pg_cache.reserve_order) * pg_cache.levels, IEE_PGTABLE); + + set_iee_page(pgd_cache.reserve_start_addr, + (1 << pgd_cache.reserve_order) * pgd_cache.levels, IEE_PGTABLE); + +#ifdef CONFIG_PTP_S + if (!pg_user_cache.init) + panic("PTP_S: pg_user_cache is not initialized"); + + set_iee_page(pg_user_cache.reserve_start_addr, + (1 << pg_user_cache.reserve_order) * pg_user_cache.levels, + IEE_USER_PGTABLE); +#endif + + for_each_possible_cpu(cpu) { + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + iee_cr0->wp_disabled_cnt = 0; + } + + ptp_mark_all_pgtable_ro(); +} + +void ptp_disable_wp(unsigned long *cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + if (iee_cr0->wp_disabled_cnt == 0) { + *cr0 = read_cr0(); + asm volatile("mov %0, %%cr0" : : "r"(*cr0 & ~X86_CR0_WP) : "memory"); + } + iee_cr0->wp_disabled_cnt++; + put_cpu(); + local_irq_restore(irq_flags); +} + +void ptp_restore_wp(unsigned long cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + iee_cr0->wp_disabled_cnt--; + if (iee_cr0->wp_disabled_cnt == 0) + asm volatile("mov %0, %%cr0" : : "r"(cr0 | X86_CR0_WP) : "memory"); + put_cpu(); + local_irq_restore(irq_flags); +} + +void ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + *wp_disabled_cnt = iee_cr0->wp_disabled_cnt; + if (*wp_disabled_cnt > 0) { + *cr0 = read_cr0(); + iee_cr0->wp_disabled_cnt = 0; + asm volatile("mov %0, %%cr0" : : "r"(*cr0 | X86_CR0_WP) : "memory"); + } + put_cpu(); + local_irq_restore(irq_flags); +} + +void ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0) +{ + unsigned long irq_flags; + int cpu; + struct iee_cr0 *iee_cr0; + + if (unlikely(!haoc_init_done)) + return; + + local_irq_save(irq_flags); + if (wp_disabled_cnt > 0) { + cpu = get_cpu(); + iee_cr0 = per_cpu_ptr(&iee_cr0s, cpu); + asm volatile("mov %0, %%cr0" : : "r"(cr0 & ~X86_CR0_WP) : "memory"); + iee_cr0->wp_disabled_cnt = wp_disabled_cnt; + put_cpu(); + } + local_irq_restore(irq_flags); +} diff --git a/arch/x86/kernel/haoc/selinuxp/Makefile b/arch/x86/kernel/haoc/selinuxp/Makefile new file mode 100644 index 0000000000000..8e5f128f5a430 --- /dev/null +++ b/arch/x86/kernel/haoc/selinuxp/Makefile @@ -0,0 +1,5 @@ +ccflags-y += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include +obj-y += selinuxp.o + +FLASK_HDR := $(srctree)/security/selinux/flask.h +$(obj)/selinuxp.o: $(FLASK_HDR) diff --git a/arch/x86/kernel/haoc/selinuxp/selinuxp.c b/arch/x86/kernel/haoc/selinuxp/selinuxp.c new file mode 100644 index 0000000000000..6b4a18dc316e5 --- /dev/null +++ b/arch/x86/kernel/haoc/selinuxp/selinuxp.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "security.h" +#include "ss/services.h" +#include +#include +#include + +void _iee_set_selinux_status_pg(unsigned long iee_offset, struct page *new_page) +{ + selinux_state.status_page = new_page; +} + +void _iee_set_selinux_enforcing(unsigned long iee_offset, bool value) +{ + selinux_state.enforcing = value; +} + +void _iee_mark_selinux_initialized(unsigned long iee_offset) +{ + /* do a synchronized write to avoid race conditions */ + smp_store_release(((bool *)&(selinux_state.initialized)), true); + pr_info("IEE: Mark selinux initialized."); +} + +void _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap) +{ + selinux_state.policycap[idx] = cap; +} + +/* + * Please make sure param iee_new_policy is from policy_jar memcache. + * Need to free new_policy after calling this func as it's only used to + * trans data from kernel. + */ +void _iee_sel_rcu_assign_policy(unsigned long iee_offset, struct selinux_policy *new_policy, + struct selinux_policy *iee_new_policy) +{ + iee_verify_type(__pa(iee_new_policy), IEE_POLICY, "policy"); + /* TODO: Verify information from incoming policy. */ + + /* Copy data from kernel to new allocated policy struct inside iee. */ + memcpy(iee_new_policy, new_policy, sizeof(struct selinux_policy)); + rcu_assign_pointer(selinux_state.policy, iee_new_policy); + pr_info("IEE: assigned rcu pointer selinux_state.policy."); +} + +void __init iee_selinuxp_init(void) +{ + unsigned long start, end; + int num_pages; + + start = (unsigned long)__iee_selinux_data_start; + end = (unsigned long)__iee_selinux_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_pages(start, num_pages, IEE_SIP_DATA); +} diff --git a/arch/x86/kernel/haoc/varp/Makefile b/arch/x86/kernel/haoc/varp/Makefile new file mode 100644 index 0000000000000..775b2da3f64b1 --- /dev/null +++ b/arch/x86/kernel/haoc/varp/Makefile @@ -0,0 +1 @@ +obj-y += varp.o \ No newline at end of file diff --git a/arch/x86/kernel/haoc/varp/varp.c b/arch/x86/kernel/haoc/varp/varp.c new file mode 100644 index 0000000000000..271a16428e94f --- /dev/null +++ b/arch/x86/kernel/haoc/varp/varp.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +void _iee_set_varp_modprobe_path(unsigned long __unused, char *data, + int maxlen, size_t len, char *buffer, + size_t *lenp) +{ + char c, *p; + + if (!data || !buffer || !lenp) + return; + p = buffer; + while ((p - buffer) < *lenp && len < maxlen - 1) { + c = *(p++); + if (c == 0 || c == '\n') + break; + data[len++] = c; + } + data[len] = 0; +} + +void __init varp_init(void) +{ + unsigned long start, end, varp_logical_addr; + int num_pages; + + /* Map .iee.varp as RO pages, the variable need to be protected can be added */ + start = (unsigned long)__iee_varp_data_start; + end = (unsigned long)__iee_varp_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_iee_page(start, num_pages, IEE_VARP); + + /* protect linear mapping */ + varp_logical_addr = (unsigned long)__va(__pa_symbol(start)); + set_iee_page(varp_logical_addr, num_pages, IEE_VARP); + + pr_info("HAOC: CONFIG_VARP enabled."); +} diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 35fd5f1444fdb..a3e8d44247dfc 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -35,6 +35,13 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#include +#endif +#ifdef CONFIG_PTP +#include +#endif #define CREATE_TRACE_POINTS #include @@ -364,6 +371,24 @@ static noinstr void default_do_nmi(struct pt_regs *regs) int handled; bool b2b = false; +#ifdef CONFIG_IEE + u64 cr0 = read_cr0(); + bool cr0_wp = cr0 & X86_CR0_WP; +#ifdef CONFIG_PTP + u64 iee_wp_flag = 0; +#endif + + if (!cr0_wp) { +#ifdef CONFIG_PTP + iee_wp_flag = this_cpu_read(iee_cr0s.wp_disabled_cnt); + this_cpu_write(iee_cr0s.wp_disabled_cnt, 0); +#endif + cr0 |= X86_CR0_WP; + // we assume that only IEE gate can disable cr0.wp. + // If cr0.wp = false, set cr0.wp = true in case of nmi happens in IEE gate. + asm volatile("mov %0,%%cr0" : "+r" (cr0) : : "memory"); + } +#endif /* * CPU-specific NMI must be processed before non-CPU-specific * NMI, otherwise we may lose it, because the CPU-specific @@ -475,6 +500,17 @@ static noinstr void default_do_nmi(struct pt_regs *regs) out: instrumentation_end(); +#ifdef CONFIG_IEE + if (!cr0_wp) { + cr0 = read_cr0() & ~X86_CR0_WP; + + asm volatile("mov %0,%%cr0" : "+r" (cr0) : : "memory"); + +#ifdef CONFIG_PTP + this_cpu_write(iee_cr0s.wp_disabled_cnt, iee_wp_flag); +#endif + } +#endif } /* diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 234851fe0ef8e..51aab86e48b9a 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -81,7 +81,11 @@ void __init native_pv_lock_init(void) static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) { +#ifdef CONFIG_PTP + ptp_tlb_remove_page(tlb, table); +#else tlb_remove_page(tlb, table); +#endif } unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index dc736a9101a80..9e1b283e03a3f 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -117,12 +117,33 @@ __iee_si_data_end = .; #ifdef CONFIG_CREDP #define CRED_DATA \ . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_start = .; \ *(.iee.cred) \ + . = ALIGN(PAGE_SIZE); \ + __iee_cred_data_end = .; \ . = ALIGN(PAGE_SIZE); #else #define CRED_DATA #endif +#ifdef CONFIG_IEE_SELINUX_P +#define IEE_SELINUX_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_start = .; \ + *(.iee.selinux) \ + . = ALIGN(PAGE_SIZE); \ + __iee_selinux_data_end = .; +#endif + +#ifdef CONFIG_VARP +#define VARP_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_start = .; \ + *(.iee.varp) \ + . = ALIGN(PAGE_SIZE); \ + __iee_varp_data_end = .; +#endif + PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(6); /* RW_ */ @@ -213,6 +234,12 @@ SECTIONS #endif #ifdef CONFIG_CREDP CRED_DATA +#endif +#ifdef CONFIG_IEE_SELINUX_P + IEE_SELINUX_DATA +#endif +#ifdef CONFIG_VARP + VARP_DATA #endif CONSTRUCTORS diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c index fe0b2e66ded93..174e0e06d44a2 100644 --- a/arch/x86/mm/ident_map.c +++ b/arch/x86/mm/ident_map.c @@ -14,7 +14,11 @@ static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, if (pmd_present(*pmd)) continue; +#ifdef CONFIG_PTP + ptp_set_pmd_ident(pmd, __pmd((addr - info->offset) | info->page_flag)); +#else set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag)); +#endif } } @@ -52,7 +56,11 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, pud_t pudval; pudval = __pud((addr - info->offset) | info->page_flag); +#ifdef CONFIG_PTP + ptp_set_pud_ident(pud, pudval); +#else set_pud(pud, pudval); +#endif continue; } @@ -65,7 +73,11 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, if (!pmd) return -ENOMEM; ident_pmd_init(info, pmd, addr, next); +#ifdef CONFIG_PTP + ptp_set_pud_ident(pud, __pud(__pa(pmd) | info->kernpg_flag)); +#else set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag)); +#endif } return 0; diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 8bfa00bc2c79e..a32e06859fee0 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -59,6 +59,10 @@ #ifdef CONFIG_IEE #include #endif +#ifdef CONFIG_IEE_SIP +#include +#include +#endif #include "mm_internal.h" #include "ident_map.c" @@ -1364,6 +1368,10 @@ static void __init preallocate_vmalloc_pages(void) void __init mem_init(void) { +#ifdef CONFIG_IEE + iee_early_init(); +#endif + pci_iommu_alloc(); /* clear_bss() already clear the empty_zero_page */ @@ -1385,19 +1393,11 @@ void __init mem_init(void) if (get_gate_vma(&init_mm)) kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); - #ifdef CONFIG_IEE - /* - * Split the linear mapping region of the kernel address space into two equally-sized parts. - * The lower region retains the original linear mapping. - * The upper region becomes the IEE linear mapping area. - * Note that the IEE mapping region is mapped with read-only permissions. - */ - if (haoc_enabled) { - pr_info("HAOC is enabled by kernel command line."); - iee_init(); - } - #endif preallocate_vmalloc_pages(); + +#ifdef CONFIG_IEE + iee_init(); +#endif } #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT @@ -1412,7 +1412,11 @@ int __init deferred_page_init_max_threads(const struct cpumask *node_cpumask) } #endif +#ifdef CONFIG_IEE_SIP +int kernel_set_to_readonly __iee_si_data; +#else int kernel_set_to_readonly; +#endif void mark_rodata_ro(void) { @@ -1422,12 +1426,17 @@ void mark_rodata_ro(void) unsigned long text_end = PFN_ALIGN(_etext); unsigned long rodata_end = PFN_ALIGN(__end_rodata); unsigned long all_end; +#ifdef CONFIG_IEE_SIP + int kernel_set_to_readonly_value = 1; +#endif printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", (end - start) >> 10); set_memory_ro(start, (end - start) >> PAGE_SHIFT); +#ifndef CONFIG_IEE_SIP kernel_set_to_readonly = 1; +#endif /* * The rodata/data/bss/brk section (but not the kernel text!) @@ -1460,6 +1469,10 @@ void mark_rodata_ro(void) (void *)rodata_end, (void *)_sdata); debug_checkwx(); +#ifdef CONFIG_IEE_SIP + iee_memcpy(&kernel_set_to_readonly, &kernel_set_to_readonly_value, + sizeof(int)); +#endif } /* diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 525794f1eefb3..f39bc18d8daf4 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -835,7 +835,11 @@ EXPORT_SYMBOL_GPL(slow_virt_to_phys); static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) { /* change init_mm */ +#ifdef CONFIG_PTP + ptp_set_sx_pte(kpte, pte); +#else set_pte_atomic(kpte, pte); +#endif #ifdef CONFIG_X86_32 if (!SHARED_KERNEL_PMD) { struct page *page; @@ -1068,7 +1072,11 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, else pr_warn_once("CPA: Cannot fixup static protections for PUD split\n"); set: +#ifdef CONFIG_PTP + ptp_set_sx_pte(pte, pfn_pte(pfn, ref_prot)); +#else set_pte(pte, pfn_pte(pfn, ref_prot)); +#endif } static int @@ -1670,7 +1678,11 @@ static int __change_page_attr(struct cpa_data *cpa, int primary) * Do we really change anything ? */ if (pte_val(old_pte) != pte_val(new_pte)) { +#ifdef CONFIG_PTP + ptp_set_sx_pte(kpte, new_pte); +#else set_pte_atomic(kpte, new_pte); +#endif cpa->flags |= CPA_FLUSHTLB; } cpa->numpages = 1; diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index cada0beeee76c..c90fe90119af1 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -6,6 +6,10 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#include +#endif #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; @@ -418,15 +422,29 @@ static inline void _pgd_free(pgd_t *pgd) } #else +#ifdef CONFIG_PTP +struct iee_cache pgd_cache = { + .object_order = PGD_ALLOCATION_ORDER, +}; +#endif + static inline pgd_t *_pgd_alloc(void) { +#ifdef CONFIG_PTP + return (pgd_t *)iee_cache_alloc(&pgd_cache, GFP_PGTABLE_USER); +#else return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, PGD_ALLOCATION_ORDER); +#endif } static inline void _pgd_free(pgd_t *pgd) { +#ifdef CONFIG_PTP + iee_cache_free(&pgd_cache, pgd); +#else free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); +#endif } #endif /* CONFIG_X86_PAE */ @@ -560,8 +578,13 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, int ret = 0; if (pte_young(*ptep)) +#ifdef CONFIG_PTP + ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *) &ptep->pte); +#else ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, (unsigned long *) &ptep->pte); +#endif return ret; } @@ -573,8 +596,13 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, int ret = 0; if (pmd_young(*pmdp)) +#ifdef CONFIG_PTP + ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pmdp); +#else ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, (unsigned long *)pmdp); +#endif return ret; } @@ -587,8 +615,13 @@ int pudp_test_and_clear_young(struct vm_area_struct *vma, int ret = 0; if (pud_young(*pudp)) +#ifdef CONFIG_PTP + ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pudp); +#else ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, (unsigned long *)pudp); +#endif return ret; } @@ -838,14 +871,22 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) for (i = 0; i < PTRS_PER_PMD; i++) { if (!pmd_none(pmd_sv[i])) { pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pte); +#else free_page((unsigned long)pte); +#endif } } free_page((unsigned long)pmd_sv); pagetable_pmd_dtor(virt_to_ptdesc(pmd)); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pmd); +#else free_page((unsigned long)pmd); +#endif return 1; } @@ -868,7 +909,11 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) /* INVLPG to clear all paging-structure caches */ flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, pte); +#else free_page((unsigned long)pte); +#endif return 1; } diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index 916247c1d2a36..a47af6f9d6775 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -139,7 +139,11 @@ pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) * The user page tables get the full PGD, accessible from * userspace: */ +#ifdef CONFIG_PTP + ptp_set_pgd(kernel_to_user_pgdp(pgdp), pgd); +#else kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; +#endif /* * If this is normal user memory, make it NX in the kernel @@ -292,7 +296,11 @@ static void __init pti_setup_vsyscall(void) if (WARN_ON(!target_pte)) return; +#ifdef CONFIG_PTP + set_pte(target_pte, *pte); +#else *target_pte = *pte; +#endif set_vsyscall_pgtable_user_bits(kernel_to_user_pgdp(swapper_pg_dir)); } #else @@ -370,14 +378,22 @@ pti_clone_pgtable(unsigned long start, unsigned long end, * code that only set this bit when supported. */ if (boot_cpu_has(X86_FEATURE_PGE)) +#ifdef CONFIG_PTP + set_pmd(pmd, pmd_set_flags(*pmd, _PAGE_GLOBAL)); +#else *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL); +#endif /* * Copy the PMD. That is, the kernelmode and usermode * tables will share the last-level page tables of this * address range */ +#ifdef CONFIG_PTP + set_pmd(target_pmd, *pmd); +#else *target_pmd = *pmd; +#endif addr = round_up(addr + 1, PMD_SIZE); @@ -401,10 +417,18 @@ pti_clone_pgtable(unsigned long start, unsigned long end, /* Set GLOBAL bit in both PTEs */ if (boot_cpu_has(X86_FEATURE_PGE)) +#ifdef CONFIG_PTP + set_pte(pte, pte_set_flags(*pte, _PAGE_GLOBAL)); +#else *pte = pte_set_flags(*pte, _PAGE_GLOBAL); +#endif /* Clone the PTE */ +#ifdef CONFIG_PTP + set_pte(target_pte, *pte); +#else *target_pte = *pte; +#endif addr = round_up(addr + 1, PAGE_SIZE); @@ -430,7 +454,11 @@ static void __init pti_clone_p4d(unsigned long addr) kernel_pgd = pgd_offset_k(addr); kernel_p4d = p4d_offset(kernel_pgd, addr); +#ifdef CONFIG_PTP + set_p4d(user_p4d, *kernel_p4d); +#else *user_p4d = *kernel_p4d; +#endif } /* @@ -461,7 +489,11 @@ static void __init pti_clone_user_shared(void) if (WARN_ON(!target_pte)) return; +#ifdef CONFIG_PTP + set_pte(target_pte, pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL)); +#else *target_pte = pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL); +#endif } } diff --git a/arch/x86/mm/ptp_ident_map.c b/arch/x86/mm/ptp_ident_map.c new file mode 100644 index 0000000000000..bd2e3e6c82b67 --- /dev/null +++ b/arch/x86/mm/ptp_ident_map.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Helper routines for building identity mapping page tables. This is + * included by both the compressed kernel and the regular kernel. + */ + +static inline void ptp_set_pte_pre_init(pte_t *ptep, pte_t pte) +{ + WRITE_ONCE(*ptep, pte); +} + +static inline void ptp_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) +{ + WRITE_ONCE(*pmdp, pmd); +} + +static inline void ptp_set_pud_pre_init(pud_t *pudp, pud_t pud) +{ + WRITE_ONCE(*pudp, pud); +} + +static inline void ptp_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) +{ + pgd_t pgd; + + if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { + WRITE_ONCE(*p4dp, p4d); + return; + } + + pgd = native_make_pgd(native_p4d_val(p4d)); + pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); + WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); +} + +static inline void ptp_set_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ + WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); +} + +static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, + unsigned long addr, unsigned long end) +{ + addr &= PMD_MASK; + for (; addr < end; addr += PMD_SIZE) { + pmd_t *pmd = pmd_page + pmd_index(addr); + + if (pmd_present(*pmd)) + continue; + + ptp_set_pmd_pre_init(pmd, __pmd((addr - info->offset) | info->page_flag)); + } +} + +static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, + unsigned long addr, unsigned long end) +{ + unsigned long next; + + for (; addr < end; addr = next) { + pud_t *pud = pud_page + pud_index(addr); + pmd_t *pmd; + + next = (addr & PUD_MASK) + PUD_SIZE; + if (next > end) + next = end; + + if (info->direct_gbpages) { + pud_t pudval; + + if (pud_present(*pud)) + continue; + + addr &= PUD_MASK; + pudval = __pud((addr - info->offset) | info->page_flag); + ptp_set_pud_pre_init(pud, pudval); + continue; + } + + if (pud_present(*pud)) { + pmd = pmd_offset(pud, 0); + ident_pmd_init(info, pmd, addr, next); + continue; + } + pmd = (pmd_t *)info->alloc_pgt_page(info->context); + if (!pmd) + return -ENOMEM; + ident_pmd_init(info, pmd, addr, next); + ptp_set_pud_pre_init(pud, __pud(__pa(pmd) | info->kernpg_flag)); + } + + return 0; +} + +static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, + unsigned long addr, unsigned long end) +{ + unsigned long next; + int result; + + for (; addr < end; addr = next) { + p4d_t *p4d = p4d_page + p4d_index(addr); + pud_t *pud; + + next = (addr & P4D_MASK) + P4D_SIZE; + if (next > end) + next = end; + + if (p4d_present(*p4d)) { + pud = pud_offset(p4d, 0); + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + + continue; + } + pud = (pud_t *)info->alloc_pgt_page(info->context); + if (!pud) + return -ENOMEM; + + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + + ptp_set_p4d_pre_init(p4d, __p4d(__pa(pud) | info->kernpg_flag)); + } + + return 0; +} + +int ptp_kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, + unsigned long pstart, unsigned long pend) +{ + unsigned long addr = pstart + info->offset; + unsigned long end = pend + info->offset; + unsigned long next; + int result; + + /* Set the default pagetable flags if not supplied */ + if (!info->kernpg_flag) + info->kernpg_flag = _KERNPG_TABLE; + + /* Filter out unsupported __PAGE_KERNEL_* bits: */ + info->kernpg_flag &= __default_kernel_pte_mask; + + for (; addr < end; addr = next) { + pgd_t *pgd = pgd_page + pgd_index(addr); + p4d_t *p4d; + + next = (addr & PGDIR_MASK) + PGDIR_SIZE; + if (next > end) + next = end; + + if (pgd_present(*pgd)) { + p4d = p4d_offset(pgd, 0); + result = ident_p4d_init(info, p4d, addr, next); + if (result) + return result; + continue; + } + + p4d = (p4d_t *)info->alloc_pgt_page(info->context); + if (!p4d) + return -ENOMEM; + result = ident_p4d_init(info, p4d, addr, next); + if (result) + return result; + if (pgtable_l5_enabled()) { + ptp_set_pgd_pre_init(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); + } else { + /* + * With p4d folded, pgd is equal to p4d. + * The pgd entry has to point to the pud page table in this case. + */ + pud_t *pud = pud_offset(p4d, 0); + + ptp_set_pgd_pre_init(pgd, __pgd(__pa(pud) | info->kernpg_flag)); + } + } + + return 0; +} diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 91d31ac422d6c..89de70744406e 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -48,6 +48,10 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#include +#endif /* * We allocate runtime services regions top-down, starting from -4G, i.e. @@ -73,7 +77,11 @@ int __init efi_alloc_page_tables(void) gfp_t gfp_mask; gfp_mask = GFP_KERNEL | __GFP_ZERO; +#ifdef CONFIG_PTP + efi_pgd = iee_cache_alloc(&pgd_cache, gfp_mask); +#else efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER); +#endif if (!efi_pgd) goto fail; @@ -116,7 +124,11 @@ void efi_sync_low_kernel_mappings(void) pgd_k = pgd_offset_k(PAGE_OFFSET); num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET); +#ifdef CONFIG_PTP + iee_memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); +#else memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); +#endif pgd_efi = efi_pgd + pgd_index(EFI_VA_END); pgd_k = pgd_offset_k(EFI_VA_END); @@ -124,7 +136,11 @@ void efi_sync_low_kernel_mappings(void) p4d_k = p4d_offset(pgd_k, 0); num_entries = p4d_index(EFI_VA_END); +#ifdef CONFIG_PTP + iee_memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); +#else memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); +#endif /* * We share all the PUD entries apart from those that map the @@ -139,13 +155,21 @@ void efi_sync_low_kernel_mappings(void) pud_k = pud_offset(p4d_k, 0); num_entries = pud_index(EFI_VA_END); +#ifdef CONFIG_PTP + iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#else memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#endif pud_efi = pud_offset(p4d_efi, EFI_VA_START); pud_k = pud_offset(p4d_k, EFI_VA_START); num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START); +#ifdef CONFIG_PTP + iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#else memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#endif } /* diff --git a/block/sed-opal.c b/block/sed-opal.c index 1a1cb35bf4b79..442cdb7b801c9 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -316,7 +316,11 @@ static int read_sed_opal_key(const char *key_name, u_char *buffer, int buflen) return PTR_ERR(kref); key = key_ref_to_ptr(kref); +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = key_validate(key); if (ret == 0) { if (buflen > key->datalen) @@ -324,7 +328,11 @@ static int read_sed_opal_key(const char *key_name, u_char *buffer, int buflen) ret = key->type->read(key, (char *)buffer, buflen); } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_ref_put(kref); diff --git a/certs/blacklist.c b/certs/blacklist.c index 675dd7a8f07ab..1d139916a9f90 100644 --- a/certs/blacklist.c +++ b/certs/blacklist.c @@ -17,6 +17,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "blacklist.h" /* @@ -91,7 +94,11 @@ static int blacklist_key_instantiate(struct key *key, #endif /* Sets safe default permissions for keys loaded by user space. */ +#ifdef CONFIG_KEYP + iee_set_key_perm(key, BLACKLIST_KEY_PERM); +#else key->perm = BLACKLIST_KEY_PERM; +#endif /* * Skips the authentication step for builtin hashes, they are not diff --git a/certs/system_keyring.c b/certs/system_keyring.c index 9de610bf1f4b2..2d1e79b2de378 100644 --- a/certs/system_keyring.c +++ b/certs/system_keyring.c @@ -93,9 +93,15 @@ int restrict_link_by_builtin_and_secondary_trusted( /* If we have a secondary trusted keyring, then that contains a link * through to the builtin keyring and the search will follow that link. */ +#ifdef CONFIG_KEYP + if (type == &key_type_keyring && + dest_keyring == secondary_trusted_keys && + payload == (union key_payload *)(builtin_trusted_keys->name_link.next)) +#else if (type == &key_type_keyring && dest_keyring == secondary_trusted_keys && payload == &builtin_trusted_keys->payload) +#endif /* Allow the builtin keyring to be added to the secondary */ return 0; @@ -122,9 +128,15 @@ int restrict_link_by_digsig_builtin_and_secondary(struct key *dest_keyring, /* If we have a secondary trusted keyring, then that contains a link * through to the builtin keyring and the search will follow that link. */ +#ifdef CONFIG_KEYP + if (type == &key_type_keyring && + dest_keyring == secondary_trusted_keys && + payload == (union key_payload *)(builtin_trusted_keys->name_link.next)) +#else if (type == &key_type_keyring && dest_keyring == secondary_trusted_keys && payload == &builtin_trusted_keys->payload) +#endif /* Allow the builtin keyring to be added to the secondary */ return 0; @@ -209,9 +221,15 @@ int restrict_link_by_builtin_secondary_and_machine( const union key_payload *payload, struct key *restrict_key) { +#ifdef CONFIG_KEYP + if (machine_trusted_keys && type == &key_type_keyring && + dest_keyring == secondary_trusted_keys && + payload == (union key_payload *)(machine_trusted_keys->name_link.next)) +#else if (machine_trusted_keys && type == &key_type_keyring && dest_keyring == secondary_trusted_keys && payload == &machine_trusted_keys->payload) +#endif /* Allow the machine keyring to be added to the secondary */ return 0; diff --git a/crypto/af_alg.c b/crypto/af_alg.c index e7fea668473d9..84cec2f219b5e 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -304,7 +304,11 @@ static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, if (IS_ERR(key)) return PTR_ERR(key); +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = ERR_PTR(-ENOPROTOOPT); if (!strcmp(key->type->name, "user") || @@ -319,21 +323,33 @@ static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, } if (IS_ERR(ret)) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return PTR_ERR(ret); } key_data = sock_kmalloc(&ask->sk, key_datalen, GFP_KERNEL); if (!key_data) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return -ENOMEM; } memcpy(key_data, ret, key_datalen); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); err = type->setkey(ask->private, key_data, key_datalen); diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index 7859b0692b42b..fb92e47d6d7a4 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -16,6 +16,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "asymmetric_keys.h" @@ -473,6 +476,18 @@ static void asymmetric_key_free_preparse(struct key_preparsed_payload *prep) static void asymmetric_key_destroy(struct key *key) { struct asymmetric_key_subtype *subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + struct asymmetric_key_ids *kids = + ((union key_payload *)(key->name_link.next))->data[asym_key_ids]; + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + void *data = ((union key_payload *)(key->name_link.next))->data[asym_crypto]; + void *auth = ((union key_payload *)(key->name_link.next))->data[asym_auth]; + + key_payload->data[asym_crypto] = NULL; + key_payload->data[asym_subtype] = NULL; + key_payload->data[asym_key_ids] = NULL; + key_payload->data[asym_auth] = NULL; +#else struct asymmetric_key_ids *kids = key->payload.data[asym_key_ids]; void *data = key->payload.data[asym_crypto]; void *auth = key->payload.data[asym_auth]; @@ -481,6 +496,7 @@ static void asymmetric_key_destroy(struct key *key) key->payload.data[asym_subtype] = NULL; key->payload.data[asym_key_ids] = NULL; key->payload.data[asym_auth] = NULL; +#endif if (subtype) { subtype->destroy(data, auth); @@ -586,8 +602,13 @@ int asymmetric_key_eds_op(struct kernel_pkey_params *params, if (key->type != &key_type_asymmetric) return -EINVAL; subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + if (!subtype || + !((union key_payload *)(key->name_link.next))->data[0]) +#else if (!subtype || !key->payload.data[0]) +#endif return -EINVAL; if (!subtype->eds_op) return -ENOTSUPP; diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index 1dcab27986a6a..2f7db6236ee23 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -30,7 +30,12 @@ MODULE_LICENSE("GPL"); static void public_key_describe(const struct key *asymmetric_key, struct seq_file *m) { +#ifdef CONFIG_KEYP + struct public_key *key = + ((union key_payload *)(asymmetric_key->name_link.next))->data[asym_crypto]; +#else struct public_key *key = asymmetric_key->payload.data[asym_crypto]; +#endif if (key) seq_printf(m, "%s.%s", key->id_type, key->pkey_algo); @@ -158,7 +163,12 @@ static int software_key_query(const struct kernel_pkey_params *params, struct kernel_pkey_query *info) { struct crypto_akcipher *tfm; +#ifdef CONFIG_KEYP + struct public_key *pkey = + ((union key_payload *)(params->key->name_link.next))->data[asym_crypto]; +#else struct public_key *pkey = params->key->payload.data[asym_crypto]; +#endif char alg_name[CRYPTO_MAX_ALG_NAME]; struct crypto_sig *sig; u8 *key, *ptr; @@ -273,7 +283,12 @@ static int software_key_query(const struct kernel_pkey_params *params, static int software_key_eds_op(struct kernel_pkey_params *params, const void *in, void *out) { +#ifdef CONFIG_KEYP + const struct public_key *pkey = + ((union key_payload *)(params->key->name_link.next))->data[asym_crypto]; +#else const struct public_key *pkey = params->key->payload.data[asym_crypto]; +#endif char alg_name[CRYPTO_MAX_ALG_NAME]; struct crypto_akcipher *tfm; struct crypto_sig *sig; @@ -453,7 +468,12 @@ EXPORT_SYMBOL_GPL(public_key_verify_signature); static int public_key_verify_signature_2(const struct key *key, const struct public_key_signature *sig) { +#ifdef CONFIG_KEYP + const struct public_key *pk = + ((union key_payload *)(key->name_link.next))->data[asym_crypto]; +#else const struct public_key *pk = key->payload.data[asym_crypto]; +#endif return public_key_verify_signature(pk, sig); } diff --git a/crypto/asymmetric_keys/signature.c b/crypto/asymmetric_keys/signature.c index 2deff81f8af50..6d0708bf095bd 100644 --- a/crypto/asymmetric_keys/signature.c +++ b/crypto/asymmetric_keys/signature.c @@ -51,8 +51,13 @@ int query_asymmetric_key(const struct kernel_pkey_params *params, if (key->type != &key_type_asymmetric) return -EINVAL; subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + if (!subtype || + !((union key_payload *)(key->name_link.next))->data[0]) +#else if (!subtype || !key->payload.data[0]) +#endif return -EINVAL; if (!subtype->query) return -ENOTSUPP; @@ -145,8 +150,13 @@ int verify_signature(const struct key *key, if (key->type != &key_type_asymmetric) return -EINVAL; subtype = asymmetric_key_subtype(key); +#ifdef CONFIG_KEYP + if (!subtype || + !((union key_payload *)(key->name_link.next))->data[0]) +#else if (!subtype || !key->payload.data[0]) +#endif return -EINVAL; if (!subtype->verify_signature) return -ENOTSUPP; diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c index 83092d93f36a6..fb12e7d0660ad 100644 --- a/drivers/firmware/efi/arm-runtime.c +++ b/drivers/firmware/efi/arm-runtime.c @@ -94,7 +94,11 @@ static int __init arm_enable_runtime_services(void) return 0; } + #ifdef CONFIG_PTP + efi_memmap_unmap_after_init(); + #else efi_memmap_unmap(); + #endif mapsize = efi.memmap.desc_size * efi.memmap.nr_map; diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 7e2f2ffd3ceea..80948eebbdc83 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2505,7 +2505,11 @@ static int set_key_encrypted(struct crypt_config *cc, struct key *key) { const struct encrypted_key_payload *ekp; +#ifdef CONFIG_KEYP + ekp = ((union key_payload *)(key->name_link.next))->data[0]; +#else ekp = key->payload.data[0]; +#endif if (!ekp) return -EKEYREVOKED; @@ -2521,7 +2525,11 @@ static int set_key_trusted(struct crypt_config *cc, struct key *key) { const struct trusted_key_payload *tkp; +#ifdef CONFIG_KEYP + tkp = ((union key_payload *)(key->name_link.next))->data[0]; +#else tkp = key->payload.data[0]; +#endif if (!tkp) return -EKEYREVOKED; @@ -2583,17 +2591,29 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string return PTR_ERR(key); } +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = set_key(cc, key); if (ret < 0) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); kfree_sensitive(new_key_string); return ret; } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); /* clear the flag since following operations may invalidate previously valid key */ diff --git a/drivers/md/dm-verity-verify-sig.c b/drivers/md/dm-verity-verify-sig.c index 1ced2ef255f32..449432b80983e 100644 --- a/drivers/md/dm-verity-verify-sig.c +++ b/drivers/md/dm-verity-verify-sig.c @@ -40,7 +40,11 @@ static int verity_verify_get_sig_from_key(const char *key_desc, if (IS_ERR(key)) return PTR_ERR(key); +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ukp = user_key_payload_locked(key); if (!ukp) { @@ -58,7 +62,11 @@ static int verity_verify_get_sig_from_key(const char *key_desc, memcpy(sig_opts->sig, ukp->data, sig_opts->sig_size); end: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return ret; diff --git a/drivers/nvdimm/security.c b/drivers/nvdimm/security.c index a03e3c45f2973..a901824dfb7c7 100644 --- a/drivers/nvdimm/security.c +++ b/drivers/nvdimm/security.c @@ -28,7 +28,11 @@ static void *key_data(struct key *key) { struct encrypted_key_payload *epayload = dereference_key_locked(key); +#ifdef CONFIG_KEYP + lockdep_assert_held_read(&KEY_SEM(key)); +#else lockdep_assert_held_read(&key->sem); +#endif return epayload->decrypted_data; } @@ -38,7 +42,11 @@ static void nvdimm_put_key(struct key *key) if (!key) return; +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); } @@ -65,10 +73,18 @@ static struct key *nvdimm_request_key(struct nvdimm *nvdimm) } else { struct encrypted_key_payload *epayload; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif epayload = dereference_key_locked(key); if (epayload->decrypted_datalen != NVDIMM_PASSPHRASE_LEN) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); key = NULL; } @@ -107,10 +123,18 @@ static struct key *nvdimm_lookup_user_key(struct nvdimm *nvdimm, dev_dbg(dev, "%s: key found: %#x\n", __func__, key_serial(key)); +#ifdef CONFIG_KEYP + down_read_nested(&KEY_SEM(key), subclass); +#else down_read_nested(&key->sem, subclass); +#endif epayload = dereference_key_locked(key); if (epayload->decrypted_datalen != NVDIMM_PASSPHRASE_LEN) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); key = NULL; } diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c index a5fbb6ed38aed..93f80a19c1947 100644 --- a/drivers/tty/serial/earlycon.c +++ b/drivers/tty/serial/earlycon.c @@ -40,7 +40,11 @@ static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size) { void __iomem *base; #ifdef CONFIG_FIX_EARLYCON_MEM + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK, FIXMAP_PAGE_IO); + #else set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK); + #endif base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE); base += paddr & ~PAGE_MASK; #else diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c index 45b42d8f64532..ac3b1d2689696 100644 --- a/drivers/usb/early/ehci-dbgp.c +++ b/drivers/usb/early/ehci-dbgp.c @@ -879,7 +879,11 @@ int __init early_dbgp_init(char *s) * FIXME I don't have the bar size so just guess PAGE_SIZE is more * than enough. 1K is the biggest I have seen. */ + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(FIX_DBGP_BASE, bar_val & PAGE_MASK, FIXMAP_PAGE_NOCACHE); + #else set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK); + #endif ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE); ehci_bar += bar_val & ~PAGE_MASK; dbgp_printk("ehci_bar: %p\n", ehci_bar); diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index 7cbb1fd872acc..9108d5401bda8 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -590,8 +590,13 @@ static void fscrypt_provisioning_key_describe(const struct key *key, { seq_puts(m, key->description); if (key_is_positive(key)) { +#ifdef CONFIG_KEYP + const struct fscrypt_provisioning_key_payload *payload = + ((union key_payload *)(key->name_link.next))->data[0]; +#else const struct fscrypt_provisioning_key_payload *payload = key->payload.data[0]; +#endif seq_printf(m, ": %u [%u]", key->datalen, payload->type); } @@ -599,7 +604,11 @@ static void fscrypt_provisioning_key_describe(const struct key *key, static void fscrypt_provisioning_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree_sensitive(key->payload.data[0]); +#endif } static struct key_type key_type_fscrypt_provisioning = { @@ -641,7 +650,11 @@ static int get_keyring_key(u32 key_id, u32 type, if (key->type != &key_type_fscrypt_provisioning) goto bad_key; +#ifdef CONFIG_KEYP + payload = ((union key_payload *)(key->name_link.next))->data[0]; +#else payload = key->payload.data[0]; +#endif /* Don't allow fscrypt v1 keys to be used as v2 keys and vice versa. */ if (payload->type != type) @@ -1030,7 +1043,12 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) down_write(&mk->mk_sem); /* If relevant, remove current user's (or all users) claim to the key */ +#ifdef CONFIG_KEYP + if (mk->mk_users && + ((struct key_struct *)(mk->mk_users->name_link.prev))->keys.nr_leaves_on_tree != 0) { +#else if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) { +#endif if (all_users) err = keyring_clear(mk->mk_users); else @@ -1039,7 +1057,12 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) up_write(&mk->mk_sem); goto out_put_key; } +#ifdef CONFIG_KEYP + if (((struct key_struct *)(mk->mk_users->name_link.prev))-> + keys.nr_leaves_on_tree != 0) { +#else if (mk->mk_users->keys.nr_leaves_on_tree != 0) { +#endif /* * Other users have still added the key too. We removed * the current user's claim to the key, but we still @@ -1162,7 +1185,12 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) if (mk->mk_users) { struct key *mk_user; +#ifdef CONFIG_KEYP + arg.user_count = + ((struct key_struct *)(mk->mk_users->name_link.prev))->keys.nr_leaves_on_tree; +#else arg.user_count = mk->mk_users->keys.nr_leaves_on_tree; +#endif mk_user = find_master_key_user(mk); if (!IS_ERR(mk_user)) { arg.status_flags |= diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 7e0bb9a75c590..a5734069cebd6 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -110,7 +110,11 @@ find_and_lock_process_key(const char *prefix, if (IS_ERR(key)) return key; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ukp = user_key_payload_locked(key); if (!ukp) /* was the key revoked before we acquired its semaphore? */ @@ -137,7 +141,11 @@ find_and_lock_process_key(const char *prefix, return key; invalid: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return ERR_PTR(-ENOKEY); } @@ -322,7 +330,11 @@ int fscrypt_setup_v1_file_key_via_subscribed_keyrings(struct fscrypt_info *ci) return PTR_ERR(key); err = fscrypt_setup_v1_file_key(ci, payload->raw); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); return err; } diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h index c586c5db18b5e..6bc056c38205a 100644 --- a/fs/ecryptfs/ecryptfs_kernel.h +++ b/fs/ecryptfs/ecryptfs_kernel.h @@ -82,7 +82,11 @@ ecryptfs_get_encrypted_key_payload_data(struct key *key) if (key->type != &key_type_encrypted) return NULL; +#ifdef CONFIG_KEYP + payload = ((union key_payload *)(key->name_link.next))->data[0]; +#else payload = key->payload.data[0]; +#endif if (!payload) return ERR_PTR(-EKEYREVOKED); diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c index 7f9f68c00ef63..3234e9ceff581 100644 --- a/fs/ecryptfs/keystore.c +++ b/fs/ecryptfs/keystore.c @@ -512,7 +512,11 @@ ecryptfs_find_global_auth_tok_for_sig( goto out_invalid_auth_tok; } +#ifdef CONFIG_KEYP + down_write(&(KEY_SEM(walker->global_auth_tok_key))); +#else down_write(&(walker->global_auth_tok_key->sem)); +#endif rc = ecryptfs_verify_auth_tok_from_key( walker->global_auth_tok_key, auth_tok); if (rc) @@ -525,7 +529,11 @@ ecryptfs_find_global_auth_tok_for_sig( rc = -ENOENT; goto out; out_invalid_auth_tok_unlock: +#ifdef CONFIG_KEYP + up_write(&(KEY_SEM(walker->global_auth_tok_key))); +#else up_write(&(walker->global_auth_tok_key->sem)); +#endif out_invalid_auth_tok: printk(KERN_WARNING "Invalidating auth tok with sig = [%s]\n", sig); walker->flags |= ECRYPTFS_AUTH_TOK_INVALID; @@ -846,7 +854,11 @@ ecryptfs_write_tag_70_packet(char *dest, size_t *remaining_bytes, mutex_unlock(s->tfm_mutex); out: if (auth_tok_key) { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); } skcipher_request_free(s->skcipher_req); @@ -1088,7 +1100,11 @@ ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, (*filename) = NULL; } if (auth_tok_key) { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); } skcipher_request_free(s->skcipher_req); @@ -1625,10 +1641,18 @@ int ecryptfs_keyring_auth_tok_for_sig(struct key **auth_tok_key, goto out; } } +#ifdef CONFIG_KEYP + down_write(&KEY_SEM((*auth_tok_key))); +#else down_write(&(*auth_tok_key)->sem); +#endif rc = ecryptfs_verify_auth_tok_from_key(*auth_tok_key, auth_tok); if (rc) { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM((*auth_tok_key))); +#else up_write(&(*auth_tok_key)->sem); +#endif key_put(*auth_tok_key); (*auth_tok_key) = NULL; goto out; @@ -1901,7 +1925,11 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, memcpy(&(candidate_auth_tok->token.private_key), &(matching_auth_tok->token.private_key), sizeof(struct ecryptfs_private_key)); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); rc = decrypt_pki_encrypted_session_key(candidate_auth_tok, crypt_stat); @@ -1909,12 +1937,20 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, memcpy(&(candidate_auth_tok->token.password), &(matching_auth_tok->token.password), sizeof(struct ecryptfs_password)); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); rc = decrypt_passphrase_encrypted_session_key( candidate_auth_tok, crypt_stat); } else { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); rc = -EINVAL; } @@ -1976,7 +2012,11 @@ pki_encrypt_session_key(struct key *auth_tok_key, crypt_stat->cipher, crypt_stat->key_size), crypt_stat, &payload, &payload_len); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); if (rc) { ecryptfs_printk(KERN_ERR, "Error generating tag 66 packet\n"); @@ -2040,7 +2080,11 @@ write_tag_1_packet(char *dest, size_t *remaining_bytes, memcpy(key_rec->enc_key, auth_tok->session_key.encrypted_key, auth_tok->session_key.encrypted_key_size); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); goto encrypted_session_key_set; } @@ -2438,7 +2482,11 @@ ecryptfs_generate_key_packet_set(char *dest_base, &max, auth_tok, crypt_stat, key_rec, &written); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); if (rc) { ecryptfs_printk(KERN_WARNING, "Error " @@ -2467,7 +2515,11 @@ ecryptfs_generate_key_packet_set(char *dest_base, } (*len) += written; } else { +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(auth_tok_key)); +#else up_write(&(auth_tok_key->sem)); +#endif key_put(auth_tok_key); ecryptfs_printk(KERN_WARNING, "Unsupported " "authentication token type\n"); diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index 2dc927ba067fe..9c6c7e094b3a8 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -202,7 +202,11 @@ static int ecryptfs_init_global_auth_toks( goto out; } else { global_auth_tok->flags &= ~ECRYPTFS_AUTH_TOK_INVALID; +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(global_auth_tok->global_auth_tok_key)); +#else up_write(&(global_auth_tok->global_auth_tok_key)->sem); +#endif } } out: diff --git a/fs/exec.c b/fs/exec.c index 3d1b405d24d41..3d8af4a18f9e5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -85,6 +85,13 @@ #include #endif +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif +#ifdef CONFIG_PTP +#include +#endif + static int bprm_creds_from_file(struct linux_binprm *bprm); int suid_dumpable = 0; @@ -752,7 +759,11 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) free_pgd_range(&tlb, old_start, old_end, new_end, next ? next->vm_start : USER_PGTABLES_CEILING); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_finish_mmu(&tlb); +#else tlb_finish_mmu(&tlb); +#endif vma_prev(&vmi); /* Shrink the vma to just the new range */ @@ -1957,10 +1968,30 @@ static int do_execveat_common(int fd, struct filename *filename, { struct linux_binprm *bprm; int retval; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif if (IS_ERR(filename)) return PTR_ERR(filename); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) { + pr_info_once("HAOC: CONFIG_IEE_IO_CHECK enabled."); + iee_cycle_verify_cred(current); + } +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif /* * We move the actual failure in case of RLIMIT_NPROC excess from * set*uid() to execve() because too many poorly written programs @@ -2032,6 +2063,9 @@ static int do_execveat_common(int fd, struct filename *filename, out_ret: putname(filename); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif return retval; } diff --git a/fs/ioctl.c b/fs/ioctl.c index 85260a3598f89..11065563c7ea6 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -27,6 +27,14 @@ #include +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif +#ifdef CONFIG_IEE_PTRP +#include +#include +#endif + /* So that the fiemap access checks can't overflow on 32 bit machines. */ #define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) @@ -43,6 +51,17 @@ */ long vfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif int error = -ENOTTY; if (!filp->f_op->unlocked_ioctl) diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c index 4eb5574699324..0543d4061de1a 100644 --- a/fs/nfs/nfs4idmap.c +++ b/fs/nfs/nfs4idmap.c @@ -47,6 +47,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #ifdef CONFIG_CREDP @@ -230,14 +233,18 @@ int nfs_idmap_init(void) if (ret < 0) goto failed_reg_legacy; +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif id_resolver_cache = cred; return 0; @@ -306,7 +313,11 @@ static struct key *nfs_idmap_request_key(const char *name, size_t namelen, mutex_unlock(&idmap->idmap_mutex); } if (!IS_ERR(rkey)) +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(rkey, KEY_FLAG_ROOT_CAN_INVAL, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_INVAL, &rkey->flags); +#endif kfree(desc); return rkey; @@ -331,7 +342,11 @@ static ssize_t nfs_idmap_get_key(const char *name, size_t namelen, } rcu_read_lock(); +#ifdef CONFIG_KEYP + iee_set_key_perm(rkey, rkey->perm | KEY_USR_VIEW); +#else rkey->perm |= KEY_USR_VIEW; +#endif ret = key_validate(rkey); if (ret < 0) diff --git a/fs/read_write.c b/fs/read_write.c index 4771701c896ba..debe6e8f4a814 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -22,9 +22,18 @@ #include #include "internal.h" +#ifdef CONFIG_IEE_PTRP +#include +#include +#endif + #include #include +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif + const struct file_operations generic_ro_fops = { .llseek = generic_file_llseek, .read_iter = generic_file_read_iter, @@ -451,6 +460,17 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) { ssize_t ret; +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif if (!(file->f_mode & FMODE_READ)) return -EBADF; if (!(file->f_mode & FMODE_CAN_READ)) @@ -565,6 +585,17 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ { ssize_t ret; +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif if (!(file->f_mode & FMODE_WRITE)) return -EBADF; if (!(file->f_mode & FMODE_CAN_WRITE)) @@ -905,6 +936,17 @@ EXPORT_SYMBOL(vfs_iter_write); static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; struct iov_iter iter; @@ -922,6 +964,17 @@ static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif +#endif struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; struct iov_iter iter; diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c index eea9ed2bef0b0..26dc520dff8eb 100644 --- a/fs/smb/client/cifs_spnego.c +++ b/fs/smb/client/cifs_spnego.c @@ -18,6 +18,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "cifsglob.h" #include "cifs_spnego.h" #include "cifs_debug.h" @@ -37,7 +40,11 @@ cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) goto error; /* attach the data */ +#ifdef CONFIG_KEYP + ((union key_payload *)(key->name_link.next))->data[0] = payload; +#else key->payload.data[0] = payload; +#endif ret = 0; error: @@ -47,7 +54,11 @@ cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) static void cifs_spnego_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree(key->payload.data[0]); +#endif } static int @@ -184,7 +195,12 @@ cifs_get_spnego_key(struct cifs_ses *sesInfo, #ifdef CONFIG_CIFS_DEBUG2 if (cifsFYI && !IS_ERR(spnego_key)) { +#ifdef CONFIG_KEYP + struct cifs_spnego_msg *msg = + ((union key_payload *)(spnego_key->name_link.next))->data[0]; +#else struct cifs_spnego_msg *msg = spnego_key->payload.data[0]; +#endif cifs_dump_mem("SPNEGO reply blob:", msg->data, min(1024U, msg->secblob_len + msg->sesskey_len)); } @@ -232,14 +248,18 @@ init_cifs_spnego(void) * instruct request_key() to use this special keyring as a cache for * the results it looks up */ +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif spnego_cred = cred; cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring)); diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index a57b263744c2a..f3cff2b47ce64 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -20,6 +20,10 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#include +#endif #include "cifspdu.h" #include "cifsglob.h" #include "cifsacl.h" @@ -81,16 +85,33 @@ cifs_idmap_key_instantiate(struct key *key, struct key_preparsed_payload *prep) * dereference payload.data! */ if (prep->datalen <= sizeof(key->payload)) { +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + key_payload->data[0] = NULL; + memcpy(key_payload, prep->data, prep->datalen); +#else key->payload.data[0] = NULL; memcpy(&key->payload, prep->data, prep->datalen); +#endif } else { payload = kmemdup(prep->data, prep->datalen, GFP_KERNEL); if (!payload) return -ENOMEM; +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + key_payload->data[0] = payload; +#else key->payload.data[0] = payload; +#endif } +#ifdef CONFIG_KEYP + iee_set_key_datalen(key, prep->datalen); +#else key->datalen = prep->datalen; +#endif return 0; } @@ -98,7 +119,11 @@ static inline void cifs_idmap_key_destroy(struct key *key) { if (key->datalen > sizeof(key->payload)) +#ifdef CONFIG_KEYP + kfree(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree(key->payload.data[0]); +#endif } static struct key_type cifs_idmap_key_type = { @@ -314,9 +339,15 @@ id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid) * there are no subauthorities and the host has 8-byte pointers, then * it could be. */ +#ifdef CONFIG_KEYP + ksid = sidkey->datalen <= sizeof(sidkey->payload) ? + (struct smb_sid *)(sidkey->name_link.next) : + (struct smb_sid *)((union key_payload *)(sidkey->name_link.next))->data[0]; +#else ksid = sidkey->datalen <= sizeof(sidkey->payload) ? (struct smb_sid *)&sidkey->payload : (struct smb_sid *)sidkey->payload.data[0]; +#endif ksid_size = CIFS_SID_BASE_SIZE + (ksid->num_subauth * sizeof(__le32)); if (ksid_size > sidkey->datalen) { @@ -425,14 +456,24 @@ sid_to_id(struct cifs_sb_info *cifs_sb, struct smb_sid *psid, if (sidtype == SIDOWNER) { kuid_t uid; uid_t id; +#ifdef CONFIG_KEYP + memcpy(&id, + &((union key_payload *)(sidkey->name_link.next))->data[0], sizeof(uid_t)); +#else memcpy(&id, &sidkey->payload.data[0], sizeof(uid_t)); +#endif uid = make_kuid(&init_user_ns, id); if (uid_valid(uid)) fuid = uid; } else { kgid_t gid; gid_t id; +#ifdef CONFIG_KEYP + memcpy(&id, &((union key_payload *)(sidkey->name_link.next))->data[0], + sizeof(gid_t)); +#else memcpy(&id, &sidkey->payload.data[0], sizeof(gid_t)); +#endif gid = make_kgid(&init_user_ns, id); if (gid_valid(gid)) fgid = gid; @@ -493,14 +534,18 @@ init_cifs_idmap(void) /* instruct request_key() to use this special keyring as a cache for * the results it looks up */ +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif root_cred = cred; cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring)); diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index fe5d21f174c47..5e165e87f3cfe 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -2208,7 +2208,11 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) is_domain = 1; } +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif upayload = user_key_payload_locked(key); if (IS_ERR_OR_NULL(upayload)) { rc = upayload ? PTR_ERR(upayload) : -EINVAL; @@ -2285,7 +2289,11 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) strscpy(ctx->workstation_name, ses->workstation_name, sizeof(ctx->workstation_name)); out_key_put: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif key_put(key); out_err: kfree(desc); diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c index bbde7180a90ac..4a8cff5a1810f 100644 --- a/fs/smb/client/sess.c +++ b/fs/smb/client/sess.c @@ -1625,7 +1625,11 @@ sess_auth_kerberos(struct sess_data *sess_data) goto out; } +#ifdef CONFIG_KEYP + msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; +#else msg = spnego_key->payload.data[0]; +#endif /* * check version field to make sure that cifs.upcall is * sending us a response in an expected form diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 7f26e6fdd6abf..2c4fb76afaea2 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -1644,7 +1644,11 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) goto out; } +#ifdef CONFIG_KEYP + msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; +#else msg = spnego_key->payload.data[0]; +#endif /* * check version field to make sure that cifs.upcall is * sending us a response in an expected form diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index e564d5ff87816..3503af364d878 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -284,7 +284,11 @@ int ubifs_init_authentication(struct ubifs_info *c) return PTR_ERR(keyring_key); } +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(keyring_key)); +#else down_read(&keyring_key->sem); +#endif if (keyring_key->type != &key_type_logon) { ubifs_err(c, "key type must be logon"); @@ -351,7 +355,11 @@ int ubifs_init_authentication(struct ubifs_info *c) if (err) crypto_free_shash(c->hash_tfm); out: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(keyring_key)); +#else up_read(&keyring_key->sem); +#endif key_put(keyring_key); return err; diff --git a/fs/verity/signature.c b/fs/verity/signature.c index 90c07573dd77b..3aafb8d467028 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -62,7 +62,12 @@ int fsverity_verify_signature(const struct fsverity_info *vi, return 0; } +#ifdef CONFIG_KEYP + if (((struct key_struct *)(fsverity_keyring->name_link.prev))-> + keys.nr_leaves_on_tree == 0) { +#else if (fsverity_keyring->keys.nr_leaves_on_tree == 0) { +#endif /* * The ".fs-verity" keyring is empty, due to builtin signatures * being supported by the kernel but not actually being used. diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 681d25fe36174..fae22f47535fe 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -210,6 +210,9 @@ struct mmu_table_batch { ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) extern void tlb_remove_table(struct mmu_gather *tlb, void *table); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +extern void ptp_tlb_remove_table(struct mmu_gather *tlb, void *table); +#endif #else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */ @@ -218,6 +221,9 @@ extern void tlb_remove_table(struct mmu_gather *tlb, void *table); * page directories and we can use the normal page batching to free them. */ #define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page)) +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +#define ptp_tlb_remove_table(tlb, page) ptp_tlb_remove_page((tlb), (page)) +#endif #endif /* CONFIG_MMU_GATHER_TABLE_FREE */ @@ -376,6 +382,9 @@ struct mmu_gather { }; void tlb_flush_mmu(struct mmu_gather *tlb); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +void ptp_tlb_flush_mmu(struct mmu_gather *tlb); +#endif static inline void __tlb_adjust_range(struct mmu_gather *tlb, unsigned long address, @@ -492,6 +501,15 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb, tlb_flush_mmu(tlb); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static inline void ptp_tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + if (__tlb_remove_page_size(tlb, page, false, page_size)) + ptp_tlb_flush_mmu(tlb); +} +#endif + static __always_inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page, bool delay_rmap) { @@ -507,15 +525,30 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) return tlb_remove_page_size(tlb, page, PAGE_SIZE); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static inline void ptp_tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + return ptp_tlb_remove_page_size(tlb, page, PAGE_SIZE); +} +#endif + static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) { +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_remove_table(tlb, pt); +#else tlb_remove_table(tlb, pt); +#endif } /* Like tlb_remove_ptdesc, but for page-like page directories. */ static inline void tlb_remove_page_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) { +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_remove_table(tlb, ptdesc_page(pt)); +#else tlb_remove_page(tlb, ptdesc_page(pt)); +#endif } static inline void tlb_change_page_size(struct mmu_gather *tlb, diff --git a/include/keys/asymmetric-subtype.h b/include/keys/asymmetric-subtype.h index d55171f640a04..d996bd0476b50 100644 --- a/include/keys/asymmetric-subtype.h +++ b/include/keys/asymmetric-subtype.h @@ -54,7 +54,11 @@ struct asymmetric_key_subtype { static inline struct asymmetric_key_subtype *asymmetric_key_subtype(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[asym_subtype]; +#else return key->payload.data[asym_subtype]; +#endif } #endif /* _KEYS_ASYMMETRIC_SUBTYPE_H */ diff --git a/include/keys/asymmetric-type.h b/include/keys/asymmetric-type.h index 69a13e1e5b2e5..0d009989e3cee 100644 --- a/include/keys/asymmetric-type.h +++ b/include/keys/asymmetric-type.h @@ -69,13 +69,21 @@ extern struct asymmetric_key_id *asymmetric_key_generate_id(const void *val_1, static inline const struct asymmetric_key_ids *asymmetric_key_ids(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[asym_key_ids]; +#else return key->payload.data[asym_key_ids]; +#endif } static inline const struct public_key *asymmetric_key_public_key(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[asym_crypto]; +#else return key->payload.data[asym_crypto]; +#endif } extern struct key *find_asymmetric_key(struct key *keyring, diff --git a/include/keys/request_key_auth-type.h b/include/keys/request_key_auth-type.h index 01e42ee5f4099..7fc483cde2684 100644 --- a/include/keys/request_key_auth-type.h +++ b/include/keys/request_key_auth-type.h @@ -28,7 +28,11 @@ struct request_key_auth { static inline struct request_key_auth *get_request_key_auth(const struct key *key) { +#ifdef CONFIG_KEYP + return ((union key_payload *)(key->name_link.next))->data[0]; +#else return key->payload.data[0]; +#endif } diff --git a/include/linux/efi.h b/include/linux/efi.h index b5c69570c9be6..1a65f01c8207d 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -28,6 +28,10 @@ #include +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +#include +#endif + struct screen_info; #define EFI_SUCCESS 0 diff --git a/include/linux/haoc-ptp.h b/include/linux/haoc-ptp.h new file mode 100644 index 0000000000000..29b4d6b4eaeea --- /dev/null +++ b/include/linux/haoc-ptp.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HAOC_PTP_H +#define _LINUX_HAOC_PTP_H + +extern void ptp_disable_wp(unsigned long *cr0); +extern void ptp_restore_wp(unsigned long cr0); +extern void ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0); +extern void ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0); + +#endif /* _LINUX_HAOC_PTP_H */ diff --git a/include/linux/key.h b/include/linux/key.h index ffd0c00e89c71..5c38bc6aedc03 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -283,6 +283,50 @@ struct key { DEEPIN_KABI_RESERVE(2) }; +#ifdef CONFIG_KEYP +struct key_union { + union { + struct list_head graveyard_link; + struct rb_node serial_node; + }; + struct rw_semaphore sem; + struct key *key; +}; + +struct key_struct { + struct { + /* Keyring bits */ + struct list_head name_link; + struct assoc_array keys; + }; + struct key *key; +}; +#define KEY_SEM(KEY) (((struct key_union *)(KEY->graveyard_link.next))->sem) +#include +extern bool haoc_enabled; +extern unsigned long long iee_rw_gate(int flag, ...); +static bool iee_set_key_usage(struct key *key, int n, int flag) +{ + if (!haoc_enabled) { + switch (flag) { + case REFCOUNT_INC: + refcount_inc(&key->usage); + break; + case REFCOUNT_SET: + refcount_set(&key->usage, n); + break; + case REFCOUNT_DEC_AND_TEST: + return refcount_dec_and_test(&key->usage); + case REFCOUNT_INC_NOT_ZERO: + return refcount_inc_not_zero(&key->usage); + } + return false; + } + + return iee_rw_gate(IEE_OP_SET_KEY_USAGE, key, n, flag); +} +#endif + extern struct key *key_alloc(struct key_type *type, const char *desc, kuid_t uid, kgid_t gid, @@ -308,7 +352,11 @@ extern void key_remove_domain(struct key_tag *domain_tag); static inline struct key *__key_get(struct key *key) { +#ifdef CONFIG_KEYP + iee_set_key_usage(key, 0, REFCOUNT_INC); +#else refcount_inc(&key->usage); +#endif return key; } @@ -481,6 +529,47 @@ static inline bool key_is_negative(const struct key *key) return key_read_state(key) < 0; } +#ifdef CONFIG_KEYP +static inline void iee_write_key_payload_rcu_data0(struct key *key, void *rcu_data0) +{ + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + WRITE_ONCE(key_payload->rcu_data0, rcu_data0); +} + +#define dereference_key_rcu(KEY) \ + (rcu_dereference(((union key_payload *)(KEY->name_link.next))->rcu_data0)) + +#define dereference_key_locked(KEY) \ + (rcu_dereference_protected(((union key_payload *)(KEY->name_link.next))->rcu_data0, \ + rwsem_is_locked(&KEY_SEM(((struct key *)(KEY)))))) + +#define iee_smp_store_release(p, v, KEY) \ +do { \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + iee_write_key_payload_rcu_data0(KEY, v); \ +} while (0) + +#define iee_rcu_assign_pointer(p, v, KEY) \ +do { \ + uintptr_t _r_a_p__v = (uintptr_t)(v); \ + rcu_check_sparse(p, __rcu); \ + \ + if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \ + iee_write_key_payload_rcu_data0(KEY, (typeof(p))(_r_a_p__v)); \ + else \ + iee_smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v), KEY); \ +} while (0) + +#define rcu_assign_keypointer(KEY, PAYLOAD) \ +do { \ + union key_payload *tmp = (union key_payload *)(KEY->name_link.next); \ + \ + iee_rcu_assign_pointer(tmp->rcu_data0, (PAYLOAD), KEY); \ +} while (0) + +#else #define dereference_key_rcu(KEY) \ (rcu_dereference((KEY)->payload.rcu_data0)) @@ -492,6 +581,7 @@ static inline bool key_is_negative(const struct key *key) do { \ rcu_assign_pointer((KEY)->payload.rcu_data0, (PAYLOAD)); \ } while (0) +#endif /* * the userspace interface diff --git a/include/linux/mm.h b/include/linux/mm.h index 7e2f930539d94..bb6ef72d75d4f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -31,6 +31,9 @@ #include #include #include +#ifdef CONFIG_PTP +#include +#endif struct mempolicy; struct anon_vma; @@ -2933,7 +2936,15 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) */ static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) { +#ifdef CONFIG_PTP + struct page *page; + void *pt_page; + + pt_page = iee_cache_alloc(&pg_cache, gfp); + page = !!pt_page ? virt_to_page(pt_page) : NULL; +#else struct page *page = alloc_pages(gfp | __GFP_COMP, order); +#endif return page_ptdesc(page); } @@ -2949,7 +2960,11 @@ static inline void pagetable_free(struct ptdesc *pt) { struct page *page = ptdesc_page(pt); +#ifdef CONFIG_PTP + iee_cache_free(&pg_cache, page_address(page)); +#else __free_pages(page, compound_order(page)); +#endif } #if USE_SPLIT_PTE_PTLOCKS diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c6878e5c39684..d87c7ac08e7e4 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1259,6 +1259,9 @@ extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma); extern void tlb_finish_mmu(struct mmu_gather *tlb); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +extern void ptp_tlb_finish_mmu(struct mmu_gather *tlb); +#endif struct vm_fault; diff --git a/include/linux/ptp-cache.h b/include/linux/ptp-cache.h new file mode 100644 index 0000000000000..b102b4e9bf1af --- /dev/null +++ b/include/linux/ptp-cache.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PTP_CACHE_H +#define _LINUX_PTP_CACHE_H +#include +#include +union freelist_aba_t { + struct { + void **freelist; + unsigned long counter; + }; + u128 full; +}; + +struct iee_cache { + union { + struct { + void **freelist; + unsigned long tid; + }; + union freelist_aba_t freelist_tid; + }; + unsigned long reserve_order; + unsigned long reserve_start_addr; + unsigned long reserve_end_addr; + unsigned long object_order; + int levels; + enum HAOC_BITMAP_TYPE name; + atomic_t count; + atomic_t fail_count; + bool init; +}; + +extern struct iee_cache pg_cache; +#ifdef CONFIG_PTP_S +extern struct iee_cache pg_user_cache; +extern bool ptp_is_user_pgtable(const void *ptp); +#endif +extern void iee_cache_init(struct iee_cache *cache, unsigned long object_order, + int levels, enum HAOC_BITMAP_TYPE bitmap_type, + unsigned long reserve_order); +extern void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp); +extern void iee_cache_free(struct iee_cache *cache, void *object); +#endif diff --git a/init/main.c b/init/main.c index 1187c46b11dca..cfb550a81208d 100644 --- a/init/main.c +++ b/init/main.c @@ -106,6 +106,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #define CREATE_TRACE_POINTS #include @@ -1044,6 +1047,10 @@ void start_kernel(void) arch_post_acpi_subsys_init(); kcsan_init(); +#if defined(CONFIG_IEE) && defined(CONFIG_X86_64) + iee_post_init(); +#endif + /* Do the rest non-__init'ed, we're now alive */ arch_call_rest_init(); diff --git a/kernel/cred.c b/kernel/cred.c index d7042a4633408..a9fa46f465d2c 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -619,11 +619,11 @@ void abort_creds(struct cred *new) atomic_long_read(&new->usage)); BUG_ON(atomic_long_read(&new->usage) < 1); - put_cred(new); #ifdef CONFIG_CREDP if (haoc_enabled) iee_abort_creds(new); #endif + put_cred(new); } EXPORT_SYMBOL(abort_creds); @@ -650,7 +650,14 @@ const struct cred *override_creds(const struct cred *new) * visible to other threads under RCU. */ get_new_cred((struct cred *)new); +#ifdef CONFIG_CREDP + if (haoc_enabled) + iee_override_creds(new); + else + rcu_assign_pointer(current->cred, new); +#else rcu_assign_pointer(current->cred, new); +#endif kdebug("override_creds() = %p{%ld}", old, atomic_long_read(&old->usage)); @@ -672,7 +679,14 @@ void revert_creds(const struct cred *old) kdebug("revert_creds(%p{%ld})", old, atomic_long_read(&old->usage)); +#ifdef CONFIG_CREDP + if (haoc_enabled) + iee_revert_creds(old); + else + rcu_assign_pointer(current->cred, old); +#else rcu_assign_pointer(current->cred, old); +#endif put_cred(override); } EXPORT_SYMBOL(revert_creds); diff --git a/kernel/fork.c b/kernel/fork.c index ec9c97caa8f21..ff6843d749bf3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -117,6 +117,16 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_IEE +#include +#endif +#ifdef CONFIG_PTP +#include +#endif + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif /* * Minimum number of threads to boot the kernel @@ -2389,8 +2399,19 @@ __latent_entropy struct task_struct *copy_process( if (!p) goto fork_out; #ifdef CONFIG_IEE_PTRP - if(haoc_enabled) + if (haoc_enabled) iee_validate_token(p); +#endif +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_IO_CHECK + if (haoc_enabled) + iee_cycle_verify_cred(current); +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(current); +# endif #endif p->flags &= ~PF_KTHREAD; if (args->kthread) @@ -2917,6 +2938,11 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) return copy_process(NULL, 0, node, &args); } +#ifdef CONFIG_PTP +void __weak ptp_disable_wp(unsigned long *cr0) { } +void __weak ptp_restore_wp(unsigned long cr0) { } +#endif + /* * Ok, this is the main fork-routine. * @@ -2933,6 +2959,9 @@ pid_t kernel_clone(struct kernel_clone_args *args) struct task_struct *p; int trace = 0; pid_t nr; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif /* * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument @@ -2966,7 +2995,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) trace = 0; } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif p = copy_process(NULL, trace, NUMA_NO_NODE, args); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif add_latent_entropy(); if (IS_ERR(p)) diff --git a/kernel/module/kmod.c b/kernel/module/kmod.c index 0800d98916921..f656d6575ab54 100644 --- a/kernel/module/kmod.c +++ b/kernel/module/kmod.c @@ -32,6 +32,10 @@ #include #include "internal.h" +#ifdef CONFIG_VARP +#include +#endif + /* * Assuming: * @@ -61,7 +65,11 @@ static DEFINE_SEMAPHORE(kmod_concurrent_max, MAX_KMOD_CONCURRENT); /* modprobe_path is set via /proc/sys. */ +#ifdef CONFIG_VARP +char modprobe_path[KMOD_PATH_LEN] __iee_varp_data = CONFIG_MODPROBE_PATH; +#else char modprobe_path[KMOD_PATH_LEN] = CONFIG_MODPROBE_PATH; +#endif static void free_modprobe_argv(struct subprocess_info *info) { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 779aa2efa09db..e38438537bd9e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -100,6 +100,9 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_PTP +#include +#endif EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); @@ -5164,6 +5167,15 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) calculate_sigpending(); } +#ifdef CONFIG_PTP +void __weak ptp_context_enable_wp(int *wp_disabled_cnt, unsigned long *cr0) { } +void __weak ptp_context_restore_wp(int wp_disabled_cnt, unsigned long cr0) { } +#endif + +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) && defined(CONFIG_ARM64) +extern void iee_cycle_verify_cred(struct task_struct *current_task); +#endif + /* * context_switch - switch to the new MM and the new thread's register state. */ @@ -5171,6 +5183,19 @@ static __always_inline struct rq * context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf) { +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) +# if defined(CONFIG_ARM64) +# ifdef CONFIG_IEE_CS_CHECK + if (haoc_enabled) { + pr_info_once("HAOC: CONFIG_IEE_CS_CHECK enabled."); + iee_cycle_verify_cred(next); + } +# endif +# elif defined(CONFIG_X86_64) + if (haoc_enabled) + iee_verify_token(next); +# endif +#endif prepare_task_switch(rq, prev, next); /* @@ -5208,9 +5233,9 @@ context_switch(struct rq *rq, struct task_struct *prev, * case 'prev->active_mm == next->mm' through * finish_task_switch()'s mmdrop(). */ -#ifdef CONFIG_IEE_PTRP - if(haoc_enabled) - iee_verify_token_pgd(next); +#if defined(CONFIG_IEE_PTRP) && !defined(CONFIG_IEE_PTRP_W) + if (haoc_enabled) + iee_verify_pgd(next); #endif switch_mm_irqs_off(prev->active_mm, next->mm, next); lru_gen_use_mm(next->mm); @@ -5227,10 +5252,22 @@ context_switch(struct rq *rq, struct task_struct *prev, prepare_lock_switch(rq, next, rf); + /* Here we just switch the register state and the stack. */ +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + int wp_disabled_cnt = 0; + unsigned long cr0 = 0; + + ptp_context_enable_wp(&wp_disabled_cnt, &cr0); +#endif + /* Here we just switch the register state and the stack. */ switch_to(prev, next, prev); barrier(); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_context_restore_wp(wp_disabled_cnt, cr0); +#endif + return finish_task_switch(prev); } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 4eee1add47412..3b72b6d7082a1 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -80,6 +80,9 @@ #ifdef CONFIG_RT_MUTEXES #include #endif +#ifdef CONFIG_VARP +#include +#endif /* shared constants to be used in various sysctls */ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; @@ -205,6 +208,37 @@ static int _proc_do_string(char *data, int maxlen, int write, return 0; } +#ifdef CONFIG_VARP +static int _iee_proc_do_string(char *data, int maxlen, int write, + char *buffer, size_t *lenp, loff_t *ppos) +{ + size_t len; + + if (!data || !maxlen || !*lenp || !buffer) { + *lenp = 0; + return 0; + } + + if (sysctl_writes_strict == SYSCTL_WRITES_STRICT) { + /* Only continue writes not past the end of buffer. */ + len = strlen(data); + if (len > maxlen - 1) + len = maxlen - 1; + + if (*ppos > len) + return 0; + len = *ppos; + } else { + /* Start writing from beginning of buffer. */ + len = 0; + } + + *ppos += *lenp; + iee_set_varp_modprobe_path(data, maxlen, len, buffer, lenp); + return 0; +} +#endif + static void warn_sysctl_write(struct ctl_table *table) { pr_warn_once("%s wrote to %s when file position was not 0!\n" @@ -262,6 +296,12 @@ int proc_dostring(struct ctl_table *table, int write, if (write) proc_first_pos_non_zero_ignore(ppos, table); +#ifdef CONFIG_VARP + if (table && table->data == modprobe_path && write) + return _iee_proc_do_string(table->data, table->maxlen, write, buffer, lenp, + ppos); +#endif + return _proc_do_string(table->data, table->maxlen, write, buffer, lenp, ppos); } diff --git a/lib/digsig.c b/lib/digsig.c index 04b5e55ed95f5..33ebb03cd2aed 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -81,7 +81,11 @@ static int digsig_verify_rsa(struct key *key, const struct user_key_payload *ukp; struct pubkey_hdr *pkh; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ukp = user_key_payload_locked(key); if (!ukp) { @@ -176,7 +180,11 @@ static int digsig_verify_rsa(struct key *key, while (--i >= 0) mpi_free(pkey[i]); err1: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif return err; } diff --git a/mm/Makefile b/mm/Makefile index 92060644c70ee..bc67de29f6c8b 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -54,6 +54,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ compaction.o show_mem.o shmem_quota.o\ interval_tree.o list_lru.o workingset.o \ debug.o gup.o mmap_lock.o $(mmu-y) +obj-$(CONFIG_PTP) += haoc/ # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 4470336b9cab5..0036ab1a396d6 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -437,7 +437,11 @@ static void __init pmd_huge_tests(struct pgtable_debug_args *args) * X86 defined pmd_set_huge() verifies that the given * PMD is not a populated non-leaf entry. */ +#ifdef CONFIG_PTP + set_pmd(args->pmdp, __pmd(0)); +#else WRITE_ONCE(*args->pmdp, __pmd(0)); +#endif WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); WARN_ON(!pmd_clear_huge(args->pmdp)); pmd = READ_ONCE(*args->pmdp); @@ -457,7 +461,11 @@ static void __init pud_huge_tests(struct pgtable_debug_args *args) * X86 defined pud_set_huge() verifies that the given * PUD is not a populated non-leaf entry. */ +#ifdef CONFIG_PTP + set_pud(args->pudp, __pud(0)); +#else WRITE_ONCE(*args->pudp, __pud(0)); +#endif WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); WARN_ON(!pud_clear_huge(args->pudp)); pud = READ_ONCE(*args->pudp); diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c index ce06b2884789a..3ee71ba8e7823 100644 --- a/mm/early_ioremap.c +++ b/mm/early_ioremap.c @@ -20,7 +20,11 @@ #include "internal.h" #ifdef CONFIG_MMU +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +int early_ioremap_debug __initdata; +#else static int early_ioremap_debug __initdata; +#endif static int __init early_ioremap_debug_setup(char *str) { @@ -30,7 +34,11 @@ static int __init early_ioremap_debug_setup(char *str) } early_param("early_ioremap_debug", early_ioremap_debug_setup); +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +int after_paging_init __initdata; +#else static int after_paging_init __initdata; +#endif pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr, unsigned long size, @@ -64,9 +72,15 @@ static inline void __init __late_clear_fixmap(enum fixed_addresses idx) } #endif +#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) +void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; +unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; +unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; +#else static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; +#endif void __init early_ioremap_setup(void) { @@ -147,7 +161,11 @@ __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot) if (after_paging_init) __late_set_fixmap(idx, phys_addr, prot); else + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(idx, phys_addr, prot); + #else __early_set_fixmap(idx, phys_addr, prot); + #endif phys_addr += PAGE_SIZE; --idx; --nrpages; @@ -199,7 +217,11 @@ void __init early_iounmap(void __iomem *addr, unsigned long size) if (after_paging_init) __late_clear_fixmap(idx); else + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR); + #else __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); + #endif --idx; --nrpages; } diff --git a/mm/haoc/Makefile b/mm/haoc/Makefile new file mode 100644 index 0000000000000..ad37f99f562d2 --- /dev/null +++ b/mm/haoc/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_PTP) += ptp-pg_cache.o diff --git a/mm/haoc/ptp-pg_cache.c b/mm/haoc/ptp-pg_cache.c new file mode 100644 index 0000000000000..fcbf1823f5998 --- /dev/null +++ b/mm/haoc/ptp-pg_cache.c @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(CONFIG_PTP_S) && defined(CONFIG_X86_64) +#include +#endif + +struct iee_cache pg_cache; +#ifdef CONFIG_PTP_S +struct iee_cache pg_user_cache; +#endif + +int expend_num; +#ifdef CONFIG_X86_64 +static LIST_HEAD(iee_cache_pending_protect_list); +static DEFINE_SPINLOCK(iee_cache_pending_protect_lock); +static atomic_t iee_cache_pending_protect_count = ATOMIC_INIT(0); + +enum iee_cache_deferred_op { + IEE_CACHE_DEFER_PROTECT, + IEE_CACHE_DEFER_FREE, +}; + +struct iee_cache_deferred_work { + struct work_struct work; + struct list_head list; + void *object; + unsigned int order; + int num_pages; + enum HAOC_BITMAP_TYPE type; + enum iee_cache_deferred_op op; + bool free_after_protect; +}; + +static void *iee_cache_alloc_pending_protect(struct iee_cache *cache, gfp_t gfp); +static bool iee_cache_mark_pending_protect_free(void *object); +#endif + +static inline void *__get_freepointer(void *object) +{ + return *((void **)object); +}; + +static inline void __set_freepointer(void *object, void *next_object) +{ + *(void **)object = next_object; +}; + +static inline void __iee_set_freepointer(void *object, void *next_object) +{ + iee_set_freeptr(object, next_object); +}; + +static inline bool __update_freelist(struct iee_cache *cache, + void *freelist_old, void *freelist_new, + unsigned long tid) +{ + union freelist_aba_t old = { .freelist = freelist_old, .counter = tid }; + union freelist_aba_t new = { .freelist = freelist_new, .counter = tid + 1 }; + + return try_cmpxchg128(&(cache->freelist_tid.full), &old.full, new.full); +} + +#ifdef CONFIG_ARM64 +static void __ptp_set_iee_pages(unsigned long start_addr, unsigned long end_addr, + struct iee_cache *cache) +{ + unsigned long addr; + + if (!haoc_enabled) + return; + + if (start_addr != ALIGN(start_addr, PMD_SIZE)) + panic("IEE: pool (HAOC_BITMAP_TYPE %u) not PMD-aligned.", + (unsigned int)cache->name); + + addr = start_addr; + while (addr < end_addr) { + set_iee_page(addr, PMD_ORDER, cache->name); + addr += PMD_SIZE; + } + flush_tlb_kernel_range(start_addr, end_addr); +} +#endif + +void __init iee_cache_init(struct iee_cache *cache, unsigned long object_order, + int levels, enum HAOC_BITMAP_TYPE bitmap_type, + unsigned long reserve_order) +{ + unsigned long addr; + unsigned long addr_next; + unsigned long reserve_pages; + unsigned long start_addr; + unsigned long end_addr; + unsigned long object_size; + struct page *page; + + if (!haoc_enabled && bitmap_type != IEE_PGTABLE +#ifdef CONFIG_PTP_S + && bitmap_type != IEE_USER_PGTABLE +#endif + ) + return; + + object_size = (1 << object_order) * PAGE_SIZE; + while (1) { + reserve_pages = (1 << reserve_order) * levels; + start_addr = (unsigned long)memblock_alloc(reserve_pages * PAGE_SIZE, + reserve_pages * PAGE_SIZE); + if (start_addr) + break; + reserve_order--; + /* Allocate pages in pmd blocks to reduce the mapping cost. */ + if (reserve_order < PMD_ORDER) + panic("IEE: fail to reserve pages for HAOC_BITMAP_TYPE %u", + (unsigned int)bitmap_type); + } + end_addr = start_addr + reserve_pages * PAGE_SIZE; + pr_err("IEE: reserve %lu pages for HAOC_BITMAP_TYPE %u, range[0x%lx, 0x%lx]", + reserve_pages, (unsigned int)bitmap_type, start_addr, end_addr); + + addr = start_addr; + addr_next = addr + object_size; + while (addr < end_addr) { + page = virt_to_page(addr); + set_page_count(page, 1); + __set_freepointer((void *)addr, (void *)addr_next); + addr += object_size; + addr_next += object_size; + } + __set_freepointer((void *)(addr - object_size), NULL); + + cache->object_order = object_order; + cache->reserve_order = reserve_order; + cache->reserve_start_addr = start_addr; + cache->reserve_end_addr = end_addr; + cache->levels = levels; + cache->freelist = (void *)start_addr; + cache->tid = 0; + cache->name = bitmap_type; + cache->init = true; + +#ifdef CONFIG_ARM64 + /* IEE for ARM64 needs to access these data by IEE addresses. */ + __ptp_set_iee_pages(start_addr, end_addr, cache); +#endif +#ifdef DEBUG + atomic_set(&cache->count, reserve_pages); + atomic_set(&cache->fail_count, 0); + pr_info("IEE: HAOC_BITMAP_TYPE %u ready. object size 0x%lx, count %d.", + (unsigned int)cache->name, object_size, atomic_read(&cache->count)); +#endif +} + +#ifdef CONFIG_ARM64 +/* Expand the cache pool with PMD_SIZE for each time. */ +static __maybe_unused int __ref iee_cache_expand(struct iee_cache *cache, gfp_t gfp) +{ + unsigned long addr; + unsigned long addr_next; + unsigned long start_addr; + unsigned long end_addr; + unsigned long object_size; + unsigned long tid; + void **freelist; + struct page *page; + + if (slab_is_available()) + start_addr = __get_free_pages(gfp, PMD_ORDER); + else + start_addr = (unsigned long)memblock_alloc(PMD_SIZE, PMD_SIZE); + + if (!start_addr) + return 0; + + addr = start_addr; + object_size = (1 << cache->object_order) * PAGE_SIZE; + addr_next = addr + object_size; + end_addr = start_addr + PMD_SIZE; + while (addr < end_addr) { + page = virt_to_page(addr); + set_page_count(page, 1); + __set_freepointer((void *)addr, (void *)addr_next); + addr += object_size; + addr_next += object_size; + } + __set_freepointer((void *)(addr - object_size), NULL); + + /* IEE for ARM64 needs to access these data by IEE addresses. */ + __ptp_set_iee_pages(start_addr, end_addr, cache); + +#ifdef DEBUG + atomic_add(1 << PMD_ORDER, &cache->count); + pr_info("IEE: HAOC_BITMAP_TYPE %u expand to count %d. Curr failed: %d", + (unsigned int)cache->name, atomic_read(&cache->count), + atomic_read(&cache->fail_count)); +#endif + +/* Fill the new allocated pages into the cache freelist. */ +redo: + tid = READ_ONCE(cache->tid); + barrier(); + freelist = READ_ONCE(cache->freelist); +#ifdef CONFIG_PTP_S + if (cache->name == IEE_USER_PGTABLE) + __set_freepointer((void *)(end_addr - object_size), freelist); + else +#endif + __iee_set_freepointer((void *)(end_addr - object_size), freelist); + if (unlikely(!__update_freelist(cache, freelist, (void *)start_addr, tid))) + goto redo; + + expend_num++; + pr_alert("gwm %s num %d , start_addr: 0x%lx, end_addr: 0x%lx, start_addr-end_addr: 0x%lx\n", + __func__, expend_num, start_addr, end_addr, start_addr - end_addr); + return 1; +} +#endif + +void *iee_cache_alloc(struct iee_cache *cache, gfp_t gfp) +{ + unsigned long tid; + size_t object_size; + void *object; + void *next_object; + +#ifdef CONFIG_PTP_S +#ifdef CONFIG_ARM64 + if (haoc_enabled && iee_init_done) +#else + if (haoc_enabled && haoc_init_done) +#endif + if ((gfp & __GFP_ACCOUNT) && cache == &pg_cache) + cache = &pg_user_cache; +#endif + + if (!cache->init) + return (void *)__get_free_pages(gfp, cache->object_order); + + object_size = (1UL << cache->object_order) * PAGE_SIZE; +redo: + tid = READ_ONCE(cache->tid); + barrier(); + object = READ_ONCE(cache->freelist); + if (unlikely(!object)) { + // slow path alloc +#ifdef CONFIG_ARM64 + // if (iee_cache_expand(cache, gfp) || READ_ONCE(cache->freelist)) + // goto redo; + + /* If the expandsion failed, alloc a singel object without RO protection to + * avoid block spliting. + */ + object = (void *)__get_free_pages(gfp, cache->object_order); + if (!object) + return NULL; + set_iee_address_valid((unsigned long)object, cache->object_order); + iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, cache->name); +#ifdef DEBUG + WARN_ONCE(1, "IEE: Failed on HAOC_BITMAP_TYPE %u expansion.", + (unsigned int)cache->name); + atomic_add(1 << cache->object_order, &cache->fail_count); + #endif +#else + if (!preemptible()) + return iee_cache_alloc_pending_protect(cache, gfp); + object = (void *)__get_free_pages(gfp, cache->object_order); + if (!object) + return NULL; + set_iee_page((unsigned long)object, 1 << cache->object_order, cache->name); +#endif + } else { + /* fast path alloc */ + next_object = __get_freepointer(object); + if (unlikely(!__update_freelist(cache, object, next_object, tid))) + goto redo; + prefetchw(next_object); + if (gfp & __GFP_ZERO) { +#ifdef CONFIG_PTP_S + if (cache == &pg_user_cache) + clear_page(object); + else +#endif + iee_memset(object, 0, object_size); + } + } + return object; +}; + +#ifdef CONFIG_X86_64 +static void iee_cache_deferred_work(struct work_struct *work) +{ + struct iee_cache_deferred_work *deferred = + container_of(work, struct iee_cache_deferred_work, work); + unsigned long addr = (unsigned long)deferred->object; + bool free_after_protect = false; + + if (deferred->op == IEE_CACHE_DEFER_PROTECT) { + set_iee_page(addr, deferred->num_pages, deferred->type); + + spin_lock(&iee_cache_pending_protect_lock); + list_del(&deferred->list); + free_after_protect = deferred->free_after_protect; + atomic_dec(&iee_cache_pending_protect_count); + spin_unlock(&iee_cache_pending_protect_lock); + + if (!free_after_protect) + goto out; + } + + unset_iee_page(addr, deferred->num_pages); + free_pages(addr, deferred->order); + +out: + kfree(deferred); +} + +static void *iee_cache_alloc_pending_protect(struct iee_cache *cache, gfp_t gfp) +{ + struct iee_cache_deferred_work *deferred; + gfp_t atomic_gfp = GFP_ATOMIC; + void *object; + + if (gfp & __GFP_ZERO) + atomic_gfp |= __GFP_ZERO; + if (gfp & __GFP_ACCOUNT) + atomic_gfp |= __GFP_ACCOUNT; + + deferred = kmalloc(sizeof(*deferred), GFP_ATOMIC); + if (!deferred) { + WARN_ONCE(1, + "IEE: failed to allocate pending PTP protect work\n"); + return NULL; + } + + object = (void *)__get_free_pages(atomic_gfp, cache->object_order); + if (!object) { + kfree(deferred); + return NULL; + } + + deferred->object = object; + deferred->order = cache->object_order; + deferred->num_pages = 1 << cache->object_order; + deferred->type = cache->name; + deferred->op = IEE_CACHE_DEFER_PROTECT; + deferred->free_after_protect = false; + INIT_WORK(&deferred->work, iee_cache_deferred_work); + + spin_lock(&iee_cache_pending_protect_lock); + list_add(&deferred->list, &iee_cache_pending_protect_list); + atomic_inc(&iee_cache_pending_protect_count); + spin_unlock(&iee_cache_pending_protect_lock); + + WARN_ONCE(1, + "IEE: returning pending-protect PTP page, count=%d\n", + atomic_read(&iee_cache_pending_protect_count)); + queue_work(system_unbound_wq, &deferred->work); + + return object; +} + +static bool iee_cache_mark_pending_protect_free(void *object) +{ + struct iee_cache_deferred_work *deferred; + bool found = false; + + spin_lock(&iee_cache_pending_protect_lock); + list_for_each_entry(deferred, &iee_cache_pending_protect_list, list) { + if (deferred->object == object) { + deferred->free_after_protect = true; + found = true; + break; + } + } + spin_unlock(&iee_cache_pending_protect_lock); + + return found; +} + +/* + * Pool overflow pages are freed from exit_mmap/zap_* with preemption + * disabled. unset_iee_page() calls set_memory_rw() and must not run there. + */ +static void iee_cache_slow_free_pages(struct iee_cache *cache, void *object) +{ + unsigned long addr = (unsigned long)object; + int num_pages = 1 << cache->object_order; + struct iee_cache_deferred_work *deferred; + + if (preemptible()) { + unset_iee_page(addr, num_pages); + free_pages(addr, cache->object_order); + return; + } + + deferred = kmalloc(sizeof(*deferred), GFP_ATOMIC); + if (!deferred) { + WARN_ONCE(1, "IEE: failed to defer PTP page free, leaking page %px\n", + object); + return; + } + + deferred->object = object; + deferred->order = cache->object_order; + deferred->num_pages = num_pages; + deferred->op = IEE_CACHE_DEFER_FREE; + INIT_WORK(&deferred->work, iee_cache_deferred_work); + queue_work(system_unbound_wq, &deferred->work); +} +#endif + +void iee_cache_free(struct iee_cache *cache, void *object) +{ + unsigned long tid; + void **freelist; + +#ifdef CONFIG_PTP_S + if (cache == &pg_cache) { + if (ptp_is_user_pgtable(object)) + cache = &pg_user_cache; + } +#endif + + if (!cache->init) { + free_pages((unsigned long)object, cache->object_order); + return; + } + +#ifdef CONFIG_X86_64 + if (unlikely((unsigned long)object < cache->reserve_start_addr + || (unsigned long)object >= cache->reserve_end_addr)) { + if (iee_cache_mark_pending_protect_free(object)) + return; + iee_cache_slow_free_pages(cache, object); + return; + } +#endif + +#ifdef CONFIG_ARM64 + if (unlikely((unsigned long)object < cache->reserve_start_addr + || (unsigned long)object >= cache->reserve_end_addr)) { + + set_iee_address_invalid((unsigned long)object, cache->object_order); + iee_set_bitmap_type((unsigned long)object, 1 << cache->object_order, IEE_NORMAL); + free_pages((unsigned long)object, cache->object_order); + return; + } +#endif + // fast path free +redo: + tid = READ_ONCE(cache->tid); + barrier(); + freelist = READ_ONCE(cache->freelist); +#ifdef CONFIG_PTP_S + if (cache == &pg_user_cache) + __set_freepointer(object, freelist); + else +#endif + __iee_set_freepointer(object, freelist); +#ifdef CONFIG_ARM64 + dsb(sy); +#endif + if (unlikely(!__update_freelist(cache, freelist, object, tid))) + goto redo; +} diff --git a/mm/huge_memory.c b/mm/huge_memory.c index d9e69d0439bd5..26282c45dade4 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2247,7 +2247,12 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); pgtable = pgtable_trans_huge_withdraw(mm, pmd); + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + WRITE_ONCE(*(&_pmd), __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) + | (PMD_TYPE_TABLE | PMD_TABLE_PXN))); + #else pmd_populate(mm, &_pmd, pgtable); + #endif pte = pte_offset_map(&_pmd, haddr); VM_BUG_ON(!pte); @@ -2421,7 +2426,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, * This's critical for some architectures (Power). */ pgtable = pgtable_trans_huge_withdraw(mm, pmd); + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + WRITE_ONCE(*(&_pmd), __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) + | (PMD_TYPE_TABLE | PMD_TABLE_PXN))); + #else pmd_populate(mm, &_pmd, pgtable); + #endif pte = pte_offset_map(&_pmd, haddr); VM_BUG_ON(!pte); diff --git a/mm/memory.c b/mm/memory.c index eaf07a4d1e0d5..d148ec2ee2647 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -90,6 +90,10 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + #include "pgalloc-track.h" #include "internal.h" #include "swap.h" @@ -5695,6 +5699,9 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, /* If the fault handler drops the mmap_lock, vma may be freed */ struct mm_struct *mm = vma->vm_mm; vm_fault_t ret; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif __set_current_state(TASK_RUNNING); @@ -5721,7 +5728,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, if (unlikely(is_vm_hugetlb_page(vma))) ret = hugetlb_fault(vma->vm_mm, vma, address, flags); else +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + { + ptp_disable_wp(&cr0); ret = __handle_mm_fault(vma, address, flags); + ptp_restore_wp(cr0); + } +#else + ret = __handle_mm_fault(vma, address, flags); +#endif lru_gen_exit_fault(); diff --git a/mm/mmap.c b/mm/mmap.c index bac026d841062..245a123c74238 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -54,6 +54,10 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + #define CREATE_TRACE_POINTS #include @@ -2343,10 +2347,17 @@ static void unmap_region(struct mm_struct *mm, struct ma_state *mas, update_hiwater_rss(mm); unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); mas_set(mas, mt_start); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + tlb_flush_mmu(&tlb); +#endif free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, next ? next->vm_start : USER_PGTABLES_CEILING, mm_wr_locked); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_finish_mmu(&tlb); +#else tlb_finish_mmu(&tlb); +#endif } /* @@ -2557,6 +2568,9 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, MA_STATE(mas_detach, &mt_detach, 0, 0); mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); mt_on_stack(mt_detach); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif /* * If we need to split any vma, do it now to save pain later. @@ -2668,8 +2682,15 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, * were isolated before we downgraded mmap_lock. */ mas_set(&mas_detach, 1); + +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif unmap_region(mm, &mas_detach, vma, prev, next, start, end, count, !unlock); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif /* Statistics and freeing VMAs */ mas_set(&mas_detach, 0); remove_mt(mm, &mas_detach); @@ -3358,6 +3379,9 @@ void exit_mmap(struct mm_struct *mm) unsigned long nr_accounted = 0; MA_STATE(mas, &mm->mm_mt, 0, 0); int count = 0; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif /* mm's last user has gone, and its about to be pulled down */ mmu_notifier_release(mm); @@ -3378,7 +3402,13 @@ void exit_mmap(struct mm_struct *mm) tlb_gather_mmu_fullmm(&tlb, mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif unmap_vmas(&tlb, &mas, vma, 0, ULONG_MAX, ULONG_MAX, false); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif mmap_read_unlock(mm); /* @@ -3389,9 +3419,16 @@ void exit_mmap(struct mm_struct *mm) mmap_write_lock(mm); mt_clear_in_rcu(&mm->mm_mt); mas_set(&mas, vma->vm_end); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + tlb_flush_mmu(&tlb); +#endif free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING, true); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_tlb_finish_mmu(&tlb); +#else tlb_finish_mmu(&tlb); +#endif /* * Walk the list again, actually closing and freeing it, with preemption diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 04469b481b93f..7d4ee1dbf829b 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -142,6 +142,51 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void __ptp_tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) +{ + struct encoded_page **pages = batch->encoded_pages; + unsigned int i, j, nr, nr_pages; + + while (batch->nr) { + if (!page_poisoning_enabled_static() && !want_init_on_free()) { + nr = min(MAX_NR_FOLIOS_PER_FREE, batch->nr); + + if (unlikely(encoded_page_flags(pages[nr - 1]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr++; + } else { + for (nr = 0, nr_pages = 0; + nr < batch->nr && nr_pages < MAX_NR_FOLIOS_PER_FREE; + nr++) { + if (unlikely(encoded_page_flags(pages[nr]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr_pages += encoded_nr_pages(pages[++nr]); + else + nr_pages++; + } + } + + for (i = 0; i < nr; i++) { + struct encoded_page *encoded = pages[i]; + struct page *page = encoded_page_ptr(encoded); + unsigned int page_count = 1; + + if (unlikely(encoded_page_flags(encoded) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + page_count = encoded_nr_pages(pages[++i]); + + for (j = 0; j < page_count; j++) + __ptp_tlb_remove_table(page + j); + } + pages += nr; + batch->nr -= nr; + + cond_resched(); + } +} +#endif + static void tlb_batch_pages_flush(struct mmu_gather *tlb) { struct mmu_gather_batch *batch; @@ -151,6 +196,17 @@ static void tlb_batch_pages_flush(struct mmu_gather *tlb) tlb->active = &tlb->local; } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void ptp_tlb_batch_pages_flush(struct mmu_gather *tlb) +{ + struct mmu_gather_batch *batch; + + for (batch = &tlb->local; batch && batch->nr; batch = batch->next) + __ptp_tlb_batch_free_encoded_pages(batch); + tlb->active = &tlb->local; +} +#endif + static void tlb_batch_list_free(struct mmu_gather *tlb) { struct mmu_gather_batch *batch, *next; @@ -348,6 +404,90 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) tlb_table_flush(tlb); } +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void __ptp_tlb_remove_table_free(struct mmu_table_batch *batch) +{ + int i; + + for (i = 0; i < batch->nr; i++) + __ptp_tlb_remove_table(batch->tables[i]); + + free_page((unsigned long)batch); +} + +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE +struct ptp_remove_table_work { + struct work_struct work; + struct mmu_table_batch *batch; +}; + +static void ptp_remove_table(struct work_struct *work) +{ + struct ptp_remove_table_work *ptp_remove_table_work = + container_of(work, struct ptp_remove_table_work, work); + + __ptp_tlb_remove_table_free(ptp_remove_table_work->batch); + kfree(ptp_remove_table_work); +} + +static void ptp_tlb_remove_table_rcu(struct rcu_head *head) +{ + struct ptp_remove_table_work *ptp_remove_table_work = + kmalloc(sizeof(struct ptp_remove_table_work), GFP_ATOMIC); + + ptp_remove_table_work->batch = container_of(head, struct mmu_table_batch, rcu); + INIT_WORK(&ptp_remove_table_work->work, ptp_remove_table); + schedule_work(&ptp_remove_table_work->work); +} + +static void ptp_tlb_remove_table_free(struct mmu_table_batch *batch) +{ + call_rcu(&batch->rcu, ptp_tlb_remove_table_rcu); +} +#else +static void ptp_tlb_remove_table_free(struct mmu_table_batch *batch) +{ + __ptp_tlb_remove_table_free(batch); +} +#endif + +static void ptp_tlb_remove_table_one(void *table) +{ + tlb_remove_table_sync_one(); + __ptp_tlb_remove_table(table); +} + +static void ptp_tlb_table_flush(struct mmu_gather *tlb) +{ + struct mmu_table_batch **batch = &tlb->batch; + + if (*batch) { + tlb_table_invalidate(tlb); + ptp_tlb_remove_table_free(*batch); + *batch = NULL; + } +} + +void ptp_tlb_remove_table(struct mmu_gather *tlb, void *table) +{ + struct mmu_table_batch **batch = &tlb->batch; + + if (*batch == NULL) { + *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); + if (*batch == NULL) { + tlb_table_invalidate(tlb); + ptp_tlb_remove_table_one(table); + return; + } + (*batch)->nr = 0; + } + + (*batch)->tables[(*batch)->nr++] = table; + if ((*batch)->nr == MAX_TABLE_BATCH) + ptp_tlb_table_flush(tlb); +} +#endif + static inline void tlb_table_init(struct mmu_gather *tlb) { tlb->batch = NULL; @@ -502,3 +642,57 @@ void tlb_finish_mmu(struct mmu_gather *tlb) #endif dec_tlb_flush_pending(tlb->mm); } + +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +static void ptp_tlb_flush_mmu_free(struct mmu_gather *tlb) +{ +#ifdef CONFIG_MMU_GATHER_TABLE_FREE + ptp_tlb_table_flush(tlb); +#endif +#ifndef CONFIG_MMU_GATHER_NO_GATHER + ptp_tlb_batch_pages_flush(tlb); +#endif +} + +void ptp_tlb_flush_mmu(struct mmu_gather *tlb) +{ + tlb_flush_mmu_tlbonly(tlb); + ptp_tlb_flush_mmu_free(tlb); +} + +void ptp_tlb_finish_mmu(struct mmu_gather *tlb) +{ + /* + * If there are parallel threads are doing PTE changes on same range + * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB + * flush by batching, one thread may end up seeing inconsistent PTEs + * and result in having stale TLB entries. So flush TLB forcefully + * if we detect parallel PTE batching threads. + * + * However, some syscalls, e.g. munmap(), may free page tables, this + * needs force flush everything in the given range. Otherwise this + * may result in having stale TLB entries for some architectures, + * e.g. aarch64, that could specify flush what level TLB. + */ + if (mm_tlb_flush_nested(tlb->mm)) { + /* + * The aarch64 yields better performance with fullmm by + * avoiding multiple CPUs spamming TLBI messages at the + * same time. + * + * On x86 non-fullmm doesn't yield significant difference + * against fullmm. + */ + tlb->fullmm = 1; + __tlb_reset_range(tlb); + tlb->freed_tables = 1; + } + + ptp_tlb_flush_mmu(tlb); + +#ifndef CONFIG_MMU_GATHER_NO_GATHER + tlb_batch_list_free(tlb); +#endif + dec_tlb_flush_pending(tlb->mm); +} +#endif diff --git a/mm/mprotect.c b/mm/mprotect.c index 76381f6f91521..59eca516a1ff5 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -37,6 +37,10 @@ #include #include +#ifdef CONFIG_PTP +#include +#endif + #include "internal.h" bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, @@ -584,6 +588,9 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + unsigned long cr0; +#endif if (newflags == oldflags) { *pprev = vma; @@ -644,7 +651,13 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; vma_set_page_prot(vma); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_disable_wp(&cr0); +#endif change_protection(tlb, vma, start, end, mm_cp_flags); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + ptp_restore_wp(cr0); +#endif /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index a78a4adf711ac..aa393f6c5780e 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -234,6 +234,22 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, } #endif +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) +struct ptp_pte_free_now_work { + struct work_struct work; + struct page *page; +}; + +static void ptp_pte_free_now(struct work_struct *work) +{ + struct ptp_pte_free_now_work *ptp_work = + container_of(work, struct ptp_pte_free_now_work, work); + + pte_free(NULL, ptp_work->page); + kfree(ptp_work); +} +#endif + /* arch define pte_free_defer in asm/pgalloc.h for its own implementation */ #ifndef pte_free_defer static void pte_free_now(struct rcu_head *head) @@ -241,7 +257,16 @@ static void pte_free_now(struct rcu_head *head) struct page *page; page = container_of(head, struct page, rcu_head); +#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) + struct ptp_pte_free_now_work *ptp_work = + kmalloc(sizeof(struct ptp_pte_free_now_work), GFP_ATOMIC); + + ptp_work->page = page; + INIT_WORK(&ptp_work->work, ptp_pte_free_now); + schedule_work(&ptp_work->work); +#else pte_free(NULL /* mm not passed and not used */, (pgtable_t)page); +#endif } void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) diff --git a/mm/slab.h b/mm/slab.h index 164ee349cdb29..81ac1cfc0d07e 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -28,6 +28,16 @@ typedef u64 freelist_full_t; #ifdef CONFIG_CREDP extern struct kmem_cache *cred_jar; #endif +#ifdef CONFIG_KEYP +extern struct kmem_cache *key_jar; +extern struct kmem_cache *key_union_jar; +extern struct kmem_cache *key_struct_jar; +extern struct kmem_cache *key_payload_jar; +#endif + +#ifdef CONFIG_IEE_SELINUX_P +extern struct kmem_cache *policy_jar; +#endif /* * Freelist pointer and counter to cmpxchg together, avoids the typical ABA @@ -725,9 +735,9 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, return s; } -#ifdef CONFIG_CREDP -#include -#endif + #if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) + #include + #endif static inline void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, gfp_t flags, size_t size, void **p, bool init, @@ -776,8 +786,16 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, p[i] = kasan_slab_alloc(s, p[i], flags, kasan_init); if (p[i] && init && (!kasan_init || !kasan_has_integrated_init())) { + #if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) + bool use_iee_memset = false; + #ifdef CONFIG_CREDP - if (haoc_enabled && s == cred_jar) + use_iee_memset |= haoc_enabled && s == cred_jar; + #endif + #ifdef CONFIG_KEYP + use_iee_memset |= haoc_enabled && s == key_jar; + #endif + if (use_iee_memset) iee_memset(p[i], 0, zero_size); else memset(p[i], 0, zero_size); diff --git a/mm/slab_common.c b/mm/slab_common.c index 2e2b43fae2c3f..91ed99c6a2057 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -145,11 +145,22 @@ int slab_unmergeable(struct kmem_cache *s) if (s->ctor) return 1; +#ifdef CONFIG_IEE_SELINUX_P + if (strcmp(s->name, "policy_jar") == 0) + return 1; +#endif + #ifdef CONFIG_HARDENED_USERCOPY if (s->usersize) return 1; #endif +#ifdef CONFIG_KEYP + if (strcmp(s->name, "key_jar") == 0 || strcmp(s->name, "key_union_jar") == 0 || + strcmp(s->name, "key_struct_jar") == 0 || strcmp(s->name, "key_payload_jar") == 0) + return 1; +#endif + /* * We may have set a slab to be unmergeable during bootstrap. */ @@ -169,6 +180,15 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, if (ctor) return NULL; +#ifdef CONFIG_KEYP + if (strcmp(name, "key_jar") == 0 || strcmp(name, "key_union_jar") == 0 || + strcmp(name, "key_struct_jar") == 0 || strcmp(name, "key_payload_jar") == 0) + return NULL; +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (strcmp(name, "policy_jar") == 0) + return NULL; +#endif size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); diff --git a/mm/slub.c b/mm/slub.c index 14465c4c0ffd8..de3d3a427a598 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -484,13 +484,27 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) #endif freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); - #ifdef CONFIG_CREDP +#ifdef CONFIG_CREDP if (haoc_enabled && s == cred_jar) { iee_set_freeptr((void **)freeptr_addr, (void *)(freelist_ptr_encode(s, fp, freeptr_addr).v)); return; } - #endif +#endif +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar) { + iee_set_freeptr((void **)freeptr_addr, + (void *)freelist_ptr_encode(s, fp, freeptr_addr).v); + return; + } +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && s == policy_jar) { + iee_set_freeptr((void **)freeptr_addr, + (void *)freelist_ptr_encode(s, fp, freeptr_addr).v); + return; + } +#endif *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); } @@ -2130,6 +2144,14 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; +#ifdef CONFIG_KEYP + if (s == key_jar) + alloc_gfp |= __GFP_ZERO; +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (s == policy_jar) + alloc_gfp |= __GFP_ZERO; +#endif slab = alloc_slab_page(alloc_gfp, node, oo); #ifdef CONFIG_IEE_PTRP @@ -2137,12 +2159,25 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab = iee_alloc_task_token_slab(s, slab, oo_order(oo)); #endif if (unlikely(!slab)) { +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar && + !IS_ENABLED(CONFIG_IEE_ALLOW_SPLIT_LM)) + return NULL; +#endif oo = s->min; alloc_gfp = flags; +#ifdef CONFIG_IEE_SELINUX_P + if (s == policy_jar) + alloc_gfp |= __GFP_ZERO; +#endif /* * Allocation may have failed due to fragmentation. * Try a lower order alloc if possible */ +#ifdef CONFIG_KEYP + if (s == key_jar) + alloc_gfp |= __GFP_ZERO; +#endif slab = alloc_slab_page(alloc_gfp, node, oo); #ifdef CONFIG_IEE_PTRP if(haoc_enabled) @@ -2156,7 +2191,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->objects = oo_objects(oo); slab->inuse = 0; slab->frozen = 0; - #ifdef CONFIG_IEE if(haoc_enabled) iee_allocate_slab_data(s, slab, oo_order(oo)); @@ -2168,10 +2202,38 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) * pages read-only here would trigger faults on those writes. * Skip enforcing RO for cred slabs when KASAN is enabled. */ - if (haoc_enabled && s == cred_jar && !IS_ENABLED(CONFIG_KASAN)) + if (haoc_enabled && s == cred_jar && !IS_ENABLED(CONFIG_KASAN)) { + #ifdef CONFIG_X86_64 + set_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << oo_order(oo), IEE_CRED); + #else + set_iee_page_type((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + oo_order(oo), IEE_CRED); + #endif + } +#endif +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar) { +#ifdef CONFIG_X86_64 + set_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << oo_order(oo), IEE_KEY); +#else + set_iee_page_type((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + oo_order(oo), IEE_KEY); +#endif + } +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && s == policy_jar) +#ifdef CONFIG_X86_64 + set_iee_pages((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << oo_order(oo), IEE_POLICY); +#else set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), - oo_order(oo)); + oo_order(oo), IEE_SELINUX); +#endif #endif + account_slab(slab, oo_order(oo), s, flags); slab->slab_cache = s; @@ -2237,11 +2299,36 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) iee_free_slab(s, slab, iee_free_cred_slab); return; #else - if (!IS_ENABLED(CONFIG_KASAN)) + if (!IS_ENABLED(CONFIG_KASAN)) { + iee_set_bitmap_type((unsigned long)page_address(folio_page(folio, 0)), + pages, IEE_NORMAL); unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); - #endif + } +#endif + } +#endif +#ifdef CONFIG_KEYP + if (haoc_enabled && s == key_jar) { +#ifdef CONFIG_X86_64 + unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), + 1 << order); +#else + unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); +#endif + } +#endif + +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && s == policy_jar) { +#ifdef CONFIG_X86_64 + unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + 1 << order); +#else + unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); +#endif } #endif + __free_pages(&folio->page, order); } @@ -4623,9 +4710,17 @@ static int calculate_sizes(struct kmem_cache *s) s->size = size; s->reciprocal_size = reciprocal_value(size); order = calculate_order(size); - #ifdef CONFIG_IEE +#ifdef CONFIG_IEE order = iee_calculate_order(s, order); - #endif +#endif +#ifdef CONFIG_KEYP + if (strcmp(s->name, "key_jar") == 0) + order = IEE_DATA_ORDER; +#endif +#ifdef CONFIG_IEE_SELINUX_P + if (haoc_enabled && strcmp(s->name, "policy_jar") == 0) + order = IEE_DATA_ORDER; +#endif if ((int)order < 0) return 0; diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index c3353cd442a5d..9ff858b7768d0 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -31,6 +31,10 @@ #include +#ifdef CONFIG_PTP +extern void *__ptp_vmemmap_alloc_block(unsigned long size, int node); +#endif + /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. @@ -167,6 +171,7 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, get_page(reuse); p = page_to_virt(reuse); } + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } @@ -175,11 +180,18 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) { + #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) + void *p = __ptp_vmemmap_alloc_block(size, node); + + if (!p) + return NULL; + #else void *p = vmemmap_alloc_block(size, node); if (!p) return NULL; memset(p, 0, size); + #endif return p; } diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 15bba470960d8..e3f815c1bbda3 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -385,7 +385,11 @@ static int get_secret(struct ceph_crypto_key *dst, const char *name, goto out; } +#ifdef CONFIG_KEYP + ckey = ((union key_payload *)(ukey->name_link.next))->data[0]; +#else ckey = ukey->payload.data[0]; +#endif err = ceph_crypto_key_clone(dst, ckey); if (err) goto out_key; diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 3397d105f74f9..353d241439e06 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -338,7 +338,11 @@ static void ceph_key_free_preparse(struct key_preparsed_payload *prep) static void ceph_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + struct ceph_crypto_key *ckey = ((union key_payload *)(key->name_link.next))->data[0]; +#else struct ceph_crypto_key *ckey = key->payload.data[0]; +#endif ceph_crypto_key_destroy(ckey); kfree(ckey); diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c index a22c036b6050d..256c31fbc5abf 100644 --- a/net/dns_resolver/dns_key.c +++ b/net/dns_resolver/dns_key.c @@ -35,6 +35,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" MODULE_DESCRIPTION("DNS Resolver"); @@ -298,7 +301,12 @@ static void dns_resolver_describe(const struct key *key, struct seq_file *m) { seq_puts(m, key->description); if (key_is_positive(key)) { +#ifdef CONFIG_KEYP + int err = PTR_ERR(((union key_payload *)(key->name_link.next))-> + data[dns_key_error]); +#else int err = PTR_ERR(key->payload.data[dns_key_error]); +#endif if (err) seq_printf(m, ": %d", err); @@ -314,7 +322,11 @@ static void dns_resolver_describe(const struct key *key, struct seq_file *m) static long dns_resolver_read(const struct key *key, char *buffer, size_t buflen) { +#ifdef CONFIG_KEYP + int err = PTR_ERR(((union key_payload *)(key->name_link.next))->data[dns_key_error]); +#else int err = PTR_ERR(key->payload.data[dns_key_error]); +#endif if (err) return err; @@ -367,14 +379,18 @@ static int __init init_dns_resolver(void) /* instruct request_key() to use this special keyring as a cache for * the results it looks up */ +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); +#else set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); - #ifdef CONFIG_CREDP +#endif +#ifdef CONFIG_CREDP iee_set_cred_thread_keyring(cred, keyring); iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); - #else +#else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif dns_resolver_cache = cred; kdebug("DNS resolver keyring: %d\n", key_serial(keyring)); diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 1a87cbd662c5a..676c0c14d45aa 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -307,7 +307,11 @@ struct rxrpc_call *rxrpc_kernel_begin_call(struct socket *sock, if (!key) key = rx->key; +#ifdef CONFIG_KEYP + if (key && !((union key_payload *)key->name_link.next)->data[0]) +#else if (key && !key->payload.data[0]) +#endif key = NULL; /* a no-security key */ memset(&p, 0, sizeof(p)); diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index 25fbef44ca9d6..749215fa60556 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -279,8 +279,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, if (ret < 0) return ret; +#ifdef CONFIG_KEYP + ret = conn->security->init_connection_security(conn, + ((union key_payload *)(conn->key->name_link.next))->data[0]); +#else ret = conn->security->init_connection_security( conn, conn->key->payload.data[0]); +#endif if (ret < 0) return ret; diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c index 73659197edffa..b5cf7d82a5b5e 100644 --- a/net/rxrpc/key.c +++ b/net/rxrpc/key.c @@ -418,7 +418,11 @@ static void rxrpc_free_preparse(struct key_preparsed_payload *prep) */ static void rxrpc_destroy(struct key *key) { +#ifdef CONFIG_KEYP + rxrpc_free_token_list(((union key_payload *)(key->name_link.next))->data[0]); +#else rxrpc_free_token_list(key->payload.data[0]); +#endif } /* @@ -431,7 +435,12 @@ static void rxrpc_describe(const struct key *key, struct seq_file *m) seq_puts(m, key->description); +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif seq_puts(m, sep); switch (token->security_index) { @@ -589,7 +598,12 @@ static long rxrpc_read(const struct key *key, size += 1 * 4; /* token count */ ntoks = 0; +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif toksize = 4; /* sec index */ switch (token->security_index) { @@ -659,7 +673,12 @@ static long rxrpc_read(const struct key *key, ENCODE(ntoks); tok = 0; +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif toksize = toksizes[tok++]; ENCODE(toksize); oldxdr = xdr; diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index 7b12fd90730d1..f0d0dca3fca0e 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -19,6 +19,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "ar-internal.h" #define RXKAD_VERSION 2 @@ -88,10 +91,19 @@ static void rxkad_free_preparse_server_key(struct key_preparsed_payload *prep) static void rxkad_destroy_server_key(struct key *key) { +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + if (key_payload->data[0]) { + crypto_free_skcipher(key_payload->data[0]); + key_payload->data[0] = NULL; + } +#else if (key->payload.data[0]) { crypto_free_skcipher(key->payload.data[0]); key->payload.data[0] = NULL; } +#endif } /* @@ -206,7 +218,11 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, return -ENOMEM; } +#ifdef CONFIG_KEYP + token = ((union key_payload *)(conn->key->name_link.next))->data[0]; +#else token = conn->key->payload.data[0]; +#endif memcpy(&iv, token->kad->session_key, sizeof(iv)); tmpbuf[0] = htonl(conn->proto.epoch); @@ -319,7 +335,11 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, } /* encrypt from the session key */ +#ifdef CONFIG_KEYP + token = ((union key_payload *)(call->conn->key->name_link.next))->data[0]; +#else token = call->conn->key->payload.data[0]; +#endif memcpy(&iv, token->kad->session_key, sizeof(iv)); sg_init_one(&sg, txb->data, txb->len); @@ -498,7 +518,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, sg_init_one(sg, data, len); /* decrypt from the session key */ +#ifdef CONFIG_KEYP + token = ((union key_payload *)(call->conn->key->name_link.next))->data[0]; +#else token = call->conn->key->payload.data[0]; +#endif memcpy(&iv, token->kad->session_key, sizeof(iv)); skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); @@ -820,7 +844,11 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, return rxrpc_abort_conn(conn, skb, RXKADLEVELFAIL, -EACCES, rxkad_abort_chall_level); +#ifdef CONFIG_KEYP + token = ((union key_payload *)(conn->key->name_link.next))->data[0]; +#else token = conn->key->payload.data[0]; +#endif /* build the response packet */ resp = kzalloc(sizeof(struct rxkad_response), GFP_NOFS); @@ -873,11 +901,25 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, *_expiry = 0; +#ifdef CONFIG_KEYP + ASSERT(((union key_payload *)(server_key->name_link.next))->data[0]); +#else ASSERT(server_key->payload.data[0] != NULL); +#endif + ASSERTCMP((unsigned long) ticket & 7UL, ==, 0); +#ifdef CONFIG_KEYP + memcpy(&iv, &((union key_payload *)(server_key->name_link.next))->data[2], sizeof(iv)); +#else memcpy(&iv, &server_key->payload.data[2], sizeof(iv)); +#endif +#ifdef CONFIG_KEYP + req = skcipher_request_alloc(((union key_payload *)(server_key->name_link.next))-> + data[0], GFP_NOFS); +#else req = skcipher_request_alloc(server_key->payload.data[0], GFP_NOFS); +#endif if (!req) return -ENOMEM; diff --git a/net/rxrpc/security.c b/net/rxrpc/security.c index 9784adc8f2759..e1be443dcf1e9 100644 --- a/net/rxrpc/security.c +++ b/net/rxrpc/security.c @@ -79,7 +79,12 @@ int rxrpc_init_client_call_security(struct rxrpc_call *call) if (ret < 0) return ret; +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif sec = rxrpc_security_lookup(token->security_index); if (sec) goto found; @@ -103,7 +108,12 @@ int rxrpc_init_client_conn_security(struct rxrpc_connection *conn) _enter("{%d},{%x}", conn->debug_id, key_serial(key)); +#ifdef CONFIG_KEYP + for (token = ((union key_payload *)(key->name_link.next))->data[0]; + token; token = token->next) { +#else for (token = key->payload.data[0]; token; token = token->next) { +#endif if (token->security_index == conn->security->security_index) goto found; } diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index 3dc54d6d4e316..c4f8886b15279 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -587,7 +587,11 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, } key = rx->key; +#ifdef CONFIG_KEYP + if (key && !((union key_payload *)rx->key->name_link.next)->data[0]) +#else if (key && !rx->key->payload.data[0]) +#endif key = NULL; memset(&cp, 0, sizeof(cp)); diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c index 9d95a6055b707..c1b0ccb921533 100644 --- a/net/rxrpc/server_key.c +++ b/net/rxrpc/server_key.c @@ -100,7 +100,11 @@ static void rxrpc_free_preparse_s(struct key_preparsed_payload *prep) static void rxrpc_destroy_s(struct key *key) { +#ifdef CONFIG_KEYP + const struct rxrpc_security *sec = ((union key_payload *)(key->name_link.next))->data[1]; +#else const struct rxrpc_security *sec = key->payload.data[1]; +#endif if (sec && sec->destroy_server_key) sec->destroy_server_key(key); @@ -108,7 +112,11 @@ static void rxrpc_destroy_s(struct key *key) static void rxrpc_describe_s(const struct key *key, struct seq_file *m) { +#ifdef CONFIG_KEYP + const struct rxrpc_security *sec = ((union key_payload *)(key->name_link.next))->data[1]; +#else const struct rxrpc_security *sec = key->payload.data[1]; +#endif seq_puts(m, key->description); if (sec && sec->describe_server_key) diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index b1ffd4cc0b448..4824249f21072 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -422,14 +422,23 @@ int evm_init_key(void) if (IS_ERR(evm_key)) return -ENOENT; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(evm_key)); + ekp = ((union key_payload *)(evm_key->name_link.next))->data[0]; +#else down_read(&evm_key->sem); ekp = evm_key->payload.data[0]; +#endif rc = evm_set_key(ekp->decrypted_data, ekp->decrypted_datalen); /* burn the original key contents */ memset(ekp->decrypted_data, 0, ekp->decrypted_datalen); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(evm_key)); +#else up_read(&evm_key->sem); +#endif key_put(evm_key); return rc; } diff --git a/security/keys/big_key.c b/security/keys/big_key.c index c3367622c683b..4a78d6171e012 100644 --- a/security/keys/big_key.c +++ b/security/keys/big_key.c @@ -164,7 +164,12 @@ void big_key_free_preparse(struct key_preparsed_payload *prep) */ void big_key_revoke(struct key *key) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif /* clear the quota */ key_payload_reserve(key, 0); @@ -177,7 +182,12 @@ void big_key_revoke(struct key *key) */ void big_key_destroy(struct key *key) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif if (payload->length > BIG_KEY_FILE_THRESHOLD) { path_put(&payload->path); @@ -210,7 +220,12 @@ int big_key_update(struct key *key, struct key_preparsed_payload *prep) */ void big_key_describe(const struct key *key, struct seq_file *m) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif seq_puts(m, key->description); @@ -226,7 +241,12 @@ void big_key_describe(const struct key *key, struct seq_file *m) */ long big_key_read(const struct key *key, char *buffer, size_t buflen) { +#ifdef CONFIG_KEYP + struct big_key_payload *payload = + to_big_key_payload(*((union key_payload *)(key->name_link.next))); +#else struct big_key_payload *payload = to_big_key_payload(key->payload); +#endif size_t datalen = payload->length; long ret; diff --git a/security/keys/dh.c b/security/keys/dh.c index da64c358474b2..8386a072903de 100644 --- a/security/keys/dh.c +++ b/security/keys/dh.c @@ -32,7 +32,11 @@ static ssize_t dh_data_from_key(key_serial_t keyid, const void **data) ret = -EOPNOTSUPP; if (key->type == &key_type_user) { +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif status = key_validate(key); if (status == 0) { const struct user_key_payload *payload; @@ -49,7 +53,11 @@ static ssize_t dh_data_from_key(key_serial_t keyid, const void **data) ret = -ENOMEM; } } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif } key_put(key); diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 1e313982af02a..354a9febbb6ea 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -314,11 +314,19 @@ static struct key *request_user_key(const char *master_desc, const u8 **master_k if (IS_ERR(ukey)) goto error; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(ukey)); +#else down_read(&ukey->sem); +#endif upayload = user_key_payload_locked(ukey); if (!upayload) { /* key was revoked before we acquired its semaphore */ +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(ukey)); +#else up_read(&ukey->sem); +#endif key_put(ukey); ukey = ERR_PTR(-EKEYREVOKED); goto error; @@ -729,7 +737,11 @@ static int encrypted_key_decrypt(struct encrypted_key_payload *epayload, if (ret < 0) pr_err("encrypted_key: failed to decrypt key (%d)\n", ret); out: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(mkey)); +#else up_read(&mkey->sem); +#endif key_put(mkey); memzero_explicit(derived_key, sizeof(derived_key)); return ret; @@ -874,7 +886,12 @@ static void encrypted_rcu_free(struct rcu_head *rcu) */ static int encrypted_update(struct key *key, struct key_preparsed_payload *prep) { +#ifdef CONFIG_KEYP + struct encrypted_key_payload *epayload = + ((union key_payload *)(key->name_link.next))->data[0]; +#else struct encrypted_key_payload *epayload = key->payload.data[0]; +#endif struct encrypted_key_payload *new_epayload; char *buf; char *new_master_desc = NULL; @@ -974,7 +991,11 @@ static long encrypted_read(const struct key *key, char *buffer, goto out; } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(mkey)); +#else up_read(&mkey->sem); +#endif key_put(mkey); memzero_explicit(derived_key, sizeof(derived_key)); @@ -983,7 +1004,11 @@ static long encrypted_read(const struct key *key, char *buffer, return asciiblob_len; out: +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(mkey)); +#else up_read(&mkey->sem); +#endif key_put(mkey); memzero_explicit(derived_key, sizeof(derived_key)); return ret; @@ -994,7 +1019,11 @@ static long encrypted_read(const struct key *key, char *buffer, */ static void encrypted_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree_sensitive(key->payload.data[0]); +#endif } struct key_type key_type_encrypted = { diff --git a/security/keys/encrypted-keys/masterkey_trusted.c b/security/keys/encrypted-keys/masterkey_trusted.c index e6d22ce77e98f..d820a9f205c7e 100644 --- a/security/keys/encrypted-keys/masterkey_trusted.c +++ b/security/keys/encrypted-keys/masterkey_trusted.c @@ -34,8 +34,13 @@ struct key *request_trusted_key(const char *trusted_desc, if (IS_ERR(tkey)) goto error; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(tkey)); + tpayload = ((union key_payload *)(tkey->name_link.next))->data[0]; +#else down_read(&tkey->sem); tpayload = tkey->payload.data[0]; +#endif *master_key = tpayload->key; *master_keylen = tpayload->key_len; error: diff --git a/security/keys/gc.c b/security/keys/gc.c index eaddaceda14ea..a312e0c4b68ae 100644 --- a/security/keys/gc.c +++ b/security/keys/gc.c @@ -8,6 +8,10 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#include +#endif #include "internal.h" /* @@ -135,18 +139,31 @@ void key_gc_keytype(struct key_type *ktype) static noinline void key_gc_unused_keys(struct list_head *keys) { while (!list_empty(keys)) { +#ifdef CONFIG_KEYP + struct key *key = + list_entry(keys->next, struct key_union, graveyard_link)->key; +#else struct key *key = list_entry(keys->next, struct key, graveyard_link); +#endif short state = key->state; +#ifdef CONFIG_KEYP + list_del(&(((struct key_union *)(key->graveyard_link.next))->graveyard_link)); +#else list_del(&key->graveyard_link); +#endif kdebug("- %u", key->serial); key_check(key); #ifdef CONFIG_KEY_NOTIFICATIONS remove_watch_list(key->watchers, key->serial); +#ifdef CONFIG_KEYP + iee_set_key_watchers(key, NULL); +#else key->watchers = NULL; +#endif #endif /* Throw away the key data if the key is instantiated */ @@ -171,7 +188,15 @@ static noinline void key_gc_unused_keys(struct list_head *keys) key_put_tag(key->domain_tag); kfree(key->description); +#ifdef CONFIG_KEYP + kmem_cache_free(key_union_jar, (struct key_union *)(key->graveyard_link.next)); + kmem_cache_free(key_struct_jar, (struct key_struct *)(key->name_link.prev)); + kmem_cache_free(key_payload_jar, (union key_payload *)(key->name_link.next)); + iee_memset(key, 0, sizeof(*key)); + barrier_data(key); +#else memzero_explicit(key, sizeof(*key)); +#endif kmem_cache_free(key_jar, key); } } @@ -223,7 +248,11 @@ static void key_garbage_collector(struct work_struct *work) continue_scanning: while (cursor) { +#ifdef CONFIG_KEYP + key = rb_entry(cursor, struct key_union, serial_node)->key; +#else key = rb_entry(cursor, struct key, serial_node); +#endif cursor = rb_next(cursor); if (refcount_read(&key->usage) == 0) @@ -232,8 +261,13 @@ static void key_garbage_collector(struct work_struct *work) if (unlikely(gc_state & KEY_GC_REAPING_DEAD_1)) { if (key->type == key_gc_dead_keytype) { gc_state |= KEY_GC_FOUND_DEAD_KEY; +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(key, KEY_FLAG_DEAD, SET_BIT_OP); + iee_set_key_perm(key, 0); +#else set_bit(KEY_FLAG_DEAD, &key->flags); key->perm = 0; +#endif goto skip_dead_key; } else if (key->type == &key_type_keyring && key->restrict_link) { @@ -339,10 +373,20 @@ static void key_garbage_collector(struct work_struct *work) */ found_unreferenced_key: kdebug("unrefd key %d", key->serial); +#ifdef CONFIG_KEYP + rb_erase(&(((struct key_union *)(key->graveyard_link.next))->serial_node), + &key_serial_tree); +#else rb_erase(&key->serial_node, &key_serial_tree); +#endif spin_unlock(&key_serial_lock); +#ifdef CONFIG_KEYP + list_add_tail(&(((struct key_union *)(key->graveyard_link.next))->graveyard_link), + &graveyard); +#else list_add_tail(&key->graveyard_link, &graveyard); +#endif gc_state |= KEY_GC_REAP_AGAIN; goto maybe_resched; @@ -370,11 +414,21 @@ static void key_garbage_collector(struct work_struct *work) destroy_dead_key: spin_unlock(&key_serial_lock); kdebug("destroy key %d", key->serial); +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); + iee_set_key_type(key, &key_type_dead); +#else down_write(&key->sem); key->type = &key_type_dead; +#endif if (key_gc_dead_keytype->destroy) key_gc_dead_keytype->destroy(key); +#ifdef CONFIG_KEYP + iee_memset((key->name_link.next), KEY_DESTROY, sizeof(key->payload)); + up_write(&KEY_SEM(key)); +#else memset(&key->payload, KEY_DESTROY, sizeof(key->payload)); up_write(&key->sem); +#endif goto maybe_resched; } diff --git a/security/keys/internal.h b/security/keys/internal.h index b7b622bc36a13..fd3c583c3c39a 100644 --- a/security/keys/internal.h +++ b/security/keys/internal.h @@ -83,12 +83,20 @@ extern unsigned key_quota_maxbytes; extern struct kmem_cache *key_jar; +#ifdef CONFIG_KEYP +extern struct kmem_cache *key_union_jar; +extern struct kmem_cache *key_struct_jar; +extern struct kmem_cache *key_payload_jar; +#endif extern struct rb_root key_serial_tree; extern spinlock_t key_serial_lock; extern struct mutex key_construction_mutex; extern wait_queue_head_t request_key_conswq; extern void key_set_index_key(struct keyring_index_key *index_key); +#ifdef CONFIG_KEYP +extern void iee_key_set_index_key(struct keyring_index_key *index_key); +#endif extern struct key_type *key_type_lookup(const char *type); extern void key_type_put(struct key_type *ktype); diff --git a/security/keys/key.c b/security/keys/key.c index 35db23d05302e..0aab7c37636ea 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -15,9 +15,19 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#include +#include +#endif #include "internal.h" struct kmem_cache *key_jar; +#ifdef CONFIG_KEYP +struct kmem_cache *key_union_jar; +struct kmem_cache *key_struct_jar; +struct kmem_cache *key_payload_jar; +#endif struct rb_root key_serial_tree; /* tree of keys indexed by serial */ DEFINE_SPINLOCK(key_serial_lock); @@ -139,9 +149,17 @@ static inline void key_alloc_serial(struct key *key) /* propose a random serial number and look for a hole for it in the * serial number tree */ do { +#ifdef CONFIG_KEYP + key_serial_t tmp; + + get_random_bytes(&tmp, sizeof(key->serial)); + + iee_set_key_serial(key, tmp >> 1); +#else get_random_bytes(&key->serial, sizeof(key->serial)); key->serial >>= 1; /* negative numbers are not permitted */ +#endif } while (key->serial < 3); spin_lock(&key_serial_lock); @@ -152,7 +170,11 @@ static inline void key_alloc_serial(struct key *key) while (*p) { parent = *p; +#ifdef CONFIG_KEYP + xkey = rb_entry(parent, struct key_union, serial_node)->key; +#else xkey = rb_entry(parent, struct key, serial_node); +#endif if (key->serial < xkey->serial) p = &(*p)->rb_left; @@ -163,8 +185,14 @@ static inline void key_alloc_serial(struct key *key) } /* we've found a suitable hole - arrange for this key to occupy it */ +#ifdef CONFIG_KEYP + rb_link_node(&(((struct key_union *)(key->graveyard_link.next))->serial_node), parent, p); + rb_insert_color(&(((struct key_union *)(key->graveyard_link.next))->serial_node), + &key_serial_tree); +#else rb_link_node(&key->serial_node, parent, p); rb_insert_color(&key->serial_node, &key_serial_tree); +#endif spin_unlock(&key_serial_lock); return; @@ -173,9 +201,19 @@ static inline void key_alloc_serial(struct key *key) * that point looking for the next unused serial number */ serial_exists: for (;;) { +#ifdef CONFIG_KEYP + key_serial_t tmp = key->serial + 1; + + iee_set_key_serial(key, tmp); +#else key->serial++; +#endif if (key->serial < 3) { +#ifdef CONFIG_KEYP + iee_set_key_serial(key, 3); +#else key->serial = 3; +#endif goto attempt_insertion; } @@ -183,7 +221,11 @@ static inline void key_alloc_serial(struct key *key) if (!parent) goto attempt_insertion; +#ifdef CONFIG_KEYP + xkey = rb_entry(parent, struct key_union, serial_node)->key; +#else xkey = rb_entry(parent, struct key, serial_node); +#endif if (key->serial < xkey->serial) goto attempt_insertion; } @@ -231,6 +273,9 @@ struct key *key_alloc(struct key_type *type, const char *desc, struct key *key; size_t desclen, quotalen; int ret; +#ifdef CONFIG_KEYP + unsigned long kflags; +#endif key = ERR_PTR(-EINVAL); if (!desc || !*desc) @@ -274,17 +319,80 @@ struct key *key_alloc(struct key_type *type, const char *desc, } /* allocate and initialise the key and its description */ +#ifdef CONFIG_KEYP + key = kmem_cache_alloc(key_jar, GFP_KERNEL); +#else key = kmem_cache_zalloc(key_jar, GFP_KERNEL); +#endif if (!key) goto no_memory_2; +#ifdef CONFIG_KEYP + struct key_union *key_union = kmem_cache_zalloc(key_union_jar, GFP_KERNEL); + struct key_struct *key_struct = kmem_cache_zalloc(key_struct_jar, GFP_KERNEL); + union key_payload *key_payload = kmem_cache_alloc(key_payload_jar, GFP_KERNEL); + + if (!key_union || !key_struct || !key_payload) + goto no_memory_3; + + key_union->key = key; + key_struct->key = key; + iee_set_key_union(key, key_union); + iee_set_key_struct(key, key_struct); + iee_set_key_payload(key, key_payload); +#endif + +#ifdef CONFIG_KEYP + struct keyring_index_key tmp = key->index_key; + + tmp.desc_len = desclen; + tmp.description = kmemdup(desc, desclen + 1, GFP_KERNEL); + iee_set_key_index_key(key, &tmp); +#else key->index_key.desc_len = desclen; key->index_key.description = kmemdup(desc, desclen + 1, GFP_KERNEL); +#endif if (!key->index_key.description) goto no_memory_3; +#ifdef CONFIG_KEYP + tmp = key->index_key; + tmp.type = type; + iee_set_key_index_key(key, &tmp); + iee_key_set_index_key(&key->index_key); +#else key->index_key.type = type; key_set_index_key(&key->index_key); +#endif +#ifdef CONFIG_KEYP + iee_set_key_usage(key, 1, REFCOUNT_SET); + init_rwsem(&KEY_SEM(key)); + lockdep_set_class(&KEY_SEM(key), &type->lock_class); + iee_set_key_user(key, user); + iee_set_key_quotalen(key, quotalen); + iee_set_key_datalen(key, type->def_datalen); + iee_set_key_uid(key, uid); + iee_set_key_gid(key, gid); + iee_set_key_perm(key, perm); + iee_set_key_expiry(key, TIME64_MAX); + iee_set_key_restrict_link(key, restrict_link); + iee_set_key_last_used_at(key, ktime_get_real_seconds()); + + kflags = key->flags; + if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) + kflags |= 1 << KEY_FLAG_IN_QUOTA; + if (flags & KEY_ALLOC_BUILT_IN) + kflags |= 1 << KEY_FLAG_BUILTIN; + if (flags & KEY_ALLOC_UID_KEYRING) + kflags |= 1 << KEY_FLAG_UID_KEYRING; + if (flags & KEY_ALLOC_SET_KEEP) + kflags |= 1 << KEY_FLAG_KEEP; + iee_set_key_flags(key, kflags); + +#ifdef KEY_DEBUGGING + iee_set_key_magic(key, KEY_DEBUG_MAGIC); +#endif +#else refcount_set(&key->usage, 1); init_rwsem(&key->sem); lockdep_set_class(&key->sem, &type->lock_class); @@ -309,6 +417,7 @@ struct key *key_alloc(struct key_type *type, const char *desc, #ifdef KEY_DEBUGGING key->magic = KEY_DEBUG_MAGIC; +#endif #endif /* let the security module know about the key */ @@ -326,6 +435,15 @@ struct key *key_alloc(struct key_type *type, const char *desc, security_error: kfree(key->description); +#ifdef CONFIG_KEYP + if (key_union) + kmem_cache_free(key_union_jar, key_union); + if (key_struct) + kmem_cache_free(key_struct_jar, key_struct); + if (key_payload) + kmem_cache_free(key_payload_jar, key_payload); + iee_memset(key, 0, sizeof(struct key)); +#endif kmem_cache_free(key_jar, key); if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) { spin_lock(&user->lock); @@ -338,6 +456,15 @@ struct key *key_alloc(struct key_type *type, const char *desc, goto error; no_memory_3: +#ifdef CONFIG_KEYP + if (key_union) + kmem_cache_free(key_union_jar, key_union); + if (key_struct) + kmem_cache_free(key_struct_jar, key_struct); + if (key_payload) + kmem_cache_free(key_payload_jar, key_payload); + iee_memset(key, 0, sizeof(struct key)); +#endif kmem_cache_free(key_jar, key); no_memory_2: if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) { @@ -391,14 +518,22 @@ int key_payload_reserve(struct key *key, size_t datalen) } else { key->user->qnbytes += delta; +#ifdef CONFIG_KEYP + iee_set_key_quotalen(key, key->quotalen + delta); +#else key->quotalen += delta; +#endif } spin_unlock(&key->user->lock); } /* change the recorded data length if that didn't generate an error */ if (ret == 0) +#ifdef CONFIG_KEYP + iee_set_key_datalen(key, datalen); +#else key->datalen = datalen; +#endif return ret; } @@ -412,8 +547,14 @@ static void mark_key_instantiated(struct key *key, int reject_error) /* Commit the payload before setting the state; barrier versus * key_read_state(). */ +#ifdef CONFIG_KEYP + compiletime_assert_atomic_type(key->state); + barrier(); + iee_set_key_state(key, (reject_error < 0) ? reject_error : KEY_IS_POSITIVE); +#else smp_store_release(&key->state, (reject_error < 0) ? reject_error : KEY_IS_POSITIVE); +#endif } /* @@ -449,13 +590,22 @@ static int __key_instantiate_and_link(struct key *key, mark_key_instantiated(key, 0); notify_key(key, NOTIFY_KEY_INSTANTIATED, 0); +#ifdef CONFIG_KEYP + if (iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, TEST_AND_CLEAR_BIT)) + awaken = 1; +#else if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) awaken = 1; +#endif /* and link it into the destination keyring */ if (keyring) { if (test_bit(KEY_FLAG_KEEP, &keyring->flags)) +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(key, KEY_FLAG_KEEP, SET_BIT_OP); +#else set_bit(KEY_FLAG_KEEP, &key->flags); +#endif __key_link(keyring, key, _edit); } @@ -465,7 +615,11 @@ static int __key_instantiate_and_link(struct key *key, key_invalidate(authkey); if (prep->expiry != TIME64_MAX) +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, prep->expiry); +#else key_set_expiry(key, prep->expiry); +#endif } } @@ -605,10 +759,19 @@ int key_reject_and_link(struct key *key, atomic_inc(&key->user->nikeys); mark_key_instantiated(key, -error); notify_key(key, NOTIFY_KEY_INSTANTIATED, -error); +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, ktime_get_real_seconds() + timeout); +#else key_set_expiry(key, ktime_get_real_seconds() + timeout); +#endif +#ifdef CONFIG_KEYP + if (iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, TEST_AND_CLEAR_BIT)) + awaken = 1; +#else if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) awaken = 1; +#endif ret = 0; @@ -647,8 +810,13 @@ void key_put(struct key *key) if (key) { key_check(key); +#ifdef CONFIG_KEYP + if (iee_set_key_usage(key, 0, REFCOUNT_DEC_AND_TEST)) + schedule_work(&key_gc_work); +#else if (refcount_dec_and_test(&key->usage)) schedule_work(&key_gc_work); +#endif } } EXPORT_SYMBOL(key_put); @@ -666,7 +834,11 @@ struct key *key_lookup(key_serial_t id) /* search the tree for the specified key */ n = key_serial_tree.rb_node; while (n) { +#ifdef CONFIG_KEYP + key = rb_entry(n, struct key_union, serial_node)->key; +#else key = rb_entry(n, struct key, serial_node); +#endif if (id < key->serial) n = n->rb_left; @@ -684,8 +856,13 @@ struct key *key_lookup(key_serial_t id) /* A key is allowed to be looked up only if someone still owns a * reference to it - otherwise it's awaiting the gc. */ +#ifdef CONFIG_KEYP + if (!iee_set_key_usage(key, 0, REFCOUNT_INC_NOT_ZERO)) + goto not_found; +#else if (!refcount_inc_not_zero(&key->usage)) goto not_found; +#endif error: spin_unlock(&key_serial_lock); @@ -723,13 +900,25 @@ void key_set_timeout(struct key *key, unsigned timeout) time64_t expiry = TIME64_MAX; /* make the changes with the locks held to prevent races */ +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif if (timeout > 0) expiry = ktime_get_real_seconds() + timeout; +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, expiry); +#else key_set_expiry(key, expiry); +#endif +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif } EXPORT_SYMBOL_GPL(key_set_timeout); @@ -762,7 +951,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, if (!key->type->update) goto error; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif ret = key->type->update(key, prep); if (ret == 0) { @@ -771,7 +964,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, notify_key(key, NOTIFY_KEY_UPDATED, 0); } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif if (ret < 0) goto error; @@ -1087,7 +1284,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) goto error; } +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif ret = key->type->update(key, &prep); if (ret == 0) { @@ -1096,7 +1297,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) notify_key(key, NOTIFY_KEY_UPDATED, 0); } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif error: if (key->type->preparse) @@ -1125,6 +1330,23 @@ void key_revoke(struct key *key) * authorisation key whilst holding the sem on a key we've just * instantiated */ +#ifdef CONFIG_KEYP + down_write_nested(&KEY_SEM(key), 1); + if (!iee_set_key_flag_bit(key, KEY_FLAG_REVOKED, TEST_AND_SET_BIT)) { + notify_key(key, NOTIFY_KEY_REVOKED, 0); + if (key->type->revoke) + key->type->revoke(key); + + /* set the death time to no more than the expiry time */ + time = ktime_get_real_seconds(); + if (key->revoked_at == 0 || key->revoked_at > time) { + iee_set_key_revoked_at(key, time); + key_schedule_gc(key->revoked_at + key_gc_delay); + } + } + + up_write(&KEY_SEM(key)); +#else down_write_nested(&key->sem, 1); if (!test_and_set_bit(KEY_FLAG_REVOKED, &key->flags)) { notify_key(key, NOTIFY_KEY_REVOKED, 0); @@ -1140,6 +1362,7 @@ void key_revoke(struct key *key) } up_write(&key->sem); +#endif } EXPORT_SYMBOL(key_revoke); @@ -1157,12 +1380,21 @@ void key_invalidate(struct key *key) key_check(key); if (!test_bit(KEY_FLAG_INVALIDATED, &key->flags)) { +#ifdef CONFIG_KEYP + down_write_nested(&KEY_SEM(key), 1); + if (!iee_set_key_flag_bit(key, KEY_FLAG_INVALIDATED, TEST_AND_SET_BIT)) { + notify_key(key, NOTIFY_KEY_INVALIDATED, 0); + key_schedule_gc_links(); + } + up_write(&KEY_SEM(key)); +#else down_write_nested(&key->sem, 1); if (!test_and_set_bit(KEY_FLAG_INVALIDATED, &key->flags)) { notify_key(key, NOTIFY_KEY_INVALIDATED, 0); key_schedule_gc_links(); } up_write(&key->sem); +#endif } } EXPORT_SYMBOL(key_invalidate); @@ -1186,9 +1418,17 @@ int generic_key_instantiate(struct key *key, struct key_preparsed_payload *prep) ret = key_payload_reserve(key, prep->quotalen); if (ret == 0) { rcu_assign_keypointer(key, prep->payload.data[0]); +#ifdef CONFIG_KEYP + union key_payload *key_payload = (union key_payload *)(key->name_link.next); + + key_payload->data[1] = prep->payload.data[1]; + key_payload->data[2] = prep->payload.data[2]; + key_payload->data[3] = prep->payload.data[3]; +#else key->payload.data[1] = prep->payload.data[1]; key->payload.data[2] = prep->payload.data[2]; key->payload.data[3] = prep->payload.data[3]; +#endif prep->payload.data[0] = NULL; prep->payload.data[1] = NULL; prep->payload.data[2] = NULL; @@ -1262,6 +1502,16 @@ void __init key_init(void) /* allocate a slab in which we can store keys */ key_jar = kmem_cache_create("key_jar", sizeof(struct key), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); +#ifdef CONFIG_KEYP + key_union_jar = kmem_cache_create("key_union_jar", sizeof(struct key_union), + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + key_struct_jar = kmem_cache_create("key_struct_jar", sizeof(struct key_struct), + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + key_payload_jar = kmem_cache_create("key_payload_jar", sizeof(union key_payload)*4, + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + if (haoc_enabled) + pr_info("HAOC: CONFIG_KEYP enabled."); + #endif /* add the special key types */ list_add_tail(&key_type_keyring.link, &key_types_list); diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index 225129a881822..1c7dc16ec8a4f 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -25,6 +25,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" #define KEY_MAX_DESC_SIZE 4096 @@ -807,11 +810,19 @@ static long __keyctl_read_key(struct key *key, char *buffer, size_t buflen) { long ret; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(key)); +#else down_read(&key->sem); +#endif ret = key_validate(key); if (ret == 0) ret = key->type->read(key, buffer, buflen); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(key)); +#else up_read(&key->sem); +#endif return ret; } @@ -981,7 +992,11 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) /* make the changes with the locks held to prevent chown/chown races */ ret = -EACCES; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif { bool is_privileged_op = false; @@ -1039,19 +1054,32 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) } zapowner = key->user; +#ifdef CONFIG_KEYP + iee_set_key_user(key, newowner); + iee_set_key_uid(key, uid); +#else key->user = newowner; key->uid = uid; +#endif } /* change the GID */ if (group != (gid_t) -1) +#ifdef CONFIG_KEYP + iee_set_key_gid(key, gid); +#else key->gid = gid; +#endif notify_key(key, NOTIFY_KEY_SETATTR, 0); ret = 0; error_put: +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif key_put(key); if (zapowner) key_user_put(zapowner); @@ -1093,16 +1121,28 @@ long keyctl_setperm_key(key_serial_t id, key_perm_t perm) /* make the changes with the locks held to prevent chown/chmod races */ ret = -EACCES; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif /* if we're not the sysadmin, we can only change a key that we own */ if (uid_eq(key->uid, current_fsuid()) || capable(CAP_SYS_ADMIN)) { +#ifdef CONFIG_KEYP + iee_set_key_perm(key, perm); +#else key->perm = perm; +#endif notify_key(key, NOTIFY_KEY_SETATTR, 0); ret = 0; } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif key_put(key); error: return ret; @@ -1856,25 +1896,45 @@ long keyctl_watch_key(key_serial_t id, int watch_queue_fd, int watch_id) if (ret < 0) goto err_watch; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif if (!key->watchers) { +#ifdef CONFIG_KEYP + iee_set_key_watchers(key, wlist); +#else key->watchers = wlist; +#endif wlist = NULL; } ret = add_watch_to_object(watch, key->watchers); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif if (ret == 0) watch = NULL; } else { ret = -EBADSLT; if (key->watchers) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(key)); +#else down_write(&key->sem); +#endif ret = remove_watch_from_object(key->watchers, wqueue, key_serial(key), false); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(key)); +#else up_write(&key->sem); +#endif } } diff --git a/security/keys/keyring.c b/security/keys/keyring.c index e105349794f23..0dbc1a81d6f83 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -19,6 +19,10 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#include +#endif #include "internal.h" /* @@ -112,7 +116,12 @@ static void keyring_publish_name(struct key *keyring) keyring->description[0] && keyring->description[0] != '.') { write_lock(&keyring_name_lock); +#ifdef CONFIG_KEYP + list_add_tail(&(((struct key_struct *)(keyring->name_link.prev))->name_link), + &ns->keyring_name_list); +#else list_add_tail(&keyring->name_link, &ns->keyring_name_list); +#endif write_unlock(&keyring_name_lock); } } @@ -140,7 +149,11 @@ static void keyring_free_preparse(struct key_preparsed_payload *prep) static int keyring_instantiate(struct key *keyring, struct key_preparsed_payload *prep) { +#ifdef CONFIG_KEYP + assoc_array_init(&((struct key_struct *)(keyring->name_link.prev))->keys); +#else assoc_array_init(&keyring->keys); +#endif /* make the keyring available by name if it has one */ keyring_publish_name(keyring); return 0; @@ -207,13 +220,91 @@ static void hash_key_type_and_desc(struct keyring_index_key *index_key) index_key->hash = hash; } +#ifdef CONFIG_KEYP +static void iee_hash_key_type_and_desc(struct keyring_index_key *index_key) +{ + const unsigned int level_shift = ASSOC_ARRAY_LEVEL_STEP; + const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK; + const char *description = index_key->description; + unsigned long hash, type; + u32 piece; + u64 acc; + int n, desc_len = index_key->desc_len; + + type = (unsigned long)index_key->type; + acc = mult_64x32_and_fold(type, desc_len + 13); + acc = mult_64x32_and_fold(acc, 9207); + piece = (unsigned long)index_key->domain_tag; + acc = mult_64x32_and_fold(acc, piece); + acc = mult_64x32_and_fold(acc, 9207); + + for (;;) { + n = desc_len; + if (n <= 0) + break; + if (n > 4) + n = 4; + piece = 0; + memcpy(&piece, description, n); + description += n; + desc_len -= n; + acc = mult_64x32_and_fold(acc, piece); + acc = mult_64x32_and_fold(acc, 9207); + } + + /* Fold the hash down to 32 bits if need be. */ + hash = acc; + if (ASSOC_ARRAY_KEY_CHUNK_SIZE == 32) + hash ^= acc >> 32; + + /* Squidge all the keyrings into a separate part of the tree to + * ordinary keys by making sure the lowest level segment in the hash is + * zero for keyrings and non-zero otherwise. + */ + if (index_key->type != &key_type_keyring && (hash & fan_mask) == 0) + hash |= (hash >> (ASSOC_ARRAY_KEY_CHUNK_SIZE - level_shift)) | 1; + else if (index_key->type == &key_type_keyring && (hash & fan_mask) != 0) + hash = (hash + (hash << level_shift)) & ~fan_mask; + struct keyring_index_key tmp = *index_key; + + tmp.hash = hash; + iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); +} + +static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), }; + +void iee_key_set_index_key(struct keyring_index_key *index_key) +{ + size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc)); + struct keyring_index_key tmp; + + iee_memcpy(index_key->desc, index_key->description, n); + + if (!index_key->domain_tag) { + if (index_key->type->flags & KEY_TYPE_NET_DOMAIN) { + tmp = *index_key; + tmp.domain_tag = current->nsproxy->net_ns->key_domain; + iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); + } else { + tmp = *index_key; + tmp.domain_tag = &default_domain_tag; + iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); + } + } + + iee_hash_key_type_and_desc(index_key); +} +#endif + /* * Finalise an index key to include a part of the description actually in the * index key, to set the domain tag and to calculate the hash. */ void key_set_index_key(struct keyring_index_key *index_key) { +#ifndef CONFIG_KEYP static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), }; +#endif size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc)); memcpy(index_key->desc, index_key->description, n); @@ -414,9 +505,15 @@ static void keyring_destroy(struct key *keyring) if (keyring->description) { write_lock(&keyring_name_lock); +#ifdef CONFIG_KEYP + if (((struct key_struct *)(keyring->name_link.prev))->name_link.next != NULL && + !list_empty(&(((struct key_struct *)(keyring->name_link.prev))->name_link))) + list_del(&(((struct key_struct *)(keyring->name_link.prev))->name_link)); +#else if (keyring->name_link.next != NULL && !list_empty(&keyring->name_link)) list_del(&keyring->name_link); +#endif write_unlock(&keyring_name_lock); } @@ -428,7 +525,12 @@ static void keyring_destroy(struct key *keyring) kfree(keyres); } +#ifdef CONFIG_KEYP + assoc_array_destroy(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops); +#else assoc_array_destroy(&keyring->keys, &keyring_assoc_array_ops); +#endif } /* @@ -442,8 +544,14 @@ static void keyring_describe(const struct key *keyring, struct seq_file *m) seq_puts(m, "[anon]"); if (key_is_positive(keyring)) { +#ifdef CONFIG_KEYP + if (((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree != 0) + seq_printf(m, ": %lu", ((struct key_struct *)(keyring->name_link.prev))-> + keys.nr_leaves_on_tree); +#else if (keyring->keys.nr_leaves_on_tree != 0) seq_printf(m, ": %lu", keyring->keys.nr_leaves_on_tree); +#endif else seq_puts(m, ": empty"); } @@ -494,8 +602,13 @@ static long keyring_read(const struct key *keyring, ctx.buffer = (key_serial_t *)buffer; ctx.buflen = buflen; ctx.count = 0; +#ifdef CONFIG_KEYP + ret = assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, + keyring_read_iterator, &ctx); +#else ret = assoc_array_iterate(&keyring->keys, keyring_read_iterator, &ctx); +#endif if (ret < 0) { kleave(" = %ld [iterate]", ret); return ret; @@ -503,7 +616,12 @@ static long keyring_read(const struct key *keyring, } /* Return the size of the buffer needed */ +#ifdef CONFIG_KEYP + ret = ((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree * + sizeof(key_serial_t); +#else ret = keyring->keys.nr_leaves_on_tree * sizeof(key_serial_t); +#endif if (ret <= buflen) kleave("= %ld [ok]", ret); else @@ -648,12 +766,23 @@ static int search_keyring(struct key *keyring, struct keyring_search_context *ct if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_DIRECT) { const void *object; +#ifdef CONFIG_KEYP + object = assoc_array_find(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, + &ctx->index_key); +#else object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops, &ctx->index_key); +#endif return object ? ctx->iterator(object, ctx) : 0; } +#ifdef CONFIG_KEYP + return assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, + ctx->iterator, ctx); +#else return assoc_array_iterate(&keyring->keys, ctx->iterator, ctx); +#endif } /* @@ -729,7 +858,11 @@ static bool search_nested_keyrings(struct key *keyring, if (!(ctx->flags & KEYRING_SEARCH_RECURSE)) goto not_this_keyring; +#ifdef CONFIG_KEYP + ptr = READ_ONCE(((struct key_struct *)(keyring->name_link.prev))->keys.root); +#else ptr = READ_ONCE(keyring->keys.root); +#endif if (!ptr) goto not_this_keyring; @@ -856,10 +989,17 @@ static bool search_nested_keyrings(struct key *keyring, key = key_ref_to_ptr(ctx->result); key_check(key); if (!(ctx->flags & KEYRING_SEARCH_NO_UPDATE_TIME)) { +#ifdef CONFIG_KEYP + iee_set_key_last_used_at(key, ctx->now); + iee_set_key_last_used_at(keyring, ctx->now); + while (sp > 0) + iee_set_key_last_used_at(stack[--sp].keyring, ctx->now); +#else key->last_used_at = ctx->now; keyring->last_used_at = ctx->now; while (sp > 0) stack[--sp].keyring->last_used_at = ctx->now; +#endif } kleave(" = true"); return true; @@ -1056,7 +1196,11 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, goto error; } +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif down_write(&keyring_serialise_restrict_sem); if (keyring->restrict_link) { @@ -1064,12 +1208,20 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, } else if (keyring_detect_restriction_cycle(keyring, restrict_link)) { ret = -EDEADLK; } else { +#ifdef CONFIG_KEYP + iee_set_key_restrict_link(keyring, restrict_link); +#else keyring->restrict_link = restrict_link; +#endif notify_key(keyring, NOTIFY_KEY_SETATTR, 0); } up_write(&keyring_serialise_restrict_sem); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif if (ret < 0) { key_put(restrict_link->key); @@ -1110,8 +1262,13 @@ key_ref_t find_key_to_update(key_ref_t keyring_ref, keyring->serial, index_key->type->name, index_key->description); guard(rcu)(); +#ifdef CONFIG_KEYP + object = assoc_array_find(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, index_key); +#else object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops, index_key); +#endif if (object) goto found; @@ -1155,7 +1312,14 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) /* Search this hash bucket for a keyring with a matching name that * grants Search permission and that hasn't been revoked */ +#ifdef CONFIG_KEYP + struct key_struct *key_struct; + + list_for_each_entry(key_struct, &ns->keyring_name_list, name_link) { + keyring = key_struct->key; +#else list_for_each_entry(keyring, &ns->keyring_name_list, name_link) { +#endif if (!kuid_has_mapping(ns, keyring->user->uid)) continue; @@ -1178,9 +1342,15 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) /* we've got a match but we might end up racing with * key_cleanup() if the keyring is currently 'dead' * (ie. it has a zero usage count) */ +#ifdef CONFIG_KEYP + if (!iee_set_key_usage(keyring, 0, REFCOUNT_INC_NOT_ZERO)) + continue; + iee_set_key_last_used_at(keyring, ktime_get_real_seconds()); +#else if (!refcount_inc_not_zero(&keyring->usage)) continue; keyring->last_used_at = ktime_get_real_seconds(); +#endif goto out; } @@ -1239,13 +1409,21 @@ static int keyring_detect_cycle(struct key *A, struct key *B) */ int __key_link_lock(struct key *keyring, const struct keyring_index_key *index_key) +#ifdef CONFIG_KEYP + __acquires(&KEY_SEM(keyring)) +#else __acquires(&keyring->sem) +#endif __acquires(&keyring_serialise_link_lock) { if (keyring->type != &key_type_keyring) return -ENOTDIR; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif /* Serialise link/link calls to prevent parallel calls causing a cycle * when linking two keyring in opposite orders. @@ -1261,8 +1439,13 @@ int __key_link_lock(struct key *keyring, */ int __key_move_lock(struct key *l_keyring, struct key *u_keyring, const struct keyring_index_key *index_key) +#ifdef CONFIG_KEYP + __acquires(&KEY_SEM(l_keyring)) + __acquires(&KEY_SEM(u_keyring)) +#else __acquires(&l_keyring->sem) __acquires(&u_keyring->sem) +#endif __acquires(&keyring_serialise_link_lock) { if (l_keyring->type != &key_type_keyring || @@ -1274,11 +1457,21 @@ int __key_move_lock(struct key *l_keyring, struct key *u_keyring, * move operation. */ if (l_keyring < u_keyring) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(l_keyring)); + down_write_nested(&KEY_SEM(u_keyring), 1); +#else down_write(&l_keyring->sem); down_write_nested(&u_keyring->sem, 1); +#endif } else { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(u_keyring)); + down_write_nested(&KEY_SEM(l_keyring), 1); +#else down_write(&u_keyring->sem); down_write_nested(&l_keyring->sem, 1); +#endif } /* Serialise link/link calls to prevent parallel calls causing a cycle @@ -1315,10 +1508,17 @@ int __key_link_begin(struct key *keyring, /* Create an edit script that will insert/replace the key in the * keyring tree. */ +#ifdef CONFIG_KEYP + edit = assoc_array_insert(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, + index_key, + NULL); +#else edit = assoc_array_insert(&keyring->keys, &keyring_assoc_array_ops, index_key, NULL); +#endif if (IS_ERR(edit)) { ret = PTR_ERR(edit); goto error; @@ -1386,7 +1586,11 @@ void __key_link(struct key *keyring, struct key *key, void __key_link_end(struct key *keyring, const struct keyring_index_key *index_key, struct assoc_array_edit *edit) +#ifdef CONFIG_KEYP + __releases(&KEY_SEM(keyring)) +#else __releases(&keyring->sem) +#endif __releases(&keyring_serialise_link_lock) { BUG_ON(index_key->type == NULL); @@ -1399,7 +1603,11 @@ void __key_link_end(struct key *keyring, } assoc_array_cancel_edit(edit); } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif if (index_key->type == &key_type_keyring) mutex_unlock(&keyring_serialise_link_lock); @@ -1412,8 +1620,13 @@ static int __key_link_check_restriction(struct key *keyring, struct key *key) { if (!keyring->restrict_link || !keyring->restrict_link->check) return 0; +#ifdef CONFIG_KEYP + return keyring->restrict_link->check(keyring, key->type, + ((union key_payload *)(key->name_link.next)), keyring->restrict_link->key); +#else return keyring->restrict_link->check(keyring, key->type, &key->payload, keyring->restrict_link->key); +#endif } /** @@ -1473,12 +1686,20 @@ EXPORT_SYMBOL(key_link); * Lock a keyring for unlink. */ static int __key_unlink_lock(struct key *keyring) +#ifdef CONFIG_KEYP + __acquires(&KEY_SEM(keyring)) +#else __acquires(&keyring->sem) +#endif { if (keyring->type != &key_type_keyring) return -ENOTDIR; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif return 0; } @@ -1492,8 +1713,13 @@ static int __key_unlink_begin(struct key *keyring, struct key *key, BUG_ON(*_edit != NULL); +#ifdef CONFIG_KEYP + edit = assoc_array_delete(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, &key->index_key); +#else edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops, &key->index_key); +#endif if (IS_ERR(edit)) return PTR_ERR(edit); @@ -1522,11 +1748,19 @@ static void __key_unlink(struct key *keyring, struct key *key, static void __key_unlink_end(struct key *keyring, struct key *key, struct assoc_array_edit *edit) +#ifdef CONFIG_KEYP + __releases(&KEY_SEM(keyring)) +#else __releases(&keyring->sem) +#endif { if (edit) assoc_array_cancel_edit(edit); +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif } /** @@ -1656,9 +1890,16 @@ int keyring_clear(struct key *keyring) if (keyring->type != &key_type_keyring) return -ENOTDIR; +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); + + edit = assoc_array_clear(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops); +#else down_write(&keyring->sem); edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops); +#endif if (IS_ERR(edit)) { ret = PTR_ERR(edit); } else { @@ -1669,7 +1910,11 @@ int keyring_clear(struct key *keyring) ret = 0; } +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif return ret; } EXPORT_SYMBOL(keyring_clear); @@ -1683,7 +1928,12 @@ static void keyring_revoke(struct key *keyring) { struct assoc_array_edit *edit; +#ifdef CONFIG_KEYP + edit = assoc_array_clear(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops); +#else edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops); +#endif if (!IS_ERR(edit)) { if (edit) assoc_array_apply_edit(edit); @@ -1729,8 +1979,13 @@ void keyring_gc(struct key *keyring, time64_t limit) /* scan the keyring looking for dead keys */ rcu_read_lock(); +#ifdef CONFIG_KEYP + result = assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, + keyring_gc_check_iterator, &limit); +#else result = assoc_array_iterate(&keyring->keys, keyring_gc_check_iterator, &limit); +#endif rcu_read_unlock(); if (result == true) goto do_gc; @@ -1740,10 +1995,17 @@ void keyring_gc(struct key *keyring, time64_t limit) return; do_gc: +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); + assoc_array_gc(&((struct key_struct *)(keyring->name_link.prev))->keys, + &keyring_assoc_array_ops, keyring_gc_select_iterator, &limit); + up_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); assoc_array_gc(&keyring->keys, &keyring_assoc_array_ops, keyring_gc_select_iterator, &limit); up_write(&keyring->sem); +#endif kleave(" [gc]"); } @@ -1782,7 +2044,11 @@ void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type) } /* Lock the keyring to ensure that a link is not in progress */ +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(keyring)); +#else down_write(&keyring->sem); +#endif keyres = keyring->restrict_link; @@ -1792,7 +2058,11 @@ void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type) keyres->key = NULL; keyres->keytype = NULL; +#ifdef CONFIG_KEYP + up_write(&KEY_SEM(keyring)); +#else up_write(&keyring->sem); +#endif kleave(" [restriction gc]"); } diff --git a/security/keys/proc.c b/security/keys/proc.c index 4f4e2c1824f18..e7f08b7d877ea 100644 --- a/security/keys/proc.c +++ b/security/keys/proc.c @@ -67,7 +67,11 @@ static struct rb_node *key_serial_next(struct seq_file *p, struct rb_node *n) n = rb_next(n); while (n) { +#ifdef CONFIG_KEYP + struct key *key = rb_entry(n, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(n, struct key, serial_node); +#endif if (kuid_has_mapping(user_ns, key->user->uid)) break; n = rb_next(n); @@ -82,7 +86,11 @@ static struct key *find_ge_key(struct seq_file *p, key_serial_t id) struct key *minkey = NULL; while (n) { +#ifdef CONFIG_KEYP + struct key *key = rb_entry(n, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(n, struct key, serial_node); +#endif if (id < key->serial) { if (!minkey || minkey->serial > key->serial) minkey = key; @@ -102,10 +110,18 @@ static struct key *find_ge_key(struct seq_file *p, key_serial_t id) for (;;) { if (kuid_has_mapping(user_ns, minkey->user->uid)) return minkey; +#ifdef CONFIG_KEYP + n = rb_next(&(((struct key_union *)(minkey->graveyard_link.next))->serial_node)); +#else n = rb_next(&minkey->serial_node); +#endif if (!n) return NULL; +#ifdef CONFIG_KEYP + minkey = rb_entry(n, struct key_union, serial_node)->key; +#else minkey = rb_entry(n, struct key, serial_node); +#endif } } @@ -123,12 +139,20 @@ static void *proc_keys_start(struct seq_file *p, loff_t *_pos) if (!key) return NULL; *_pos = key->serial; +#ifdef CONFIG_KEYP + return &(((struct key_union *)(key->graveyard_link.next))->serial_node); +#else return &key->serial_node; +#endif } static inline key_serial_t key_node_serial(struct rb_node *n) { +#ifdef CONFIG_KEYP + struct key *key = rb_entry(n, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(n, struct key, serial_node); +#endif return key->serial; } @@ -153,7 +177,11 @@ static void proc_keys_stop(struct seq_file *p, void *v) static int proc_keys_show(struct seq_file *m, void *v) { struct rb_node *_p = v; +#ifdef CONFIG_KEYP + struct key *key = rb_entry(_p, struct key_union, serial_node)->key; +#else struct key *key = rb_entry(_p, struct key, serial_node); +#endif unsigned long flags; key_ref_t key_ref, skey_ref; time64_t now, expiry; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index aa56394380fd7..067c225a466a5 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -20,6 +20,9 @@ #ifdef CONFIG_CREDP #include #endif +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" /* Session keyring create vs join semaphore */ @@ -393,9 +396,15 @@ void key_fsuid_changed(struct cred *new_cred) { /* update the ownership of the thread keyring */ if (new_cred->thread_keyring) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(new_cred->thread_keyring)); + iee_set_key_uid(new_cred->thread_keyring, new_cred->fsuid); + up_write(&KEY_SEM(new_cred->thread_keyring)); +#else down_write(&new_cred->thread_keyring->sem); new_cred->thread_keyring->uid = new_cred->fsuid; up_write(&new_cred->thread_keyring->sem); +#endif } } @@ -406,9 +415,15 @@ void key_fsgid_changed(struct cred *new_cred) { /* update the ownership of the thread keyring */ if (new_cred->thread_keyring) { +#ifdef CONFIG_KEYP + down_write(&KEY_SEM(new_cred->thread_keyring)); + iee_set_key_gid(new_cred->thread_keyring, new_cred->fsgid); + up_write(&KEY_SEM(new_cred->thread_keyring)); +#else down_write(&new_cred->thread_keyring->sem); new_cred->thread_keyring->gid = new_cred->fsgid; up_write(&new_cred->thread_keyring->sem); +#endif } } @@ -572,7 +587,14 @@ key_ref_t search_process_keyrings_rcu(struct keyring_search_context *ctx) const struct cred *cred = ctx->cred; if (key_validate(cred->request_key_auth) == 0) { +#ifdef CONFIG_KEYP + union key_payload *tmp = + (union key_payload *)(ctx->cred->request_key_auth->name_link.next); + + rka = tmp->data[0]; +#else rka = ctx->cred->request_key_auth->payload.data[0]; +#endif //// was search_process_keyrings() [ie. recursive] ctx->cred = rka->cred; @@ -740,17 +762,30 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, if (!ctx.cred->request_key_auth) goto error; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(ctx.cred->request_key_auth)); +#else down_read(&ctx.cred->request_key_auth->sem); +#endif if (test_bit(KEY_FLAG_REVOKED, &ctx.cred->request_key_auth->flags)) { key_ref = ERR_PTR(-EKEYREVOKED); key = NULL; } else { +#ifdef CONFIG_KEYP + rka = ((union key_payload *)(ctx.cred->request_key_auth->name_link.next)) + ->data[0]; +#else rka = ctx.cred->request_key_auth->payload.data[0]; +#endif key = rka->dest_keyring; __key_get(key); } +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(ctx.cred->request_key_auth)); +#else up_read(&ctx.cred->request_key_auth->sem); +#endif if (!key) goto error; key_ref = make_key_ref(key, 1); @@ -819,7 +854,11 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, if (ret < 0) goto invalid_key; +#ifdef CONFIG_KEYP + iee_set_key_last_used_at(key, ktime_get_real_seconds()); +#else key->last_used_at = ktime_get_real_seconds(); +#endif error: put_cred(ctx.cred); @@ -949,30 +988,11 @@ void key_change_session_keyring(struct callback_head *twork) } #ifdef CONFIG_CREDP - iee_set_cred_uid(new, old->uid); - iee_set_cred_euid(new, old->euid); - iee_set_cred_suid(new, old->suid); - iee_set_cred_fsuid(new, old->fsuid); - iee_set_cred_gid(new, old->gid); - iee_set_cred_egid(new, old->egid); - iee_set_cred_sgid(new, old->sgid); - iee_set_cred_fsgid(new, old->fsgid); - iee_set_cred_user(new, get_uid(old->user)); - iee_set_cred_ucounts(new, old->ucounts); - iee_set_cred_user_ns(new, get_user_ns(old->user_ns)); - iee_set_cred_group_info(new, get_group_info(old->group_info)); - - iee_set_cred_securebits(new, old->securebits); - iee_set_cred_cap_inheritable(new, old->cap_inheritable); - iee_set_cred_cap_permitted(new, old->cap_permitted); - iee_set_cred_cap_effective(new, old->cap_effective); - iee_set_cred_cap_ambient(new, old->cap_ambient); - iee_set_cred_cap_bset(new, old->cap_bset); - - iee_set_cred_jit_keyring(new, old->jit_keyring); - iee_set_cred_thread_keyring(new, key_get(old->thread_keyring)); - iee_set_cred_process_keyring(new, key_get(old->process_keyring)); - #else + if (haoc_enabled) + iee_fill_cred_for_session_keyring(new, old); + else + #endif + { new-> uid = old-> uid; new-> euid = old-> euid; new-> suid = old-> suid; @@ -996,7 +1016,7 @@ void key_change_session_keyring(struct callback_head *twork) new->jit_keyring = old->jit_keyring; new->thread_keyring = key_get(old->thread_keyring); new->process_keyring = key_get(old->process_keyring); - #endif + } security_transfer_creds(new, old); diff --git a/security/keys/request_key.c b/security/keys/request_key.c index a7673ad86d18d..79fe763555e1d 100644 --- a/security/keys/request_key.c +++ b/security/keys/request_key.c @@ -14,6 +14,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" #include @@ -285,13 +288,21 @@ static int construct_get_dest_keyring(struct key **_dest_keyring) case KEY_REQKEY_DEFL_REQUESTOR_KEYRING: if (cred->request_key_auth) { authkey = cred->request_key_auth; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(authkey)); +#else down_read(&authkey->sem); +#endif rka = get_request_key_auth(authkey); if (!test_bit(KEY_FLAG_REVOKED, &authkey->flags)) dest_keyring = key_get(rka->dest_keyring); +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(authkey)); +#else up_read(&authkey->sem); +#endif if (dest_keyring) { do_perm_check = false; break; @@ -398,7 +409,11 @@ static int construct_alloc_key(struct keyring_search_context *ctx, if (IS_ERR(key)) goto alloc_failed; +#ifdef CONFIG_KEYP + iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, SET_BIT_OP); +#else set_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags); +#endif if (dest_keyring) { ret = __key_link_lock(dest_keyring, &key->index_key); diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c index bd34317b58cc8..2a53337729bb2 100644 --- a/security/keys/request_key_auth.c +++ b/security/keys/request_key_auth.c @@ -118,19 +118,27 @@ static void free_request_key_auth(struct request_key_auth *rka) /* * Take a reference to the request-key authorisation payload so callers can - * drop authkey->sem before doing operations that may sleep. + * drop the auth key semaphore before doing operations that may sleep. */ struct request_key_auth *request_key_auth_get(struct key *authkey) { struct request_key_auth *rka; +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(authkey)); +#else down_read(&authkey->sem); +#endif rka = dereference_key_locked(authkey); if (rka && !test_bit(KEY_FLAG_REVOKED, &authkey->flags)) refcount_inc(&rka->usage); else rka = NULL; +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(authkey)); +#else up_read(&authkey->sem); +#endif return rka; } @@ -173,7 +181,12 @@ static void request_key_auth_revoke(struct key *key) */ static void request_key_auth_destroy(struct key *key) { +#ifdef CONFIG_KEYP + union key_payload *tmp = (union key_payload *)(key->name_link.next); + struct request_key_auth *rka = rcu_access_pointer(tmp->rcu_data0); +#else struct request_key_auth *rka = rcu_access_pointer(key->payload.rcu_data0); +#endif kenter("{%d}", key->serial); if (rka) { @@ -213,22 +226,43 @@ struct key *request_key_auth_new(struct key *target, const char *op, * another process */ if (cred->request_key_auth) { /* it is - use that instantiation context here too */ +#ifdef CONFIG_KEYP + down_read(&KEY_SEM(cred->request_key_auth)); +#else down_read(&cred->request_key_auth->sem); +#endif /* if the auth key has been revoked, then the key we're * servicing is already instantiated */ if (test_bit(KEY_FLAG_REVOKED, &cred->request_key_auth->flags)) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(cred->request_key_auth)); +#else up_read(&cred->request_key_auth->sem); +#endif ret = -EKEYREVOKED; goto error_free_rka; } - irka = cred->request_key_auth->payload.data[0]; + irka = dereference_key_locked(cred->request_key_auth); + if (!irka) { +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(cred->request_key_auth)); +#else + up_read(&cred->request_key_auth->sem); +#endif + ret = -EKEYREVOKED; + goto error_free_rka; + } rka->cred = get_cred(irka->cred); rka->pid = irka->pid; +#ifdef CONFIG_KEYP + up_read(&KEY_SEM(cred->request_key_auth)); +#else up_read(&cred->request_key_auth->sem); +#endif } else { /* it isn't - use this process as the context */ diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index fee1ab2c734d3..29037ece458b2 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -233,7 +233,11 @@ static int trusted_update(struct key *key, struct key_preparsed_payload *prep) if (key_is_negative(key)) return -ENOKEY; +#ifdef CONFIG_KEYP + p = ((union key_payload *)(key->name_link.next))->data[0]; +#else p = key->payload.data[0]; +#endif if (!p->migratable) return -EPERM; if (datalen <= 0 || datalen > 32767 || !prep->data) @@ -307,7 +311,11 @@ static long trusted_read(const struct key *key, char *buffer, */ static void trusted_destroy(struct key *key) { +#ifdef CONFIG_KEYP + kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); +#else kfree_sensitive(key->payload.data[0]); +#endif } struct key_type key_type_trusted = { diff --git a/security/keys/user_defined.c b/security/keys/user_defined.c index 749e2a4dcb13e..23a3012927d9d 100644 --- a/security/keys/user_defined.c +++ b/security/keys/user_defined.c @@ -12,6 +12,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "internal.h" static int logon_vet_description(const char *desc); @@ -109,7 +112,11 @@ int user_update(struct key *key, struct key_preparsed_payload *prep) return ret; /* attach the new data, displacing the old */ +#ifdef CONFIG_KEYP + iee_set_key_expiry(key, prep->expiry); +#else key->expiry = prep->expiry; +#endif if (key_is_positive(key)) zap = dereference_key_locked(key); rcu_assign_keypointer(key, prep->payload.data[0]); @@ -145,7 +152,11 @@ EXPORT_SYMBOL(user_revoke); */ void user_destroy(struct key *key) { +#ifdef CONFIG_KEYP + struct user_key_payload *upayload = ((union key_payload *)(key->name_link.next))->data[0]; +#else struct user_key_payload *upayload = key->payload.data[0]; +#endif kfree_sensitive(upayload); } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 8560e2fc9dc07..dce4479cfe307 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -93,6 +93,13 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif +#ifdef CONFIG_IEE_SELINUX_P +#include +#include +#endif #include "avc.h" #include "objsec.h" @@ -107,7 +114,11 @@ #define SELINUX_INODE_INIT_XATTRS 1 +#ifdef CONFIG_IEE_SELINUX_P +struct selinux_state selinux_state __section(".iee.selinux"); +#else struct selinux_state selinux_state; +#endif /* SECMARK reference count */ static atomic_t selinux_secmark_refcount = ATOMIC_INIT(0); @@ -6715,7 +6726,11 @@ static int selinux_key_alloc(struct key *k, const struct cred *cred, else ksec->sid = tsec->sid; +#ifdef CONFIG_KEYP + iee_set_key_security(k, ksec); +#else k->security = ksec; +#endif return 0; } @@ -6723,7 +6738,11 @@ static void selinux_key_free(struct key *k) { struct key_security_struct *ksec = k->security; +#ifdef CONFIG_KEYP + iee_set_key_security(k, NULL); +#else k->security = NULL; +#endif kfree(ksec); } @@ -7418,15 +7437,51 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { #endif }; +#ifdef CONFIG_IEE_SELINUX_P +struct kmem_cache *policy_jar; + +static void policy_cache_init(void) +{ + struct selinux_policy *unused; + + policy_jar = kmem_cache_create("policy_jar", sizeof(struct selinux_policy), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + /* Test this cache */ + unused = kmem_cache_alloc(policy_jar, GFP_KERNEL); + kmem_cache_free(policy_jar, unused); + + if (haoc_enabled) + pr_info("IEE SELINUXP: policy cache created."); +} +#endif + static __init int selinux_init(void) { pr_info("SELinux: Initializing.\n"); memset(&selinux_state, 0, sizeof(selinux_state)); +#ifdef CONFIG_IEE_SELINUX_P + WRITE_ONCE(selinux_state.enforcing, selinux_enforcing_boot); +#else enforcing_set(selinux_enforcing_boot); +#endif selinux_avc_init(); +#ifdef CONFIG_IEE_SELINUX_P + /* Put selinux_status inside IEE. */ + /* Prepare mutex lock and write the ptr to mutex->owner. */ + struct mutex *status_lock = kzalloc(GFP_KERNEL, sizeof(struct mutex)); + struct mutex *policy_mutex = kzalloc(GFP_KERNEL, sizeof(struct mutex)); + + mutex_init(status_lock); + mutex_init(policy_mutex); + selinux_state.status_lock.owner.counter = (s64)status_lock; + selinux_state.policy_mutex.owner.counter = (s64)policy_mutex; + + policy_cache_init(); +#else mutex_init(&selinux_state.status_lock); mutex_init(&selinux_state.policy_mutex); +#endif /* Set the security state for the initial task. */ cred_init_security(); diff --git a/security/selinux/ima.c b/security/selinux/ima.c index aa34da9b0aebd..498a1f9c8fd18 100644 --- a/security/selinux/ima.c +++ b/security/selinux/ima.c @@ -9,6 +9,9 @@ */ #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif #include "security.h" #include "ima.h" @@ -74,7 +77,11 @@ void selinux_ima_measure_state_locked(void) size_t policy_len; int rc = 0; +#ifdef CONFIG_IEE_SELINUX_P + lockdep_assert_held(iee_get_selinux_policy_lock()); +#else lockdep_assert_held(&selinux_state.policy_mutex); +#endif state_str = selinux_ima_collect_state(); if (!state_str) { @@ -112,9 +119,17 @@ void selinux_ima_measure_state_locked(void) */ void selinux_ima_measure_state(void) { +#ifdef CONFIG_IEE_SELINUX_P + lockdep_assert_not_held(iee_get_selinux_policy_lock()); + + mutex_lock(iee_get_selinux_policy_lock()); + selinux_ima_measure_state_locked(); + mutex_unlock(iee_get_selinux_policy_lock()); +#else lockdep_assert_not_held(&selinux_state.policy_mutex); mutex_lock(&selinux_state.policy_mutex); selinux_ima_measure_state_locked(); mutex_unlock(&selinux_state.policy_mutex); +#endif } diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index a9de89af8fdc5..8ba4c3c95a50e 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -113,11 +113,13 @@ static inline bool selinux_initialized(void) return smp_load_acquire(&selinux_state.initialized); } +#ifndef CONFIG_IEE_SELINUX_P static inline void selinux_mark_initialized(void) { /* do a synchronized write to avoid race conditions */ smp_store_release(&selinux_state.initialized, true); } +#endif #ifdef CONFIG_SECURITY_SELINUX_DEVELOP static inline bool enforcing_enabled(void) @@ -125,10 +127,13 @@ static inline bool enforcing_enabled(void) return READ_ONCE(selinux_state.enforcing); } +#ifndef CONFIG_IEE_SELINUX_P static inline void enforcing_set(bool value) { WRITE_ONCE(selinux_state.enforcing, value); } +#endif + #else static inline bool enforcing_enabled(void) { diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c index 3e28fa2444fbd..00621fb1f979e 100644 --- a/security/selinux/selinuxfs.c +++ b/security/selinux/selinuxfs.c @@ -31,6 +31,9 @@ #include #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif /* selinuxfs pseudo filesystem for exporting the security policy API. Based on the proc code and the fs/nfsd/nfsctl.c code. */ @@ -349,7 +352,11 @@ static int sel_open_policy(struct inode *inode, struct file *filp) BUG_ON(filp->private_data); +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif rc = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__READ_POLICY, NULL); @@ -379,11 +386,19 @@ static int sel_open_policy(struct inode *inode, struct file *filp) filp->private_data = plm; +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif return 0; err: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif if (plm) vfree(plm->data); @@ -585,7 +600,11 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, goto out; } +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif length = security_load_policy(data, count, &load_state); if (length) { pr_warn_ratelimited("SELinux: failed to load policy\n"); @@ -607,7 +626,11 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, audit_get_sessionid(current)); out_unlock: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif out: vfree(data); return length; @@ -1192,7 +1215,11 @@ static ssize_t sel_read_bool(struct file *filep, char __user *buf, unsigned index = file_inode(filep)->i_ino & SEL_INO_MASK; const char *name = filep->f_path.dentry->d_name.name; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif ret = -EINVAL; if (index >= fsi->bool_num || strcmp(name, @@ -1211,14 +1238,22 @@ static ssize_t sel_read_bool(struct file *filep, char __user *buf, } length = scnprintf(page, PAGE_SIZE, "%d %d", cur_enforcing, fsi->bool_pending_values[index]); +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif ret = simple_read_from_buffer(buf, count, ppos, page, length); out_free: free_page((unsigned long)page); return ret; out_unlock: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif goto out_free; } @@ -1243,7 +1278,11 @@ static ssize_t sel_write_bool(struct file *filep, const char __user *buf, if (IS_ERR(page)) return PTR_ERR(page); +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif length = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__SETBOOL, @@ -1267,7 +1306,11 @@ static ssize_t sel_write_bool(struct file *filep, const char __user *buf, length = count; out: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif kfree(page); return length; } @@ -1298,7 +1341,11 @@ static ssize_t sel_commit_bools_write(struct file *filep, if (IS_ERR(page)) return PTR_ERR(page); +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_policy_lock()); +#else mutex_lock(&selinux_state.policy_mutex); +#endif length = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__SETBOOL, @@ -1319,7 +1366,11 @@ static ssize_t sel_commit_bools_write(struct file *filep, length = count; out: +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_policy_lock()); +#else mutex_unlock(&selinux_state.policy_mutex); +#endif kfree(page); return length; } diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 03427046c8242..bf1068f0f7645 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -49,6 +49,9 @@ #include #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif #include "flask.h" #include "avc.h" @@ -2114,9 +2117,14 @@ static void security_load_policycaps(struct selinux_policy *policy) p = &policy->policydb; +#ifdef CONFIG_IEE_SELINUX_P + for (i = 0; i < ARRAY_SIZE(selinux_state.policycap); i++) + iee_set_sel_policy_cap(i, ebitmap_get_bit(&p->policycaps, i)); +#else for (i = 0; i < ARRAY_SIZE(selinux_state.policycap); i++) WRITE_ONCE(selinux_state.policycap[i], ebitmap_get_bit(&p->policycaps, i)); +#endif for (i = 0; i < ARRAY_SIZE(selinux_policycap_names); i++) pr_info("SELinux: policy capability %s=%d\n", @@ -2179,6 +2187,9 @@ void selinux_policy_commit(struct selinux_load_state *load_state) { struct selinux_state *state = &selinux_state; struct selinux_policy *oldpolicy, *newpolicy = load_state->policy; +#ifdef CONFIG_IEE_SELINUX_P + struct selinux_policy *temppolicy; +#endif unsigned long flags; u32 seqno; @@ -2203,10 +2214,20 @@ void selinux_policy_commit(struct selinux_load_state *load_state) /* Install the new policy. */ if (oldpolicy) { sidtab_freeze_begin(oldpolicy->sidtab, &flags); +#ifdef CONFIG_IEE_SELINUX_P + iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); + kfree(newpolicy); +#else rcu_assign_pointer(state->policy, newpolicy); +#endif sidtab_freeze_end(oldpolicy->sidtab, &flags); } else { +#ifdef CONFIG_IEE_SELINUX_P + iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); + kfree(newpolicy); +#else rcu_assign_pointer(state->policy, newpolicy); +#endif } /* Load the policycaps from the new policy */ @@ -2224,7 +2245,20 @@ void selinux_policy_commit(struct selinux_load_state *load_state) /* Free the old policy */ synchronize_rcu(); +#ifdef CONFIG_IEE_SELINUX_P + /* + * Normal free process includes setting freed objects pointers to be NULL, however it + * would be hard as old policy is already inside IEE. So Make a kernel copy of the old + * policy to free objects it points to. + */ + if (oldpolicy) { + temppolicy = kmemdup(oldpolicy, sizeof(*temppolicy), GFP_KERNEL); + selinux_policy_free(temppolicy); + kfree(oldpolicy); + } +#else selinux_policy_free(oldpolicy); +#endif kfree(load_state->convert_data); /* Notify others of the policy change */ @@ -3022,6 +3056,9 @@ int security_set_bools(u32 len, int *values) { struct selinux_state *state = &selinux_state; struct selinux_policy *newpolicy, *oldpolicy; +#ifdef CONFIG_IEE_SELINUX_P + struct selinux_policy *temppolicy; +#endif int rc; u32 i, seqno = 0; @@ -3075,7 +3112,12 @@ int security_set_bools(u32 len, int *values) seqno = newpolicy->latest_granting; /* Install the new policy */ +#ifdef CONFIG_IEE_SELINUX_P + iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); + kfree(newpolicy); +#else rcu_assign_pointer(state->policy, newpolicy); +#endif /* * Free the conditional portions of the old policydb @@ -3083,7 +3125,20 @@ int security_set_bools(u32 len, int *values) * structure itself but not what it references. */ synchronize_rcu(); +#ifdef CONFIG_IEE_SELINUX_P + /* + * Normal free process includes setting freed objects pointers to be NULL, however it + * would be hard as old policy is already inside IEE. So Make a kernel copy of the old + * policy to free objects it points to. + */ + temppolicy = kmemdup(oldpolicy, sizeof(*temppolicy), GFP_KERNEL); + if (!temppolicy) + return -ENOMEM; + selinux_policy_cond_free(temppolicy); + kfree(oldpolicy); +#else selinux_policy_cond_free(oldpolicy); +#endif /* Notify others of the policy change */ selinux_notify_policy_change(seqno); diff --git a/security/selinux/status.c b/security/selinux/status.c index dffca22ce6f74..25a4589c261b7 100644 --- a/security/selinux/status.c +++ b/security/selinux/status.c @@ -10,6 +10,9 @@ #include #include #include +#ifdef CONFIG_IEE_SELINUX_P +#include +#endif #include "avc.h" #include "security.h" @@ -44,9 +47,17 @@ struct page *selinux_kernel_status_page(void) struct selinux_kernel_status *status; struct page *result = NULL; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_status_lock()); +#else mutex_lock(&selinux_state.status_lock); +#endif if (!selinux_state.status_page) { +#ifdef CONFIG_IEE_SELINUX_P + iee_set_selinux_status_pg(alloc_page(GFP_KERNEL|__GFP_ZERO)); +#else selinux_state.status_page = alloc_page(GFP_KERNEL|__GFP_ZERO); +#endif if (selinux_state.status_page) { status = page_address(selinux_state.status_page); @@ -66,7 +77,11 @@ struct page *selinux_kernel_status_page(void) } } result = selinux_state.status_page; +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_status_lock()); +#else mutex_unlock(&selinux_state.status_lock); +#endif return result; } @@ -80,7 +95,11 @@ void selinux_status_update_setenforce(bool enforcing) { struct selinux_kernel_status *status; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_status_lock()); +#else mutex_lock(&selinux_state.status_lock); +#endif if (selinux_state.status_page) { status = page_address(selinux_state.status_page); @@ -92,7 +111,11 @@ void selinux_status_update_setenforce(bool enforcing) smp_wmb(); status->sequence++; } +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_status_lock()); +#else mutex_unlock(&selinux_state.status_lock); +#endif } /* @@ -105,7 +128,11 @@ void selinux_status_update_policyload(u32 seqno) { struct selinux_kernel_status *status; +#ifdef CONFIG_IEE_SELINUX_P + mutex_lock(iee_get_selinux_status_lock()); +#else mutex_lock(&selinux_state.status_lock); +#endif if (selinux_state.status_page) { status = page_address(selinux_state.status_page); @@ -118,5 +145,9 @@ void selinux_status_update_policyload(u32 seqno) smp_wmb(); status->sequence++; } +#ifdef CONFIG_IEE_SELINUX_P + mutex_unlock(iee_get_selinux_status_lock()); +#else mutex_unlock(&selinux_state.status_lock); +#endif } diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index a0b5c658fe4e5..a365679e1d699 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -44,6 +44,9 @@ #include #include #include +#ifdef CONFIG_KEYP +#include +#endif #include "smack.h" #define TRANS_TRUE "TRUE" @@ -4426,7 +4429,11 @@ static int smack_key_alloc(struct key *key, const struct cred *cred, { struct smack_known *skp = smk_of_task(smack_cred(cred)); +#ifdef CONFIG_KEYP + iee_set_key_security(key, skp); +#else key->security = skp; +#endif return 0; } @@ -4438,7 +4445,11 @@ static int smack_key_alloc(struct key *key, const struct cred *cred, */ static void smack_key_free(struct key *key) { +#ifdef CONFIG_KEYP + iee_set_key_security(key, NULL); +#else key->security = NULL; +#endif } /**