Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 10 additions & 14 deletions arch/x86/lib/copy_user_avx2.S
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@

303:
addl %ecx,%edx/* ecx is zerorest also */
jmp Lavx2_copy_user_handle_tail
jmp .Lavx2_copy_user_handle_tail

_ASM_EXTABLE_UA(300b, 303b)
_ASM_EXTABLE_UA(301b, 303b)
Expand All @@ -81,9 +81,9 @@ SYM_FUNC_START(copy_user_avx2_pf64_nt_string)
ASM_STAC
ALIGN_DESTINATION_32

/* if len < 256 jmp to Lless_than_256_bytes_cpy */
/* if len < 256 jmp to .Lless_than_256_bytes_cpy */
cmpq $256, %rdx
jb Lless_than_256_bytes_cpy
jb .Lless_than_256_bytes_cpy

/*
* Check if src is aligned, for 32-bit aligned ones,
Expand Down Expand Up @@ -142,7 +142,7 @@ large_block_nt_aligned_cpy:

vzeroupper
sfence
jmp Lless_than_256_bytes_cpy
jmp .Lless_than_256_bytes_cpy

large_block_nt_unaligned_cpy:
PREFETCH(PREFETCH_DISTANCE(%rsi))
Expand Down Expand Up @@ -193,11 +193,11 @@ large_block_nt_unaligned_cpy:

vzeroupper
sfence
jmp Lless_than_256_bytes_cpy
jmp .Lless_than_256_bytes_cpy

88:
vzeroupper
jmp Lavx2_copy_user_handle_tail
jmp .Lavx2_copy_user_handle_tail

_ASM_EXTABLE_UA(32b, 88b)
_ASM_EXTABLE_UA(33b, 88b)
Expand Down Expand Up @@ -234,9 +234,6 @@ large_block_nt_unaligned_cpy:
_ASM_EXTABLE_UA(61b, 88b)
_ASM_EXTABLE_UA(62b, 88b)
_ASM_EXTABLE_UA(63b, 88b)
SYM_FUNC_END(copy_user_avx2_pf64_nt_string)
EXPORT_SYMBOL(copy_user_avx2_pf64_nt_string)

/*
* If len < 256 bytes, then we use rep mov directly.
*
Expand All @@ -248,7 +245,7 @@ EXPORT_SYMBOL(copy_user_avx2_pf64_nt_string)
* Output:
* eax uncopied bytes or 0 if successful.
*/
SYM_CODE_START_LOCAL(Lless_than_256_bytes_cpy)
.Lless_than_256_bytes_cpy:
movl %edx, %ecx
90:
rep movsb
Expand All @@ -264,7 +261,6 @@ SYM_CODE_START_LOCAL(Lless_than_256_bytes_cpy)
RET

_ASM_EXTABLE_UA(90b, 99b)
SYM_CODE_END(Lless_than_256_bytes_cpy)

/*
* Try to copy last bytes and clear the rest if needed.
Expand All @@ -280,8 +276,7 @@ SYM_CODE_END(Lless_than_256_bytes_cpy)
* Output:
* eax uncopied bytes or 0 if successful.
*/

SYM_CODE_START_LOCAL(Lavx2_copy_user_handle_tail)
.Lavx2_copy_user_handle_tail:
movq %rdx,%rcx
cmp $X86_TRAP_MC,%eax /* check if X86_TRAP_MC */
je 3f
Expand All @@ -297,7 +292,8 @@ SYM_CODE_START_LOCAL(Lavx2_copy_user_handle_tail)
RET

_ASM_EXTABLE_UA(1b, 2b)
SYM_CODE_END(Lavx2_copy_user_handle_tail)
SYM_FUNC_END(copy_user_avx2_pf64_nt_string)
EXPORT_SYMBOL(copy_user_avx2_pf64_nt_string)

/*
* Called when task schedule. we call fpu_save_%ymm0_7 to save old
Expand Down
65 changes: 32 additions & 33 deletions arch/x86/lib/copy_user_sse2.S
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@

203:
addl %ecx,%edx/* ecx is zerorest also */
jmp Lsse2_copy_user_handle_tail
jmp .Lsse2_copy_user_handle_tail

_ASM_EXTABLE_UA(200b, 203b)
_ASM_EXTABLE_UA(201b, 203b)
Expand Down Expand Up @@ -157,7 +157,7 @@ SYM_FUNC_START(copy_user_sse2_opt_string)
movl %ecx,%edx /* ecx is zerorest also */
100:
sfence
jmp Lsse2_copy_user_handle_tail
jmp .Lsse2_copy_user_handle_tail

_ASM_EXTABLE_UA(10b, 100b)
_ASM_EXTABLE_UA(11b, 100b)
Expand All @@ -182,35 +182,6 @@ SYM_FUNC_START(copy_user_sse2_opt_string)
_ASM_EXTABLE_UA(29b, 100b)

_ASM_EXTABLE_UA(80b, 99b)
SYM_FUNC_END(copy_user_sse2_opt_string)
EXPORT_SYMBOL(copy_user_sse2_opt_string)

SYM_FUNC_START(fpu_restore_xmm0_3)
ASM_STAC
movdqu (%rsi),%xmm0
movdqu 16(%rsi),%xmm1
movdqu 32(%rsi),%xmm2
movdqu 48(%rsi),%xmm3

xorl %eax,%eax
ASM_CLAC
RET//ret
SYM_FUNC_END(fpu_restore_xmm0_3)
EXPORT_SYMBOL(fpu_restore_xmm0_3)

SYM_FUNC_START(fpu_save_xmm0_3)
ASM_STAC

movdqu %xmm0,(%rdi)
movdqu %xmm1,16(%rdi)
movdqu %xmm2,32(%rdi)
movdqu %xmm3,48(%rdi)

xorl %eax,%eax
ASM_CLAC
RET//ret
SYM_FUNC_END(fpu_save_xmm0_3)
EXPORT_SYMBOL(fpu_save_xmm0_3)

/*
* Try to copy last bytes and clear the rest if needed.
Expand All @@ -226,7 +197,7 @@ EXPORT_SYMBOL(fpu_save_xmm0_3)
* Output:
* eax uncopied bytes or 0 if successful.
*/
SYM_CODE_START_LOCAL(Lsse2_copy_user_handle_tail)
.Lsse2_copy_user_handle_tail:
movq %rdx,%rcx
/*
* The trap number and error code are both 32 bits.
Expand All @@ -250,6 +221,34 @@ SYM_CODE_START_LOCAL(Lsse2_copy_user_handle_tail)
RET

_ASM_EXTABLE_UA(1b, 2b)
SYM_CODE_END(Lsse2_copy_user_handle_tail)
SYM_FUNC_END(copy_user_sse2_opt_string)
EXPORT_SYMBOL(copy_user_sse2_opt_string)

SYM_FUNC_START(fpu_restore_xmm0_3)
ASM_STAC
movdqu (%rsi),%xmm0
movdqu 16(%rsi),%xmm1
movdqu 32(%rsi),%xmm2
movdqu 48(%rsi),%xmm3

xorl %eax,%eax
ASM_CLAC
RET//ret
SYM_FUNC_END(fpu_restore_xmm0_3)
EXPORT_SYMBOL(fpu_restore_xmm0_3)

SYM_FUNC_START(fpu_save_xmm0_3)
ASM_STAC

movdqu %xmm0,(%rdi)
movdqu %xmm1,16(%rdi)
movdqu %xmm2,32(%rdi)
movdqu %xmm3,48(%rdi)

xorl %eax,%eax
ASM_CLAC
RET//ret
SYM_FUNC_END(fpu_save_xmm0_3)
EXPORT_SYMBOL(fpu_save_xmm0_3)

/*****************************************************************************/