diff --git a/arch/x86/lib/copy_user_avx2.S b/arch/x86/lib/copy_user_avx2.S index a2a785aaccb21..ade681a99c984 100644 --- a/arch/x86/lib/copy_user_avx2.S +++ b/arch/x86/lib/copy_user_avx2.S @@ -66,7 +66,7 @@ 303: addl %ecx,%edx/* ecx is zerorest also */ - jmp Lavx2_copy_user_handle_tail + jmp .Lavx2_copy_user_handle_tail _ASM_EXTABLE_UA(300b, 303b) _ASM_EXTABLE_UA(301b, 303b) @@ -81,9 +81,9 @@ SYM_FUNC_START(copy_user_avx2_pf64_nt_string) ASM_STAC ALIGN_DESTINATION_32 - /* if len < 256 jmp to Lless_than_256_bytes_cpy */ + /* if len < 256 jmp to .Lless_than_256_bytes_cpy */ cmpq $256, %rdx - jb Lless_than_256_bytes_cpy + jb .Lless_than_256_bytes_cpy /* * Check if src is aligned, for 32-bit aligned ones, @@ -142,7 +142,7 @@ large_block_nt_aligned_cpy: vzeroupper sfence - jmp Lless_than_256_bytes_cpy + jmp .Lless_than_256_bytes_cpy large_block_nt_unaligned_cpy: PREFETCH(PREFETCH_DISTANCE(%rsi)) @@ -193,11 +193,11 @@ large_block_nt_unaligned_cpy: vzeroupper sfence - jmp Lless_than_256_bytes_cpy + jmp .Lless_than_256_bytes_cpy 88: vzeroupper - jmp Lavx2_copy_user_handle_tail + jmp .Lavx2_copy_user_handle_tail _ASM_EXTABLE_UA(32b, 88b) _ASM_EXTABLE_UA(33b, 88b) @@ -234,9 +234,6 @@ large_block_nt_unaligned_cpy: _ASM_EXTABLE_UA(61b, 88b) _ASM_EXTABLE_UA(62b, 88b) _ASM_EXTABLE_UA(63b, 88b) -SYM_FUNC_END(copy_user_avx2_pf64_nt_string) -EXPORT_SYMBOL(copy_user_avx2_pf64_nt_string) - /* * If len < 256 bytes, then we use rep mov directly. * @@ -248,7 +245,7 @@ EXPORT_SYMBOL(copy_user_avx2_pf64_nt_string) * Output: * eax uncopied bytes or 0 if successful. */ -SYM_CODE_START_LOCAL(Lless_than_256_bytes_cpy) +.Lless_than_256_bytes_cpy: movl %edx, %ecx 90: rep movsb @@ -264,7 +261,6 @@ SYM_CODE_START_LOCAL(Lless_than_256_bytes_cpy) RET _ASM_EXTABLE_UA(90b, 99b) -SYM_CODE_END(Lless_than_256_bytes_cpy) /* * Try to copy last bytes and clear the rest if needed. @@ -280,8 +276,7 @@ SYM_CODE_END(Lless_than_256_bytes_cpy) * Output: * eax uncopied bytes or 0 if successful. */ - -SYM_CODE_START_LOCAL(Lavx2_copy_user_handle_tail) +.Lavx2_copy_user_handle_tail: movq %rdx,%rcx cmp $X86_TRAP_MC,%eax /* check if X86_TRAP_MC */ je 3f @@ -297,7 +292,8 @@ SYM_CODE_START_LOCAL(Lavx2_copy_user_handle_tail) RET _ASM_EXTABLE_UA(1b, 2b) -SYM_CODE_END(Lavx2_copy_user_handle_tail) +SYM_FUNC_END(copy_user_avx2_pf64_nt_string) +EXPORT_SYMBOL(copy_user_avx2_pf64_nt_string) /* * Called when task schedule. we call fpu_save_%ymm0_7 to save old diff --git a/arch/x86/lib/copy_user_sse2.S b/arch/x86/lib/copy_user_sse2.S index 5422ff03ce2e6..9ce7d73f58d96 100644 --- a/arch/x86/lib/copy_user_sse2.S +++ b/arch/x86/lib/copy_user_sse2.S @@ -65,7 +65,7 @@ 203: addl %ecx,%edx/* ecx is zerorest also */ - jmp Lsse2_copy_user_handle_tail + jmp .Lsse2_copy_user_handle_tail _ASM_EXTABLE_UA(200b, 203b) _ASM_EXTABLE_UA(201b, 203b) @@ -157,7 +157,7 @@ SYM_FUNC_START(copy_user_sse2_opt_string) movl %ecx,%edx /* ecx is zerorest also */ 100: sfence - jmp Lsse2_copy_user_handle_tail + jmp .Lsse2_copy_user_handle_tail _ASM_EXTABLE_UA(10b, 100b) _ASM_EXTABLE_UA(11b, 100b) @@ -182,35 +182,6 @@ SYM_FUNC_START(copy_user_sse2_opt_string) _ASM_EXTABLE_UA(29b, 100b) _ASM_EXTABLE_UA(80b, 99b) -SYM_FUNC_END(copy_user_sse2_opt_string) -EXPORT_SYMBOL(copy_user_sse2_opt_string) - -SYM_FUNC_START(fpu_restore_xmm0_3) - ASM_STAC - movdqu (%rsi),%xmm0 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - - xorl %eax,%eax - ASM_CLAC - RET//ret -SYM_FUNC_END(fpu_restore_xmm0_3) -EXPORT_SYMBOL(fpu_restore_xmm0_3) - -SYM_FUNC_START(fpu_save_xmm0_3) - ASM_STAC - - movdqu %xmm0,(%rdi) - movdqu %xmm1,16(%rdi) - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - - xorl %eax,%eax - ASM_CLAC - RET//ret -SYM_FUNC_END(fpu_save_xmm0_3) -EXPORT_SYMBOL(fpu_save_xmm0_3) /* * Try to copy last bytes and clear the rest if needed. @@ -226,7 +197,7 @@ EXPORT_SYMBOL(fpu_save_xmm0_3) * Output: * eax uncopied bytes or 0 if successful. */ -SYM_CODE_START_LOCAL(Lsse2_copy_user_handle_tail) +.Lsse2_copy_user_handle_tail: movq %rdx,%rcx /* * The trap number and error code are both 32 bits. @@ -250,6 +221,34 @@ SYM_CODE_START_LOCAL(Lsse2_copy_user_handle_tail) RET _ASM_EXTABLE_UA(1b, 2b) -SYM_CODE_END(Lsse2_copy_user_handle_tail) +SYM_FUNC_END(copy_user_sse2_opt_string) +EXPORT_SYMBOL(copy_user_sse2_opt_string) + +SYM_FUNC_START(fpu_restore_xmm0_3) + ASM_STAC + movdqu (%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + + xorl %eax,%eax + ASM_CLAC + RET//ret +SYM_FUNC_END(fpu_restore_xmm0_3) +EXPORT_SYMBOL(fpu_restore_xmm0_3) + +SYM_FUNC_START(fpu_save_xmm0_3) + ASM_STAC + + movdqu %xmm0,(%rdi) + movdqu %xmm1,16(%rdi) + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + + xorl %eax,%eax + ASM_CLAC + RET//ret +SYM_FUNC_END(fpu_save_xmm0_3) +EXPORT_SYMBOL(fpu_save_xmm0_3) /*****************************************************************************/