diff --git a/.gitignore b/.gitignore index be760d1..70d78de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ README.md +.vscode # Prerequisites *.d @@ -32,3 +33,5 @@ README.md *.exe *.out *.app +.claude/ +.opencode/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 04399a5..e0af349 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,48 +8,21 @@ set(CMAKE_MACOSX_RPATH 0) set(LIBCO_VERSION 0.5) # Set cflags -set(CMAKE_C_FLAGS ${CMAKE_C_FLAGS} -g -fno-strict-aliasing -O2 -Wall -export-dynamic -Wall -pipe -D_GNU_SOURCE -D_REENTRANT -fPIC -Wno-deprecated -m64) - +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fno-strict-aliasing -O2 -Wall -export-dynamic -Wall -pipe -D_GNU_SOURCE -D_REENTRANT -fPIC -Wno-deprecated") + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") + message(STATUS "x86_64 architecture detected") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m64 -mtune=generic") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + message(STATUS "ARM64 architecture detected") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a -mtune=generic") +else() + message(STATUS "Unknown architecture: ${CMAKE_SYSTEM_PROCESSOR}") +endif() # Use c and asm enable_language(C ASM) -# Add source files -set(SOURCE_FILES - co_epoll.cpp - co_hook_sys_call.cpp - co_routine.cpp - coctx.cpp - coctx_swap.S) - -# Add static and shared library target -add_library(colib_static STATIC ${SOURCE_FILES}) -add_library(colib_shared SHARED ${SOURCE_FILES}) - -# Set library output name -set_target_properties(colib_static PROPERTIES OUTPUT_NAME colib) -set_target_properties(colib_shared PROPERTIES OUTPUT_NAME colib) - -set_target_properties(colib_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) -set_target_properties(colib_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) - -# Set shared library version, will generate libcolib.${LIBCO_VERSION}.so and a symbol link named libcolib.so -# For mac osx, the extension name will be .dylib -set_target_properties(colib_shared PROPERTIES VERSION ${LIBCO_VERSION} SOVERSION ${LIBCO_VERSION}) - - - -# Macro for add example target -macro(add_example_target EXAMPLE_TARGET) - add_executable("example_${EXAMPLE_TARGET}" "example_${EXAMPLE_TARGET}.cpp") - target_link_libraries("example_${EXAMPLE_TARGET}" colib_static pthread dl) -endmacro(add_example_target) +set(LIBCO_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib") -add_example_target(closure) -add_example_target(cond) -add_example_target(copystack) -add_example_target(echocli) -add_example_target(echosvr) -add_example_target(poll) -add_example_target(setenv) -add_example_target(specific) -add_example_target(thread) +add_subdirectory(lib) +add_subdirectory(examples) \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..8bf8e50 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 2.8) + +macro(add_example_target EXAMPLE_TARGET) + get_filename_component(EXAMPLE_NAME ${EXAMPLE_TARGET} NAME) + if(${EXAMPLE_TARGET} MATCHES "\\.cpp$") + string(REPLACE ".cpp" "" EXAMPLE_NAME ${EXAMPLE_NAME}) + endif() + + add_executable(${EXAMPLE_NAME} ${EXAMPLE_TARGET}) + target_include_directories(${EXAMPLE_NAME} PUBLIC ${LIBCO_INCLUDE_DIR}) + target_link_libraries(${EXAMPLE_NAME} PUBLIC colib_static pthread dl) + + add_test(NAME ${EXAMPLE_NAME} COMMAND ${EXAMPLE_NAME}) +endmacro() + +file(GLOB_RECURSE EXAMPLE_TARGETS "example_*") +foreach(EXAMPLE_TARGET ${EXAMPLE_TARGETS}) + add_example_target(${EXAMPLE_TARGET}) +endforeach() \ No newline at end of file diff --git a/example_closure.cpp b/examples/example_closure.cpp similarity index 100% rename from example_closure.cpp rename to examples/example_closure.cpp diff --git a/example_cond.cpp b/examples/example_cond.cpp similarity index 100% rename from example_cond.cpp rename to examples/example_cond.cpp diff --git a/example_copystack.cpp b/examples/example_copystack.cpp similarity index 100% rename from example_copystack.cpp rename to examples/example_copystack.cpp diff --git a/example_echocli.cpp b/examples/example_echocli.cpp similarity index 100% rename from example_echocli.cpp rename to examples/example_echocli.cpp diff --git a/example_echosvr.cpp b/examples/example_echosvr.cpp similarity index 100% rename from example_echosvr.cpp rename to examples/example_echosvr.cpp diff --git a/example_poll.cpp b/examples/example_poll.cpp similarity index 100% rename from example_poll.cpp rename to examples/example_poll.cpp diff --git a/example_setenv.cpp b/examples/example_setenv.cpp similarity index 100% rename from example_setenv.cpp rename to examples/example_setenv.cpp diff --git a/example_specific.cpp b/examples/example_specific.cpp similarity index 100% rename from example_specific.cpp rename to examples/example_specific.cpp diff --git a/example_thread.cpp b/examples/example_thread.cpp similarity index 100% rename from example_thread.cpp rename to examples/example_thread.cpp diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 0000000..2b6021f --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 2.8) + +# Use c and asm +enable_language(C ASM) + +# Add source files +set(SOURCE_FILES + co_epoll.cpp + co_hook_sys_call.cpp + co_routine.cpp + coctx.cpp + co_comm.cpp + co_memcpy.c + coctx_swap.S) + +# Add static and shared library target +add_library(colib_static STATIC ${SOURCE_FILES}) +add_library(colib_shared SHARED ${SOURCE_FILES}) + +target_compile_definitions(colib_static PRIVATE _GNU_SOURCE _REENTRANT) +target_compile_definitions(colib_shared PRIVATE _GNU_SOURCE _REENTRANT) + +# Set library output name +set_target_properties(colib_static PROPERTIES OUTPUT_NAME colib) +set_target_properties(colib_shared PROPERTIES OUTPUT_NAME colib) + +set_target_properties(colib_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) +set_target_properties(colib_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) + +# Set shared library version, will generate libcolib.${LIBCO_VERSION}.so and a symbol link named libcolib.so +# For mac osx, the extension name will be .dylib +set_target_properties(colib_shared PROPERTIES VERSION ${LIBCO_VERSION} SOVERSION ${LIBCO_VERSION}) \ No newline at end of file diff --git a/co_closure.h b/lib/co_closure.h similarity index 97% rename from co_closure.h rename to lib/co_closure.h index dd39841..d67abc6 100644 --- a/co_closure.h +++ b/lib/co_closure.h @@ -18,6 +18,11 @@ #ifndef __CO_CLOSURE_H__ #define __CO_CLOSURE_H__ + +#ifdef __cplusplus +extern "C" { +#endif + struct stCoClosure_t { public: @@ -92,6 +97,8 @@ public:\ #define co_func_end } - +#ifdef __cplusplus +} #endif +#endif \ No newline at end of file diff --git a/co_comm.cpp b/lib/co_comm.cpp similarity index 100% rename from co_comm.cpp rename to lib/co_comm.cpp diff --git a/co_comm.h b/lib/co_comm.h similarity index 100% rename from co_comm.h rename to lib/co_comm.h diff --git a/co_epoll.cpp b/lib/co_epoll.cpp similarity index 100% rename from co_epoll.cpp rename to lib/co_epoll.cpp diff --git a/co_epoll.h b/lib/co_epoll.h similarity index 100% rename from co_epoll.h rename to lib/co_epoll.h diff --git a/co_hook_sys_call.cpp b/lib/co_hook_sys_call.cpp similarity index 98% rename from co_hook_sys_call.cpp rename to lib/co_hook_sys_call.cpp index d41e4d4..7c4d124 100644 --- a/co_hook_sys_call.cpp +++ b/lib/co_hook_sys_call.cpp @@ -98,7 +98,6 @@ typedef int (*unsetenv_pfn_t)(const char *name); typedef char *(*getenv_pfn_t)(const char *name); typedef hostent* (*gethostbyname_pfn_t)(const char *name); typedef res_state (*__res_state_pfn_t)(); -typedef int (*__poll_pfn_t)(struct pollfd fds[], nfds_t nfds, int timeout); typedef int (*gethostbyname_r_pfn_t)(const char* __restrict name, struct hostent* __restrict __result_buf, char* __restrict __buf, size_t __buflen, struct hostent** __restrict __result, int* __restrict __h_errnop); static socket_pfn_t g_sys_socket_func = (socket_pfn_t)dlsym(RTLD_NEXT,"socket"); @@ -128,9 +127,6 @@ static __res_state_pfn_t g_sys___res_state_func = (__res_state_pfn_t)dlsym(RTLD static gethostbyname_pfn_t g_sys_gethostbyname_func = (gethostbyname_pfn_t)dlsym(RTLD_NEXT, "gethostbyname"); static gethostbyname_r_pfn_t g_sys_gethostbyname_r_func = (gethostbyname_r_pfn_t)dlsym(RTLD_NEXT, "gethostbyname_r"); -static __poll_pfn_t g_sys___poll_func = (__poll_pfn_t)dlsym(RTLD_NEXT, "__poll"); - - /* static pthread_getspecific_pfn_t g_sys_pthread_getspecific_func = (pthread_getspecific_pfn_t)dlsym(RTLD_NEXT,"pthread_getspecific"); @@ -970,10 +966,6 @@ extern "C" return &(__co_state_wrap->state); } - int __poll(struct pollfd fds[], nfds_t nfds, int timeout) - { - return poll(fds, nfds, timeout); - } } struct hostbuf_wrap @@ -1035,5 +1027,4 @@ void co_enable_hook_sys() //这函数必须在这里,否则本文件会被忽略 { co->cEnableSysHook = 1; } -} - +} \ No newline at end of file diff --git a/lib/co_memcpy.c b/lib/co_memcpy.c new file mode 100644 index 0000000..d00cacc --- /dev/null +++ b/lib/co_memcpy.c @@ -0,0 +1,138 @@ +#include "co_memcpy.h" +#include +#include "memcpy_def.h" +#include "memcpy_neon_impl.h" + +#define SIZE_128B 128 +#define SIZE_1KB 1024 +#define SIZE_2KB 2048 +#define SIZE_4KB 4096 +#define SIZE_8KB 8192 +#define SIZE_12KB 12288 + +static void *co_memcpy_save_aarch64(void *restrict dst, const void *restrict src, unsigned long len) +{ + void *dst_ret = dst; +#ifdef __aarch64__ + if (len < SIZE_128B) { + memcpy_rem(dst, src, len); + return dst_ret; + } + + unsigned long align_offset = (unsigned long)dst & 0x3f; + if (align_offset) { + memcpy_rem(dst, src, align_offset); + dst += align_offset; + src += align_offset; + len -= align_offset; + } + + int len_count = len / BLOCK_SIZE; + int rem = len % BLOCK_SIZE; + if (len <= SIZE_1KB) { + memcpy_blk(dst, src, len_count, memcpy_blk_save); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_2KB) { + memcpy_256B_save(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_4KB) { + memcpy_384B_save(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_8KB) { + memcpy_512B_save(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_12KB) { + memcpy_1024B_save(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else { + memcpy_1536B_save(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } +#endif + return dst_ret; +} + +static void *co_memcpy_swap_aarch64(void *restrict dst, const void *restrict src, unsigned long len) +{ + void *dst_ret = dst; +#ifdef __aarch64__ + if (len < SIZE_128B) { + memcpy_rem(dst, src, len); + return dst_ret; + } + + unsigned long align_offset = (unsigned long)dst & 0x3f; + if (align_offset) { + memcpy_rem(dst, src, align_offset); + dst += align_offset; + src += align_offset; + len -= align_offset; + } + + int len_count = len / BLOCK_SIZE; + int rem = len % BLOCK_SIZE; + if (len <= SIZE_1KB) { + memcpy_blk(dst, src, len_count, memcpy_blk_swap); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_2KB) { + memcpy_384B_swap(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_8KB) { + memcpy_512B_swap(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else if (len <= SIZE_12KB) { + memcpy_1024B_swap(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } else { + memcpy_1536B_swap(dst, src, len_count, rem); + if (rem) { + memcpy_rem(dst, src, rem); + } + } +#endif + return dst_ret; +} + +typedef void *(*memcpy_func_name)(void *restrict dst, const void *restrict src, unsigned long len); + +static memcpy_func_name memcpy_save_func = (memcpy_func_name)memcpy; +static memcpy_func_name memcpy_swap_func = (memcpy_func_name)memcpy; + +static void __attribute__((constructor)) model_init(void) +{ +#ifdef __aarch64__ + memcpy_save_func = (memcpy_func_name)co_memcpy_save_aarch64; + memcpy_swap_func = (memcpy_func_name)co_memcpy_swap_aarch64; +#endif + return; +} + +void *co_memcpy_save(void *restrict dst, const void *restrict src, unsigned long len) +{ + return memcpy_save_func(dst, src, len); +} + +void *co_memcpy_swap(void *restrict dst, const void *restrict src, unsigned long len) +{ + return memcpy_swap_func(dst, src, len); +} diff --git a/lib/co_memcpy.h b/lib/co_memcpy.h new file mode 100644 index 0000000..cdbf2b4 --- /dev/null +++ b/lib/co_memcpy.h @@ -0,0 +1,15 @@ +#ifndef CO_MEMCPY_H +#define CO_MEMCPY_H + +#ifdef __cplusplus +extern "C" { +#endif + +void *co_memcpy_save(void *__restrict dst, const void *__restrict src, unsigned long len); +void *co_memcpy_swap(void *__restrict dst, const void *__restrict src, unsigned long len); + +#ifdef __cplusplus +} +#endif + +#endif /* CO_MEMCPY_H */ \ No newline at end of file diff --git a/co_routine.cpp b/lib/co_routine.cpp similarity index 93% rename from co_routine.cpp rename to lib/co_routine.cpp index 352ae7e..308aa22 100644 --- a/co_routine.cpp +++ b/lib/co_routine.cpp @@ -40,9 +40,16 @@ #include #include +#ifdef __aarch64__ +#include "co_memcpy.h" +#endif + extern "C" { extern void coctx_swap( coctx_t *,coctx_t* ) asm("coctx_swap"); +#ifdef __aarch64__ + extern void get_sp(void*) asm("get_sp"); +#endif }; using namespace std; stCoRoutine_t *GetCurrCo( stCoRoutineEnv_t *env ); @@ -278,6 +285,13 @@ stStackMem_t* co_alloc_stackmem(unsigned int stack_size) stShareStack_t* co_alloc_sharestack(int count, int stack_size) { + int default_stack_size = 128 * 1024; + if (stack_size <= 0) { + stack_size = default_stack_size; + } + if (count <= 0) { + count = 1; + } stShareStack_t* share_stack = (stShareStack_t*)malloc(sizeof(stShareStack_t)); share_stack->alloc_idx = 0; share_stack->stack_size = stack_size; @@ -293,6 +307,25 @@ stShareStack_t* co_alloc_sharestack(int count, int stack_size) return share_stack; } +void co_free_sharestack(stShareStack_t* share_stack) { + if (!share_stack) { + return; + } + for (int i = 0; i < share_stack->count; i++) + { + if (share_stack->stack_array[i] == nullptr) { + continue; + } + free(share_stack->stack_array[i]->stack_buffer); + share_stack->stack_array[i]->stack_buffer = nullptr; + free(share_stack->stack_array[i]); + share_stack->stack_array[i] = nullptr; + } + free(share_stack->stack_array); + share_stack->stack_array = nullptr; + free(share_stack); +} + static stStackMem_t* co_get_stackmem(stShareStack_t* share_stack) { if (!share_stack) @@ -373,7 +406,9 @@ stTimeout_t *AllocTimeout( int iSize ) void FreeTimeout( stTimeout_t *apTimeout ) { free( apTimeout->pItems ); + apTimeout->pItems = nullptr; free ( apTimeout ); + apTimeout = nullptr; } int AddTimeout( stTimeout_t *apTimeout,stTimeoutItem_t *apItem ,unsigned long long allNow ) { @@ -533,15 +568,18 @@ void co_free( stCoRoutine_t *co ) if (!co->cIsShareStack) { free(co->stack_mem->stack_buffer); + co->stack_mem->stack_buffer = nullptr; free(co->stack_mem); + co->stack_mem = nullptr; } //walkerdu fix at 2018-01-20 //存在内存泄漏 else { - if(co->save_buffer) + if (co->save_buffer) { free(co->save_buffer); - + co->save_buffer = nullptr; + } if(co->stack_mem->occupy_co == co) co->stack_mem->occupy_co = NULL; } @@ -629,16 +667,27 @@ void save_stack_buffer(stCoRoutine_t* occupy_co) occupy_co->save_buffer = (char*)malloc(len); //malloc buf; occupy_co->save_size = len; +#ifdef __aarch64__ + co_memcpy_save(occupy_co->save_buffer, occupy_co->stack_sp, len); +#else memcpy(occupy_co->save_buffer, occupy_co->stack_sp, len); +#endif } void co_swap(stCoRoutine_t* curr, stCoRoutine_t* pending_co) { stCoRoutineEnv_t* env = co_get_curr_thread_env(); - //get curr stack sp + // get a stack pointer indicating the place from + // which to save the data brfore context swipping. +#ifdef __aarch64__ + void* c{nullptr}; + get_sp(&c); + curr->stack_sp = (char*)c; +#else char c; curr->stack_sp= &c; +#endif if (!pending_co->cIsShareStack) { @@ -673,7 +722,11 @@ void co_swap(stCoRoutine_t* curr, stCoRoutine_t* pending_co) //resume stack buffer if (update_pending_co->save_buffer && update_pending_co->save_size > 0) { +#ifdef __aarch64__ + co_memcpy_swap(update_pending_co->stack_sp, update_pending_co->save_buffer, update_pending_co->save_size); +#else memcpy(update_pending_co->stack_sp, update_pending_co->save_buffer, update_pending_co->save_size); +#endif } } } @@ -739,6 +792,22 @@ static short EpollEvent2Poll( uint32_t events ) static __thread stCoRoutineEnv_t* gCoEnvPerThread = NULL; +void co_destruct_curr_thread_env() { + if(!co_get_curr_thread_env()) { + return; + } + + struct stCoRoutine_t *self = gCoEnvPerThread->pCallStack[0]; + if (self == nullptr) { + return; + } + + co_free(self); + FreeEpoll(gCoEnvPerThread->pEpoll); + free(gCoEnvPerThread); + gCoEnvPerThread = nullptr; +} + void co_init_curr_thread_env() { gCoEnvPerThread = (stCoRoutineEnv_t*)calloc( 1, sizeof(stCoRoutineEnv_t) ); @@ -893,7 +962,9 @@ void FreeEpoll( stCoEpoll_t *ctx ) if( ctx ) { free( ctx->pstActiveList ); + ctx->pstActiveList = nullptr; free( ctx->pstTimeoutList ); + ctx->pstTimeoutList = nullptr; FreeTimeout( ctx->pTimeout ); co_epoll_res_free( ctx->result ); } @@ -1193,4 +1264,4 @@ stCoCondItem_t *co_cond_pop( stCoCond_t *link ) PopHead( link ); } return p; -} +} \ No newline at end of file diff --git a/co_routine.h b/lib/co_routine.h similarity index 93% rename from co_routine.h rename to lib/co_routine.h index d6f4789..ae415b3 100644 --- a/co_routine.h +++ b/lib/co_routine.h @@ -23,6 +23,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + //1.struct struct stCoRoutine_t; @@ -51,6 +55,7 @@ void co_yield( stCoRoutine_t *co ); void co_yield_ct(); //ct = current thread void co_release( stCoRoutine_t *co ); void co_reset(stCoRoutine_t * co); +void co_destruct_curr_thread_env(); stCoRoutine_t *co_self(); @@ -84,10 +89,16 @@ int co_cond_timedwait( stCoCond_t *,int timeout_ms ); //7.share stack stShareStack_t* co_alloc_sharestack(int iCount, int iStackSize); +void co_free_sharestack(stShareStack_t* share_stack); //8.init envlist for hook get/set env void co_set_env_list( const char *name[],size_t cnt); void co_log_err( const char *fmt,... ); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/co_routine_inner.h b/lib/co_routine_inner.h similarity index 100% rename from co_routine_inner.h rename to lib/co_routine_inner.h diff --git a/co_routine_specific.h b/lib/co_routine_specific.h similarity index 97% rename from co_routine_specific.h rename to lib/co_routine_specific.h index 1b451ca..a32b576 100644 --- a/co_routine_specific.h +++ b/lib/co_routine_specific.h @@ -20,6 +20,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + /* invoke only once in the whole program CoRoutineSetSpecificCallBack(CoRoutineGetSpecificFunc_t pfnGet,CoRoutineSetSpecificFunc_t pfnSet) @@ -84,3 +88,6 @@ public:\ \ static clsRoutineData_routine_##name y; +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/coctx.cpp b/lib/coctx.cpp similarity index 67% rename from coctx.cpp rename to lib/coctx.cpp index d5eeed1..7f5655a 100644 --- a/coctx.cpp +++ b/lib/coctx.cpp @@ -81,15 +81,57 @@ enum { kRSP = 13, }; +//------------- // 64 bit -extern "C" { -extern void coctx_swap(coctx_t*, coctx_t*) asm("coctx_swap"); +// low | regs[0] : x0 | +// | regs[1] : x1 | +// | regs[2] : x2 | +// | regs[3] : x3 | +// | regs[4] : x4 | +// | regs[5] : x5 | +// | regs[6] : x6 | +// | regs[7] : x7 | +// | regs[8] : x8 | +// | regs[9] : x9 | +// | regs[10]: x10 | +// | regs[11]: x11 | +// | regs[12]: x12 | +// | regs[13]: x13 | +// | regs[14]: x14 | +// | regs[15]: x15 | +// | regs[16]: x16 | +// | regs[17]: x17 | +// | regs[18]: x18 | +// | regs[19]: x19 | +// | regs[20]: x20 | +// | regs[21]: x21 | +// | regs[22]: x22 | +// | regs[23]: x23 | +// | regs[24]: x24 | +// | regs[25]: x25 | +// | regs[26]: x26 | +// | regs[27]: x27 | +// | regs[28]: x28 | +// | regs[29]: x29 | +// | regs[30]: x30 | // ret func addr +// hig | regs[31]: sp | +enum { + kARG1 = 0, + kARG2 = 1, + kRETAddr_ARM64 = 30, + kSP_ARM64 = 31, }; -#if defined(__i386__) + int coctx_init(coctx_t* ctx) { memset(ctx, 0, sizeof(*ctx)); return 0; } + +// 64 bit +extern "C" { +extern void coctx_swap(coctx_t*, coctx_t*) asm("coctx_swap"); +}; +#if defined(__i386__) int coctx_make(coctx_t* ctx, coctx_pfn_t pfn, const void* s, const void* s1) { // make room for coctx_param char* sp = ctx->ss_sp + ctx->ss_size - sizeof(coctx_param_t); @@ -123,9 +165,19 @@ int coctx_make(coctx_t* ctx, coctx_pfn_t pfn, const void* s, const void* s1) { ctx->regs[kRSI] = (char*)s1; return 0; } +#elif defined(__aarch64__) +int coctx_make(coctx_t* ctx, coctx_pfn_t pfn, const void* s, const void* s1) { + char* sp = ctx->ss_sp + ctx->ss_size - sizeof(void*); + sp = (char*)((unsigned long)sp & -16LL); -int coctx_init(coctx_t* ctx) { - memset(ctx, 0, sizeof(*ctx)); + memset(ctx->regs, 0, sizeof(ctx->regs)); + void** ret_addr = (void**)(sp); + *ret_addr = (void*)pfn; + + ctx->regs[kSP_ARM64] = sp; + ctx->regs[kRETAddr_ARM64] = (char*)pfn; + ctx->regs[kARG1] = (char*)s; + ctx->regs[kARG2] = (char*)s1; return 0; } diff --git a/coctx.h b/lib/coctx.h similarity index 94% rename from coctx.h rename to lib/coctx.h index c1fdfa9..7d01be1 100644 --- a/coctx.h +++ b/lib/coctx.h @@ -29,8 +29,10 @@ struct coctx_t { #if defined(__i386__) void *regs[ 8 ]; -#else +#elif defined(__x86_64__) void *regs[ 14 ]; +#elif defined(__aarch64__) + void *regs[ 32 ]; #endif size_t ss_size; char *ss_sp; diff --git a/coctx_swap.S b/lib/coctx_swap.S similarity index 65% rename from coctx_swap.S rename to lib/coctx_swap.S index 0e4ce1c..2cfd7a4 100644 --- a/coctx_swap.S +++ b/lib/coctx_swap.S @@ -80,4 +80,50 @@ coctx_swap: movq 64(%rsi), %rsi ret + +#elif defined(__aarch64__) + stp x0, x1, [x0] + stp x2, x3, [x0, #16] + stp x4, x5, [x0, #32] + stp x6, x7, [x0, #48] + stp x8, x9, [x0, #64] + stp x10, x11, [x0, #80] + stp x12, x13, [x0, #96] + stp x14, x15, [x0, #112] + stp x16, x17, [x0, #128] + stp x18, x19, [x0, #144] + stp x20, x21, [x0, #160] + stp x22, x23, [x0, #176] + stp x24, x25, [x0, #192] + stp x26, x27, [x0, #208] + stp x28, x29, [x0, #224] + mov x2, sp + stp x30, x2, [x0, #240] + + ldp x2, x3, [x1, #16] + ldp x4, x5, [x1, #32] + ldp x6, x7, [x1, #48] + ldp x8, x9, [x1, #64] + ldp x10, x11, [x1, #80] + ldp x12, x13, [x1, #96] + ldp x14, x15, [x1, #112] + ldp x16, x17, [x1, #128] + ldp x18, x19, [x1, #144] + ldp x20, x21, [x1, #160] + ldp x22, x23, [x1, #176] + ldp x24, x25, [x1, #192] + ldp x26, x27, [x1, #208] + ldp x28, x29, [x1, #224] + ldp x30, x0, [x1, #240] + mov sp, x0 + ldp x0, x1, [x1] + + ret + +.globl get_sp +get_sp: + mov x1, sp + str x1, [x0] + ret + #endif diff --git a/lib/memcpy_def.h b/lib/memcpy_def.h new file mode 100644 index 0000000..7b413be --- /dev/null +++ b/lib/memcpy_def.h @@ -0,0 +1,101 @@ +#ifndef MEMCPY_DEF_H +#define MEMCPY_DEF_H + +#define CACHE_LINE 64 +#define BLOCK_SIZE (2 * CACHE_LINE) + +#define memcpy_blk(temp_dst, temp_src, len_count, memcpy_func) \ + do \ + { \ + for (int i = 0; i < (len_count); i++) \ + { \ + memcpy_func(temp_dst, temp_src); \ + temp_dst += BLOCK_SIZE; \ + temp_src += BLOCK_SIZE; \ + } \ + } while (0) + +#define memcpy_rem(dst, src, len) \ + do \ + { \ + asm volatile("add x4, %1, %2\n" \ + "add x5, %0, %2\n" \ + "cmp %2, #32\n" \ + "b.hi .copy32_127%=\n" \ + "cmp %2, #16\n" \ + "b.lo .copy16%=\n" \ + "ldr q0, [%1]\n" \ + "ldr q1, [x4, #-16]\n" \ + "str q0, [%0]\n" \ + "str q1, [x5, #-16]\n" \ + "b .exit%=\n" \ + ".copy16%=: \n" \ + "tbz %2, #3, .copy8%=\n" \ + "ldr x6, [%1]\n" \ + "ldr x7, [x4, #-8]\n" \ + "str x6, [%0]\n" \ + "str x7, [x5, #-8]\n" \ + "b .exit%=\n" \ + ".copy8%=: \n" \ + "tbz %2, #2, .copy4%=\n" \ + "ldr w6, [%1]\n" \ + "ldr w8, [x4, #-4]\n" \ + "str w6, [%0]\n" \ + "str w8, [x5, #-4]\n" \ + "b .exit%=\n" \ + ".copy4%=: \n" \ + "cbz %2, .copy0%=\n" \ + "lsr x14, %2, #1\n" \ + "ldrb w6, [%1]\n" \ + "ldrb w10, [x4, -1]\n" \ + "ldrb w8, [%1, x14]\n" \ + "strb w6, [%0]\n" \ + "strb w8, [%0, x14]\n" \ + "strb w10, [x5, -1]\n" \ + ".copy0%=: \n" \ + "b .exit%=\n" \ + ".p2align 4\n" \ + ".copy32_127%=: \n" \ + "ldp q0, q1, [%1]\n" \ + "ldp q2, q3, [x4, #-32]\n" \ + "cmp %2, #64\n" \ + "b.hi .copy127%=\n" \ + "stp q0, q1, [%0]\n" \ + "stp q2, q3, [x5, #-32]\n" \ + "b .exit%=\n" \ + ".p2align 4\n" \ + ".copy127%=: \n" \ + "ldp q4, q5, [%1, #32]\n" \ + "cmp %2, #96\n" \ + "b.ls .copy96%=\n" \ + "ldp q6, q7, [x4, #-64]\n" \ + "stp q6, q7, [x5, #-64]\n" \ + ".copy96%=: \n" \ + "stp q0, q1, [%0]\n" \ + "stp q4, q5, [%0, #32]\n" \ + "stp q2, q3, [x5, #-32]\n" \ + ".exit%=: \n" \ + :: "r"(dst), "r"(src), "r"(len) \ + : "x4", "x5", "x6", "x7", "x14", "q0", "q1", "q2", "q3", \ + "q4", "q5", "q6", "q7", "w6", "w8", "w10"); \ + } while (0) + +#define memcpy_blk_save(dst, src) memcpy_blk_neon(dst, src, "ldp", "stnp") +#define memcpy_blk_swap(dst, src) memcpy_blk_neon(dst, src, "ldnp", "stp") + +#define memcpy_256B_save(dst, src, len, rem) memcpy_256B_neon(dst, src, len, rem, "ldp", "stnp") +#define memcpy_256B_swap(dst, src, len, rem) memcpy_256B_neon(dst, src, len, rem, "ldnp", "stp") + +#define memcpy_384B_save(dst, src, len, rem) memcpy_384B_neon(dst, src, len, rem, "ldp", "stnp") +#define memcpy_384B_swap(dst, src, len, rem) memcpy_384B_neon(dst, src, len, rem, "ldnp", "stp") + +#define memcpy_512B_save(dst, src, len, rem) memcpy_512B_neon(dst, src, len, rem, "ldp", "stnp") +#define memcpy_512B_swap(dst, src, len, rem) memcpy_512B_neon(dst, src, len, rem, "ldnp", "stp") + +#define memcpy_1024B_save(dst, src, len, rem) memcpy_1024B_neon(dst, src, len, rem, "ldp", "stnp") +#define memcpy_1024B_swap(dst, src, len, rem) memcpy_1024B_neon(dst, src, len, rem, "ldnp", "stp") + +#define memcpy_1536B_save(dst, src, len, rem) memcpy_1536B_neon(dst, src, len, rem, "ldp", "stnp") +#define memcpy_1536B_swap(dst, src, len, rem) memcpy_1536B_neon(dst, src, len, rem, "ldnp", "stp") + +#endif /* MEMCPY_DEF_H */ \ No newline at end of file diff --git a/lib/memcpy_neon_impl.h b/lib/memcpy_neon_impl.h new file mode 100644 index 0000000..0868b55 --- /dev/null +++ b/lib/memcpy_neon_impl.h @@ -0,0 +1,463 @@ +#ifndef MEMCPY_NEON_IMPL_H +#define MEMCPY_NEON_IMPL_H + +#ifdef __aarch64__ +#include +#ifdef __cplusplus +extern "C" +{ +#endif + +#define memcpy_blk_neon(dst, src, ld_ins, st_ins) \ + do \ + { \ + asm volatile( \ + ld_ins " q0, q1, [%0]\n" \ + ld_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q4, q5, [%0, #64]\n" \ + ld_ins " q6, q7, [%0, #96]\n" \ + st_ins " q0, q1, [%1]\n" \ + st_ins " q2, q3, [%1, #32]\n" \ + st_ins " q4, q5, [%1, #64]\n" \ + st_ins " q6, q7, [%1, #96]\n" \ + : \ + : "r"(src), "r"(dst) \ + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7"); \ + } while (0) + +#define memcpy_384B_neon(dst, src, len, rem, ld_ins, st_ins) \ + do \ + { \ + asm volatile( \ + "0:\n" \ + "cmp %2, #3\n" \ + "b.lt 1f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q20, q21, [%1, #192]\n" \ + ld_ins " q22, q23, [%1, #224]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + ld_ins " q24, q25, [%1, #256]\n" \ + ld_ins " q26, q27, [%1, #288]\n" \ + st_ins " q20, q21, [%0, #192]\n" \ + st_ins " q22, q23, [%0, #224]\n" \ + ld_ins " q28, q29, [%1, #320]\n" \ + ld_ins " q30, q31, [%1, #352]\n" \ + st_ins " q24, q25, [%0, #256]\n" \ + st_ins " q26, q27, [%0, #288]\n" \ + "sub %2, %2, #3\n" \ + st_ins " q28, q29, [%0, #320]\n" \ + st_ins " q30, q31, [%0, #352]\n" \ + "add %1, %1, #384\n" \ + "add %0, %0, #384\n" \ + "b 0b\n" \ + "1:\n" \ + "cmp %2, #1\n" \ + "b.lt 2f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + "sub %2, %2, #1\n" \ + "add %1, %1, #128\n" \ + "add %0, %0, #128\n" \ + "b 1b\n" \ + "2:\n" \ + "cbz %3, 5f\n" \ + "3:\n" \ + "cmp %3, #32\n" \ + "b.lt 5f\n" \ + ld_ins " q0, q1, [%1]\n" \ + st_ins " q0, q1, [%0]\n" \ + "add %1, %1, #32\n" \ + "add %0, %0, #32\n" \ + "subs %3, %3, #32\n" \ + "b.ge 3b\n" \ + "5:\n" \ + : "+r"(dst), "+r"(src), "+r"(len), "+r"(rem) \ + : \ + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \ + "q16", "q17", "q18", "q19", "q20", "q21", "q22", "q23", \ + "q24", "q25", "q26", "q27", "q28", "q29", "q30", "q31"); \ + } while (0) + +#define memcpy_256B_neon(dst, src, len, rem, ld_ins, st_ins) \ + do \ + { \ + asm volatile( \ + "0:\n" \ + "cmp %2, #2\n" \ + "b.lt 1f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q0, q1, [%1, #128]\n" \ + ld_ins " q2, q3, [%1, #160]\n" \ + ld_ins " q4, q5, [%1, #192]\n" \ + ld_ins " q6, q7, [%1, #224]\n" \ + st_ins " q0, q1, [%0, #128]\n" \ + st_ins " q2, q3, [%0, #160]\n" \ + st_ins " q4, q5, [%0, #192]\n" \ + st_ins " q6, q7, [%0, #224]\n" \ + "sub %2, %2, #2\n" \ + "add %1, %1, #256\n" \ + "add %0, %0, #256\n" \ + "b 0b\n" \ + "1:\n" \ + "cmp %2, #1\n" \ + "b.lt 2f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + "sub %2, %2, #1\n" \ + "add %1, %1, #128\n" \ + "add %0, %0, #128\n" \ + "b 1b\n" \ + "2:\n" \ + "cbz %3, 5f\n" \ + "3:\n" \ + "cmp %3, #32\n" \ + "b.lt 5f\n" \ + ld_ins " q0, q1, [%1]\n" \ + st_ins " q0, q1, [%0]\n" \ + "add %1, %1, #32\n" \ + "add %0, %0, #32\n" \ + "subs %3, %3, #32\n" \ + "b.ge 3b\n" \ + "5:\n" \ + : "+r"(dst), "+r"(src), "+r"(len), "+r"(rem) \ + : \ + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7"); \ + } while (0) + +#define memcpy_512B_neon(dst, src, len, rem, ld_ins, st_ins) \ + do \ + { \ + asm volatile( \ + "0:\n" \ + "cmp %2, #4\n" \ + "b.lt 1f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q20, q21, [%1, #192]\n" \ + ld_ins " q22, q23, [%1, #224]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + ld_ins " q0, q1, [%1, #256]\n" \ + ld_ins " q2, q3, [%1, #288]\n" \ + st_ins " q20, q21, [%0, #192]\n" \ + st_ins " q22, q23, [%0, #224]\n" \ + ld_ins " q4, q5, [%1, #320]\n" \ + ld_ins " q6, q7, [%1, #352]\n" \ + st_ins " q0, q1, [%0, #256]\n" \ + st_ins " q2, q3, [%0, #288]\n" \ + st_ins " q4, q5, [%0, #320]\n" \ + st_ins " q6, q7, [%0, #352]\n" \ + "sub %2, %2, #4\n" \ + "add %1, %1, #512\n" \ + "add %0, %0, #512\n" \ + "b 0b\n" \ + "1:\n" \ + "cmp %2, #1\n" \ + "b.lt 2f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + "sub %2, %2, #1\n" \ + "add %1, %1, #128\n" \ + "add %0, %0, #128\n" \ + "b 1b\n" \ + "2:\n" \ + "cbz %3, 5f\n" \ + "3:\n" \ + "cmp %3, #32\n" \ + "b.lt 5f\n" \ + ld_ins " q0, q1, [%1]\n" \ + st_ins " q0, q1, [%0]\n" \ + "add %1, %1, #32\n" \ + "add %0, %0, #32\n" \ + "subs %3, %3, #32\n" \ + "b.ge 3b\n" \ + "5:\n" \ + : "+r"(dst), "+r"(src), "+r"(len), "+r"(rem) \ + : \ + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \ + "q16", "q17", "q18", "q19", "q20", "q21", "q22", "q23"); \ + } while (0) + +#define memcpy_1024B_neon(dst, src, len, rem, ld_ins, st_ins) \ + do \ + { \ + asm volatile( \ + "0:\n" \ + "cmp %2, #8\n" \ + "b.lt 1f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q20, q21, [%1, #192]\n" \ + ld_ins " q22, q23, [%1, #224]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + ld_ins " q0, q1, [%1, #256]\n" \ + ld_ins " q2, q3, [%1, #288]\n" \ + st_ins " q20, q21, [%0, #192]\n" \ + st_ins " q22, q23, [%0, #224]\n" \ + ld_ins " q4, q5, [%1, #320]\n" \ + ld_ins " q6, q7, [%1, #352]\n" \ + st_ins " q0, q1, [%0, #256]\n" \ + st_ins " q2, q3, [%0, #288]\n" \ + ld_ins " q16, q17, [%1, #384]\n" \ + ld_ins " q18, q19, [%1, #416]\n" \ + st_ins " q4, q5, [%0, #320]\n" \ + st_ins " q6, q7, [%0, #352]\n" \ + ld_ins " q20, q21, [%1, #448]\n" \ + ld_ins " q22, q23, [%1, #480]\n" \ + st_ins " q16, q17, [%0, #384]\n" \ + st_ins " q18, q19, [%0, #416]\n" \ + st_ins " q20, q21, [%0, #448]\n" \ + st_ins " q22, q23, [%0, #480]\n" \ + "sub %2, %2, #8\n" \ + "add %1, %1, #1024\n" \ + "add %0, %0, #1024\n" \ + "b 0b\n" \ + "1:\n" \ + "cmp %2, #4\n" \ + "b.lt 2f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q20, q21, [%1, #192]\n" \ + ld_ins " q22, q23, [%1, #224]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + st_ins " q20, q21, [%0, #192]\n" \ + st_ins " q22, q23, [%0, #224]\n" \ + "sub %2, %2, #4\n" \ + "add %1, %1, #512\n" \ + "add %0, %0, #512\n" \ + "b 1b\n" \ + "2:\n" \ + "cmp %2, #1\n" \ + "b.lt 3f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + "sub %2, %2, #1\n" \ + "add %1, %1, #128\n" \ + "add %0, %0, #128\n" \ + "b 2b\n" \ + "3:\n" \ + "cbz %3, 5f\n" \ + "4:\n" \ + "cmp %3, #32\n" \ + "b.lt 5f\n" \ + ld_ins " q0, q1, [%1]\n" \ + st_ins " q0, q1, [%0]\n" \ + "add %1, %1, #32\n" \ + "add %0, %0, #32\n" \ + "subs %3, %3, #32\n" \ + "b.ge 4b\n" \ + "5:\n" \ + : "+r"(dst), "+r"(src), "+r"(len), "+r"(rem) \ + : \ + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \ + "q16", "q17", "q18", "q19", "q20", "q21", "q22", "q23"); \ + } while (0) + +#define memcpy_1536B_neon(dst, src, len, rem, ld_ins, st_ins) \ + do \ + { \ + asm volatile( \ + "0:\n" \ + "cmp %2, #12\n" \ + "b.lt 1f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q20, q21, [%1, #192]\n" \ + ld_ins " q22, q23, [%1, #224]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + ld_ins " q0, q1, [%1, #256]\n" \ + ld_ins " q2, q3, [%1, #288]\n" \ + st_ins " q20, q21, [%0, #192]\n" \ + st_ins " q22, q23, [%0, #224]\n" \ + ld_ins " q4, q5, [%1, #320]\n" \ + ld_ins " q6, q7, [%1, #352]\n" \ + st_ins " q0, q1, [%0, #256]\n" \ + st_ins " q2, q3, [%0, #288]\n" \ + ld_ins " q16, q17, [%1, #384]\n" \ + ld_ins " q18, q19, [%1, #416]\n" \ + st_ins " q4, q5, [%0, #320]\n" \ + st_ins " q6, q7, [%0, #352]\n" \ + ld_ins " q20, q21, [%1, #448]\n" \ + ld_ins " q22, q23, [%1, #480]\n" \ + st_ins " q16, q17, [%0, #384]\n" \ + st_ins " q18, q19, [%0, #416]\n" \ + ld_ins " q0, q1, [%1, #512]\n" \ + ld_ins " q2, q3, [%1, #544]\n" \ + st_ins " q20, q21, [%0, #448]\n" \ + st_ins " q22, q23, [%0, #480]\n" \ + ld_ins " q4, q5, [%1, #576]\n" \ + ld_ins " q6, q7, [%1, #608]\n" \ + st_ins " q0, q1, [%0, #512]\n" \ + st_ins " q2, q3, [%0, #544]\n" \ + ld_ins " q16, q17, [%1, #640]\n" \ + ld_ins " q18, q19, [%1, #672]\n" \ + st_ins " q4, q5, [%0, #576]\n" \ + st_ins " q6, q7, [%0, #608]\n" \ + ld_ins " q20, q21, [%1, #704]\n" \ + ld_ins " q22, q23, [%1, #736]\n" \ + st_ins " q16, q17, [%0, #640]\n" \ + st_ins " q18, q19, [%0, #672]\n" \ + st_ins " q20, q21, [%0, #704]\n" \ + st_ins " q22, q23, [%0, #736]\n" \ + "sub %2, %2, #12\n" \ + "add %1, %1, #1536\n" \ + "add %0, %0, #1536\n" \ + "b 0b\n" \ + "1:\n" \ + "cmp %2, #8\n" \ + "b.lt 2f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + ld_ins " q20, q21, [%1, #192]\n" \ + ld_ins " q22, q23, [%1, #224]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + st_ins " q20, q21, [%0, #192]\n" \ + st_ins " q22, q23, [%0, #224]\n" \ + "sub %2, %2, #8\n" \ + "add %1, %1, #1024\n" \ + "add %0, %0, #1024\n" \ + "b 1b\n" \ + "2:\n" \ + "cmp %2, #4\n" \ + "b.lt 3f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + ld_ins " q16, q17, [%1, #128]\n" \ + ld_ins " q18, q19, [%1, #160]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + st_ins " q16, q17, [%0, #128]\n" \ + st_ins " q18, q19, [%0, #160]\n" \ + "sub %2, %2, #4\n" \ + "add %1, %1, #512\n" \ + "add %0, %0, #512\n" \ + "b 2b\n" \ + "3:\n" \ + "cmp %2, #1\n" \ + "b.lt 4f\n" \ + ld_ins " q0, q1, [%1]\n" \ + ld_ins " q2, q3, [%1, #32]\n" \ + ld_ins " q4, q5, [%1, #64]\n" \ + ld_ins " q6, q7, [%1, #96]\n" \ + st_ins " q0, q1, [%0]\n" \ + st_ins " q2, q3, [%0, #32]\n" \ + st_ins " q4, q5, [%0, #64]\n" \ + st_ins " q6, q7, [%0, #96]\n" \ + "sub %2, %2, #1\n" \ + "add %1, %1, #128\n" \ + "add %0, %0, #128\n" \ + "b 3b\n" \ + "4:\n" \ + "cbz %3, 6f\n" \ + "5:\n" \ + "cmp %3, #32\n" \ + "b.lt 6f\n" \ + ld_ins " q0, q1, [%1]\n" \ + st_ins " q0, q1, [%0]\n" \ + "add %1, %1, #32\n" \ + "add %0, %0, #32\n" \ + "subs %3, %3, #32\n" \ + "b.ge 5b\n" \ + "6:\n" \ + : "+r"(dst), "+r"(src), "+r"(len), "+r"(rem) \ + : \ + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \ + "q16", "q17", "q18", "q19", "q20", "q21", "q22", "q23"); \ + } while (0) + +#ifdef __cplusplus +} + +#endif + +#endif +#endif