libs/newlib-cygwin
libs
/
newlib-cygwin
mirror of git://sourceware.org/git/newlib-cygwin.git
4
0
Fork 0
Code Issues Projects Releases Wiki Activity

* libc/machine/aarch64/strlen.S (strlen): Improve performance.

This commit is contained in:
Corinna Vinschen 2015-01-20 10:11:56 +00:00
parent 88171c62d3
commit 086cd00d24
2 changed files with 179 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
newlib/ChangeLog
View File

@ -1,3 +1,7 @@
2015-01-19 Wilco Dijkstra wdijkstr@arm.com
* libc/machine/aarch64/strlen.S (strlen): Improve performance.
2015-01-19 Stefan Wallentowitz <stefan.wallentowitz@tum.de> 2015-01-19 Stefan Wallentowitz <stefan.wallentowitz@tum.de>
* configure.host: Add extra system for OpenRISC baremetal. * configure.host: Add extra system for OpenRISC baremetal.

248
newlib/libc/machine/aarch64/strlen.S
View File

@ -1,16 +1,16 @@
/* Copyright (c) 2013, Linaro Limited /* Copyright (c) 2013-2015, Linaro Limited
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met: modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright * Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
* Neither the name of the Linaro nor the * Neither the name of the Linaro nor the
names of its contributors may be used to endorse or promote products names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
@ -30,9 +30,13 @@
/* Assumptions: /* Assumptions:
* *
* ARMv8-a, AArch64 * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
*/ */
/* To test the page crossing code path more thoroughly, compile with
-DTEST_PAGE_CROSS - this will force all calls through the slower
entry path. This option is not intended for production use. */
/* Arguments and results. */ /* Arguments and results. */
#define srcin x0 #define srcin x0
#define len x0 #define len x0
@ -41,15 +45,15 @@
#define src x1 #define src x1
#define data1 x2 #define data1 x2
#define data2 x3 #define data2 x3
#define data2a x4 #define has_nul1 x4
#define has_nul1 x5 #define has_nul2 x5
#define has_nul2 x6 #define tmp1 x4
#define tmp1 x7 #define tmp2 x5
#define tmp2 x8 #define tmp3 x6
#define tmp3 x9 #define tmp4 x7
#define tmp4 x10 #define zeroones x8
#define zeroones x11
#define pos x12 #define L(l) .L ## l
.macro def_fn f p2align=0 .macro def_fn f p2align=0
.text .text
@ -59,78 +63,176 @@
\f: \f:
.endm .endm
/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
(=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
can be done in parallel across the entire word. A faster check
(X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
false hits for characters 129..255. */
#define REP8_01 0x0101010101010101 #define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080 #define REP8_80 0x8080808080808080
/* Start of critial section -- keep to one 64Byte cache line. */ #ifdef TEST_PAGE_CROSS
def_fn strlen p2align=6 # define MIN_PAGE_SIZE 15
mov zeroones, #REP8_01 #else
bic src, srcin, #15 # define MIN_PAGE_SIZE 4096
ands tmp1, srcin, #15
b.ne .Lmisaligned
/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
(=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
can be done in parallel across the entire word. */
/* The inner loop deals with two Dwords at a time. This has a
slightly higher start-up cost, but we should win quite quickly,
especially on cores with a high number of issue slots per
cycle, as we get much better parallelism out of the operations. */
.Lloop:
ldp data1, data2, [src], #16
.Lrealigned:
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
sub tmp3, data2, zeroones
orr tmp4, data2, #REP8_7f
bic has_nul1, tmp1, tmp2
bics has_nul2, tmp3, tmp4
ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
b.eq .Lloop
/* End of critical section -- keep to one 64Byte cache line. */
sub len, src, srcin
cbz has_nul1, .Lnul_in_data2
#ifdef __AARCH64EB__
mov data2, data1
#endif #endif
sub len, len, #8
mov has_nul2, has_nul1 /* Since strings are short on average, we check the first 16 bytes
.Lnul_in_data2: of the string for a NUL character. In order to do an unaligned ldp
safely we have to do a page cross check first. If there is a NUL
byte we calculate the length from the 2 8-byte words using
conditional select to reduce branch mispredictions (it is unlikely
strlen will be repeatedly called on strings with the same length).
If the string is longer than 16 bytes, we align src so don't need
further page cross checks, and process 32 bytes per iteration
using the fast NUL check. If we encounter non-ASCII characters,
fallback to a second loop using the full NUL check.
If the page cross check fails, we read 16 bytes from an aligned
address, remove any characters before the string, and continue
in the main loop using aligned loads. Since strings crossing a
page in the first 16 bytes are rare (probability of
16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.
AArch64 systems have a minimum page size of 4k. We don't bother
checking for larger page sizes - the cost of setting up the correct
page size is just not worth the extra gain from a small reduction in
the cases taking the slow path. Note that we only care about
whether the first fetch, which may be misaligned, crosses a page
boundary. */
def_fn strlen p2align=6
and tmp1, srcin, MIN_PAGE_SIZE - 1
mov zeroones, REP8_01
cmp tmp1, MIN_PAGE_SIZE - 16
b.gt L(page_cross)
ldp data1, data2, [srcin]
#ifdef __AARCH64EB__ #ifdef __AARCH64EB__
/* For big-endian, carry propagation (if the final byte in the /* For big-endian, carry propagation (if the final byte in the
string is 0x01) means we cannot use has_nul directly. The string is 0x01) means we cannot use has_nul1/2 directly.
easiest way to get the correct byte is to byte-swap the data Since we expect strings to be small and early-exit,
and calculate the syndrome a second time. */ byte-swap the data now so has_null1/2 will be correct. */
rev data1, data1
rev data2, data2 rev data2, data2
sub tmp1, data2, zeroones
orr tmp2, data2, #REP8_7f
bic has_nul2, tmp1, tmp2
#endif #endif
sub len, len, #8 sub tmp1, data1, zeroones
rev has_nul2, has_nul2 orr tmp2, data1, REP8_7f
clz pos, has_nul2 sub tmp3, data2, zeroones
add len, len, pos, lsr #3 /* Bits to bytes. */ orr tmp4, data2, REP8_7f
bics has_nul1, tmp1, tmp2
bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
beq L(main_loop_entry)
/* Enter with C = has_nul1 == 0. */
csel has_nul1, has_nul1, has_nul2, cc
mov len, 8
rev has_nul1, has_nul1
clz tmp1, has_nul1
csel len, xzr, len, cc
add len, len, tmp1, lsr 3
ret ret
.Lmisaligned: /* The inner loop processes 32 bytes per iteration and uses the fast
cmp tmp1, #8 NUL check. If we encounter non-ASCII characters, use a second
neg tmp1, tmp1 loop with the accurate NUL check. */
ldp data1, data2, [src], #16 .p2align 4
lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ L(main_loop_entry):
mov tmp2, #~0 bic src, srcin, 15
sub src, src, 16
L(main_loop):
ldp data1, data2, [src, 32]!
.Lpage_cross_entry:
sub tmp1, data1, zeroones
sub tmp3, data2, zeroones
orr tmp2, tmp1, tmp3
tst tmp2, zeroones, lsl 7
bne 1f
ldp data1, data2, [src, 16]
sub tmp1, data1, zeroones
sub tmp3, data2, zeroones
orr tmp2, tmp1, tmp3
tst tmp2, zeroones, lsl 7
beq L(main_loop)
add src, src, 16
1:
/* The fast check failed, so do the slower, accurate NUL check. */
orr tmp2, data1, REP8_7f
orr tmp4, data2, REP8_7f
bics has_nul1, tmp1, tmp2
bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
beq L(nonascii_loop)
/* Enter with C = has_nul1 == 0. */
L(tail):
#ifdef __AARCH64EB__ #ifdef __AARCH64EB__
/* Big-endian. Early bytes are at MSB. */ /* For big-endian, carry propagation (if the final byte in the
lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ string is 0x01) means we cannot use has_nul1/2 directly. The
easiest way to get the correct byte is to byte-swap the data
and calculate the syndrome a second time. */
csel data1, data1, data2, cc
rev data1, data1
sub tmp1, data1, zeroones
orr tmp2, data1, REP8_7f
bic has_nul1, tmp1, tmp2
#else
csel has_nul1, has_nul1, has_nul2, cc
#endif
sub len, src, srcin
rev has_nul1, has_nul1
add tmp2, len, 8
clz tmp1, has_nul1
csel len, len, tmp2, cc
add len, len, tmp1, lsr 3
ret
L(nonascii_loop):
ldp data1, data2, [src, 16]!
sub tmp1, data1, zeroones
orr tmp2, data1, REP8_7f
sub tmp3, data2, zeroones
orr tmp4, data2, REP8_7f
bics has_nul1, tmp1, tmp2
bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
bne L(tail)
ldp data1, data2, [src, 16]!
sub tmp1, data1, zeroones
orr tmp2, data1, REP8_7f
sub tmp3, data2, zeroones
orr tmp4, data2, REP8_7f
bics has_nul1, tmp1, tmp2
bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
beq L(nonascii_loop)
b L(tail)
/* Load 16 bytes from [srcin & ~15] and force the bytes that precede
srcin to 0x7f, so we ignore any NUL bytes before the string.
Then continue in the aligned loop. */
L(page_cross):
bic src, srcin, 15
ldp data1, data2, [src]
lsl tmp1, srcin, 3
mov tmp4, -1
#ifdef __AARCH64EB__
/* Big-endian. Early bytes are at MSB. */
lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
#else #else
/* Little-endian. Early bytes are at LSB. */ /* Little-endian. Early bytes are at LSB. */
lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
#endif #endif
orr data1, data1, tmp2 orr tmp1, tmp1, REP8_80
orr data2a, data2, tmp2 orn data1, data1, tmp1
csinv data1, data1, xzr, le orn tmp2, data2, tmp1
csel data2, data2, data2a, le tst srcin, 8
b .Lrealigned csel data1, data1, tmp4, eq
csel data2, data2, tmp2, eq
b L(page_cross_entry)
.size strlen, . - strlen .size strlen, . - strlen
#endif #endif