1/*- 2 * Copyright (c) 2018 The FreeBSD Foundation 3 * 4 * This software was developed by Mateusz Guzik <[email protected]> 5 * under sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <machine/asm.h> 30__FBSDID("$FreeBSD$"); 31 32#define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 33 34/* 35 * memmove(dst, src, cnt) 36 * rdi, rsi, rdx 37 */ 38 39/* 40 * Register state at entry is supposed to be as follows: 41 * rdi - destination 42 * rsi - source 43 * rdx - count 44 * 45 * The macro possibly clobbers the above and: rcx, r8, r9, 10 46 * It does not clobber rax nor r11. 47 */ 48.macro MEMMOVE erms overlap begin end 49 \begin 50 51 /* 52 * For sizes 0..32 all data is read before it is written, so there 53 * is no correctness issue with direction of copying. 54 */ 55 cmpq $32,%rcx 56 jbe 101632f 57 58.if \overlap == 1 59 movq %rdi,%r8 60 subq %rsi,%r8 61 cmpq %rcx,%r8 /* overlapping && src < dst? */ 62 jb 2f 63.endif 64 65 cmpq $256,%rcx 66 ja 1256f 67 68 ALIGN_TEXT 69103200: 70 movq (%rsi),%rdx 71 movq %rdx,(%rdi) 72 movq 8(%rsi),%rdx 73 movq %rdx,8(%rdi) 74 movq 16(%rsi),%rdx 75 movq %rdx,16(%rdi) 76 movq 24(%rsi),%rdx 77 movq %rdx,24(%rdi) 78 leaq 32(%rsi),%rsi 79 leaq 32(%rdi),%rdi 80 subq $32,%rcx 81 cmpq $32,%rcx 82 jae 103200b 83 cmpb $0,%cl 84 jne 101632f 85 \end 86 ret 87 ALIGN_TEXT 88101632: 89 cmpb $16,%cl 90 jl 100816f 91 movq (%rsi),%rdx 92 movq 8(%rsi),%r8 93 movq -16(%rsi,%rcx),%r9 94 movq -8(%rsi,%rcx),%r10 95 movq %rdx,(%rdi) 96 movq %r8,8(%rdi) 97 movq %r9,-16(%rdi,%rcx) 98 movq %r10,-8(%rdi,%rcx) 99 \end 100 ret 101 ALIGN_TEXT 102100816: 103 cmpb $8,%cl 104 jl 100408f 105 movq (%rsi),%rdx 106 movq -8(%rsi,%rcx),%r8 107 movq %rdx,(%rdi) 108 movq %r8,-8(%rdi,%rcx,) 109 \end 110 ret 111 ALIGN_TEXT 112100408: 113 cmpb $4,%cl 114 jl 100204f 115 movl (%rsi),%edx 116 movl -4(%rsi,%rcx),%r8d 117 movl %edx,(%rdi) 118 movl %r8d,-4(%rdi,%rcx) 119 \end 120 ret 121 ALIGN_TEXT 122100204: 123 cmpb $2,%cl 124 jl 100001f 125 movzwl (%rsi),%edx 126 movzwl -2(%rsi,%rcx),%r8d 127 movw %dx,(%rdi) 128 movw %r8w,-2(%rdi,%rcx) 129 \end 130 ret 131 ALIGN_TEXT 132100001: 133 cmpb $1,%cl 134 jl 100000f 135 movb (%rsi),%dl 136 movb %dl,(%rdi) 137100000: 138 \end 139 ret 140 141 ALIGN_TEXT 1421256: 143 testb $15,%dil 144 jnz 100f 145.if \erms == 1 146 rep 147 movsb 148.else 149 shrq $3,%rcx /* copy by 64-bit words */ 150 rep 151 movsq 152 movq %rdx,%rcx 153 andl $7,%ecx /* any bytes left? */ 154 jne 100408b 155.endif 156 \end 157 ret 158100: 159 movq (%rsi),%r8 160 movq 8(%rsi),%r9 161 movq %rdi,%r10 162 movq %rdi,%rcx 163 andq $15,%rcx 164 leaq -16(%rdx,%rcx),%rdx 165 neg %rcx 166 leaq 16(%rdi,%rcx),%rdi 167 leaq 16(%rsi,%rcx),%rsi 168 movq %rdx,%rcx 169.if \erms == 1 170 rep 171 movsb 172 movq %r8,(%r10) 173 movq %r9,8(%r10) 174.else 175 shrq $3,%rcx /* copy by 64-bit words */ 176 rep 177 movsq 178 movq %r8,(%r10) 179 movq %r9,8(%r10) 180 movq %rdx,%rcx 181 andl $7,%ecx /* any bytes left? */ 182 jne 100408b 183.endif 184 \end 185 ret 186 187.if \overlap == 1 188 /* 189 * Copy backwards. 190 */ 191 ALIGN_TEXT 1922: 193 cmpq $256,%rcx 194 ja 2256f 195 196 leaq -8(%rdi,%rcx),%rdi 197 leaq -8(%rsi,%rcx),%rsi 198 199 cmpq $32,%rcx 200 jb 2016f 201 202 ALIGN_TEXT 2032032: 204 movq (%rsi),%rdx 205 movq %rdx,(%rdi) 206 movq -8(%rsi),%rdx 207 movq %rdx,-8(%rdi) 208 movq -16(%rsi),%rdx 209 movq %rdx,-16(%rdi) 210 movq -24(%rsi),%rdx 211 movq %rdx,-24(%rdi) 212 leaq -32(%rsi),%rsi 213 leaq -32(%rdi),%rdi 214 subq $32,%rcx 215 cmpq $32,%rcx 216 jae 2032b 217 cmpb $0,%cl 218 jne 2016f 219 \end 220 ret 221 ALIGN_TEXT 2222016: 223 cmpb $16,%cl 224 jl 2008f 225 movq (%rsi),%rdx 226 movq %rdx,(%rdi) 227 movq -8(%rsi),%rdx 228 movq %rdx,-8(%rdi) 229 subb $16,%cl 230 jz 2000f 231 leaq -16(%rsi),%rsi 232 leaq -16(%rdi),%rdi 2332008: 234 cmpb $8,%cl 235 jl 2004f 236 movq (%rsi),%rdx 237 movq %rdx,(%rdi) 238 subb $8,%cl 239 jz 2000f 240 leaq -8(%rsi),%rsi 241 leaq -8(%rdi),%rdi 2422004: 243 cmpb $4,%cl 244 jl 2002f 245 movl 4(%rsi),%edx 246 movl %edx,4(%rdi) 247 subb $4,%cl 248 jz 2000f 249 leaq -4(%rsi),%rsi 250 leaq -4(%rdi),%rdi 2512002: 252 cmpb $2,%cl 253 jl 2001f 254 movw 6(%rsi),%dx 255 movw %dx,6(%rdi) 256 subb $2,%cl 257 jz 2000f 258 leaq -2(%rsi),%rsi 259 leaq -2(%rdi),%rdi 2602001: 261 cmpb $1,%cl 262 jl 2000f 263 movb 7(%rsi),%dl 264 movb %dl,7(%rdi) 2652000: 266 \end 267 ret 268 ALIGN_TEXT 2692256: 270 std 271.if \erms == 1 272 leaq -1(%rdi,%rcx),%rdi 273 leaq -1(%rsi,%rcx),%rsi 274 rep 275 movsb 276 cld 277.else 278 leaq -8(%rdi,%rcx),%rdi 279 leaq -8(%rsi,%rcx),%rsi 280 shrq $3,%rcx 281 rep 282 movsq 283 cld 284 movq %rdx,%rcx 285 andb $7,%cl 286 jne 2004b 287.endif 288 \end 289 ret 290.endif 291.endm 292 293 294.macro MEMMOVE_BEGIN 295 movq %rdi,%rax 296 movq %rdx,%rcx 297.endm 298 299.macro MEMMOVE_END 300.endm 301 302#ifndef MEMCPY 303ENTRY(memmove) 304 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 305END(memmove) 306#else 307ENTRY(memcpy) 308 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 309END(memcpy) 310#endif 311