1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Quick'n'dirty IP checksum ... 7 * 8 * Copyright (C) 1998, 1999 Ralf Baechle 9 * Copyright (C) 1999 Silicon Graphics, Inc. 10 * Copyright (C) 2007 Maciej W. Rozycki 11 * Copyright (C) 2014 Imagination Technologies Ltd. 12 */ 13#include <linux/errno.h> 14#include <asm/asm.h> 15#include <asm/asm-offsets.h> 16#include <asm/regdef.h> 17 18#ifdef CONFIG_64BIT 19/* 20 * As we are sharing code base with the mips32 tree (which use the o32 ABI 21 * register definitions). We need to redefine the register definitions from 22 * the n64 ABI register naming to the o32 ABI register naming. 23 */ 24#undef t0 25#undef t1 26#undef t2 27#undef t3 28#define t0 $8 29#define t1 $9 30#define t2 $10 31#define t3 $11 32#define t4 $12 33#define t5 $13 34#define t6 $14 35#define t7 $15 36 37#define USE_DOUBLE 38#endif 39 40#ifdef USE_DOUBLE 41 42#define LOAD ld 43#define LOAD32 lwu 44#define ADD daddu 45#define NBYTES 8 46 47#else 48 49#define LOAD lw 50#define LOAD32 lw 51#define ADD addu 52#define NBYTES 4 53 54#endif /* USE_DOUBLE */ 55 56#define UNIT(unit) ((unit)*NBYTES) 57 58#define ADDC(sum,reg) \ 59 ADD sum, reg; \ 60 sltu v1, sum, reg; \ 61 ADD sum, v1; \ 62 63#define ADDC32(sum,reg) \ 64 addu sum, reg; \ 65 sltu v1, sum, reg; \ 66 addu sum, v1; \ 67 68#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \ 69 LOAD _t0, (offset + UNIT(0))(src); \ 70 LOAD _t1, (offset + UNIT(1))(src); \ 71 LOAD _t2, (offset + UNIT(2))(src); \ 72 LOAD _t3, (offset + UNIT(3))(src); \ 73 ADDC(sum, _t0); \ 74 ADDC(sum, _t1); \ 75 ADDC(sum, _t2); \ 76 ADDC(sum, _t3) 77 78#ifdef USE_DOUBLE 79#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \ 80 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) 81#else 82#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \ 83 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \ 84 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3) 85#endif 86 87/* 88 * a0: source address 89 * a1: length of the area to checksum 90 * a2: partial checksum 91 */ 92 93#define src a0 94#define sum v0 95 96 .text 97 .set noreorder 98 .align 5 99LEAF(csum_partial) 100 move sum, zero 101 move t7, zero 102 103 sltiu t8, a1, 0x8 104 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */ 105 move t2, a1 106 107 andi t7, src, 0x1 /* odd buffer? */ 108 109.Lhword_align: 110 beqz t7, .Lword_align 111 andi t8, src, 0x2 112 113 lbu t0, (src) 114 LONG_SUBU a1, a1, 0x1 115#ifdef __MIPSEL__ 116 sll t0, t0, 8 117#endif 118 ADDC(sum, t0) 119 PTR_ADDU src, src, 0x1 120 andi t8, src, 0x2 121 122.Lword_align: 123 beqz t8, .Ldword_align 124 sltiu t8, a1, 56 125 126 lhu t0, (src) 127 LONG_SUBU a1, a1, 0x2 128 ADDC(sum, t0) 129 sltiu t8, a1, 56 130 PTR_ADDU src, src, 0x2 131 132.Ldword_align: 133 bnez t8, .Ldo_end_words 134 move t8, a1 135 136 andi t8, src, 0x4 137 beqz t8, .Lqword_align 138 andi t8, src, 0x8 139 140 LOAD32 t0, 0x00(src) 141 LONG_SUBU a1, a1, 0x4 142 ADDC(sum, t0) 143 PTR_ADDU src, src, 0x4 144 andi t8, src, 0x8 145 146.Lqword_align: 147 beqz t8, .Loword_align 148 andi t8, src, 0x10 149 150#ifdef USE_DOUBLE 151 ld t0, 0x00(src) 152 LONG_SUBU a1, a1, 0x8 153 ADDC(sum, t0) 154#else 155 lw t0, 0x00(src) 156 lw t1, 0x04(src) 157 LONG_SUBU a1, a1, 0x8 158 ADDC(sum, t0) 159 ADDC(sum, t1) 160#endif 161 PTR_ADDU src, src, 0x8 162 andi t8, src, 0x10 163 164.Loword_align: 165 beqz t8, .Lbegin_movement 166 LONG_SRL t8, a1, 0x7 167 168#ifdef USE_DOUBLE 169 ld t0, 0x00(src) 170 ld t1, 0x08(src) 171 ADDC(sum, t0) 172 ADDC(sum, t1) 173#else 174 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4) 175#endif 176 LONG_SUBU a1, a1, 0x10 177 PTR_ADDU src, src, 0x10 178 LONG_SRL t8, a1, 0x7 179 180.Lbegin_movement: 181 beqz t8, 1f 182 andi t2, a1, 0x40 183 184.Lmove_128bytes: 185 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) 186 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4) 187 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4) 188 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4) 189 LONG_SUBU t8, t8, 0x01 190 .set reorder /* DADDI_WAR */ 191 PTR_ADDU src, src, 0x80 192 bnez t8, .Lmove_128bytes 193 .set noreorder 194 1951: 196 beqz t2, 1f 197 andi t2, a1, 0x20 198 199.Lmove_64bytes: 200 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) 201 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4) 202 PTR_ADDU src, src, 0x40 203 2041: 205 beqz t2, .Ldo_end_words 206 andi t8, a1, 0x1c 207 208.Lmove_32bytes: 209 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) 210 andi t8, a1, 0x1c 211 PTR_ADDU src, src, 0x20 212 213.Ldo_end_words: 214 beqz t8, .Lsmall_csumcpy 215 andi t2, a1, 0x3 216 LONG_SRL t8, t8, 0x2 217 218.Lend_words: 219 LOAD32 t0, (src) 220 LONG_SUBU t8, t8, 0x1 221 ADDC(sum, t0) 222 .set reorder /* DADDI_WAR */ 223 PTR_ADDU src, src, 0x4 224 bnez t8, .Lend_words 225 .set noreorder 226 227/* unknown src alignment and < 8 bytes to go */ 228.Lsmall_csumcpy: 229 move a1, t2 230 231 andi t0, a1, 4 232 beqz t0, 1f 233 andi t0, a1, 2 234 235 /* Still a full word to go */ 236 ulw t1, (src) 237 PTR_ADDIU src, 4 238#ifdef USE_DOUBLE 239 dsll t1, t1, 32 /* clear lower 32bit */ 240#endif 241 ADDC(sum, t1) 242 2431: move t1, zero 244 beqz t0, 1f 245 andi t0, a1, 1 246 247 /* Still a halfword to go */ 248 ulhu t1, (src) 249 PTR_ADDIU src, 2 250 2511: beqz t0, 1f 252 sll t1, t1, 16 253 254 lbu t2, (src) 255 nop 256 257#ifdef __MIPSEB__ 258 sll t2, t2, 8 259#endif 260 or t1, t2 261 2621: ADDC(sum, t1) 263 264 /* fold checksum */ 265#ifdef USE_DOUBLE 266 dsll32 v1, sum, 0 267 daddu sum, v1 268 sltu v1, sum, v1 269 dsra32 sum, sum, 0 270 addu sum, v1 271#endif 272 273 /* odd buffer alignment? */ 274#ifdef CONFIG_CPU_MIPSR2 275 wsbh v1, sum 276 movn sum, v1, t7 277#else 278 beqz t7, 1f /* odd buffer alignment? */ 279 lui v1, 0x00ff 280 addu v1, 0x00ff 281 and t0, sum, v1 282 sll t0, t0, 8 283 srl sum, sum, 8 284 and sum, sum, v1 285 or sum, sum, t0 2861: 287#endif 288 .set reorder 289 /* Add the passed partial csum. */ 290 ADDC32(sum, a2) 291 jr ra 292 .set noreorder 293 END(csum_partial) 294 295 296/* 297 * checksum and copy routines based on memcpy.S 298 * 299 * csum_partial_copy_nocheck(src, dst, len, sum) 300 * __csum_partial_copy_kernel(src, dst, len, sum, errp) 301 * 302 * See "Spec" in memcpy.S for details. Unlike __copy_user, all 303 * function in this file use the standard calling convention. 304 */ 305 306#define src a0 307#define dst a1 308#define len a2 309#define psum a3 310#define sum v0 311#define odd t8 312#define errptr t9 313 314/* 315 * The exception handler for loads requires that: 316 * 1- AT contain the address of the byte just past the end of the source 317 * of the copy, 318 * 2- src_entry <= src < AT, and 319 * 3- (dst - src) == (dst_entry - src_entry), 320 * The _entry suffix denotes values when __copy_user was called. 321 * 322 * (1) is set up up by __csum_partial_copy_from_user and maintained by 323 * not writing AT in __csum_partial_copy 324 * (2) is met by incrementing src by the number of bytes copied 325 * (3) is met by not doing loads between a pair of increments of dst and src 326 * 327 * The exception handlers for stores stores -EFAULT to errptr and return. 328 * These handlers do not need to overwrite any data. 329 */ 330 331#define EXC(inst_reg,addr,handler) \ 3329: inst_reg, addr; \ 333 .section __ex_table,"a"; \ 334 PTR 9b, handler; \ 335 .previous 336 337#ifdef USE_DOUBLE 338 339#define LOAD ld 340#define LOADL ldl 341#define LOADR ldr 342#define STOREL sdl 343#define STORER sdr 344#define STORE sd 345#define ADD daddu 346#define SUB dsubu 347#define SRL dsrl 348#define SLL dsll 349#define SLLV dsllv 350#define SRLV dsrlv 351#define NBYTES 8 352#define LOG_NBYTES 3 353 354#else 355 356#define LOAD lw 357#define LOADL lwl 358#define LOADR lwr 359#define STOREL swl 360#define STORER swr 361#define STORE sw 362#define ADD addu 363#define SUB subu 364#define SRL srl 365#define SLL sll 366#define SLLV sllv 367#define SRLV srlv 368#define NBYTES 4 369#define LOG_NBYTES 2 370 371#endif /* USE_DOUBLE */ 372 373#ifdef CONFIG_CPU_LITTLE_ENDIAN 374#define LDFIRST LOADR 375#define LDREST LOADL 376#define STFIRST STORER 377#define STREST STOREL 378#define SHIFT_DISCARD SLLV 379#define SHIFT_DISCARD_REVERT SRLV 380#else 381#define LDFIRST LOADL 382#define LDREST LOADR 383#define STFIRST STOREL 384#define STREST STORER 385#define SHIFT_DISCARD SRLV 386#define SHIFT_DISCARD_REVERT SLLV 387#endif 388 389#define FIRST(unit) ((unit)*NBYTES) 390#define REST(unit) (FIRST(unit)+NBYTES-1) 391 392#define ADDRMASK (NBYTES-1) 393 394#ifndef CONFIG_CPU_DADDI_WORKAROUNDS 395 .set noat 396#else 397 .set at=v1 398#endif 399 400LEAF(__csum_partial_copy_kernel) 401FEXPORT(__csum_partial_copy_to_user) 402FEXPORT(__csum_partial_copy_from_user) 403 PTR_ADDU AT, src, len /* See (1) above. */ 404#ifdef CONFIG_64BIT 405 move errptr, a4 406#else 407 lw errptr, 16(sp) 408#endif 409FEXPORT(csum_partial_copy_nocheck) 410 move sum, zero 411 move odd, zero 412 /* 413 * Note: dst & src may be unaligned, len may be 0 414 * Temps 415 */ 416 /* 417 * The "issue break"s below are very approximate. 418 * Issue delays for dcache fills will perturb the schedule, as will 419 * load queue full replay traps, etc. 420 * 421 * If len < NBYTES use byte operations. 422 */ 423 sltu t2, len, NBYTES 424 and t1, dst, ADDRMASK 425 bnez t2, .Lcopy_bytes_checklen 426 and t0, src, ADDRMASK 427 andi odd, dst, 0x1 /* odd buffer? */ 428 bnez t1, .Ldst_unaligned 429 nop 430 bnez t0, .Lsrc_unaligned_dst_aligned 431 /* 432 * use delay slot for fall-through 433 * src and dst are aligned; need to compute rem 434 */ 435.Lboth_aligned: 436 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter 437 beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES 438 nop 439 SUB len, 8*NBYTES # subtract here for bgez loop 440 .align 4 4411: 442EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 443EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 444EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 445EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 446EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) 447EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy) 448EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy) 449EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy) 450 SUB len, len, 8*NBYTES 451 ADD src, src, 8*NBYTES 452EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 453 ADDC(sum, t0) 454EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 455 ADDC(sum, t1) 456EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 457 ADDC(sum, t2) 458EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 459 ADDC(sum, t3) 460EXC( STORE t4, UNIT(4)(dst), .Ls_exc) 461 ADDC(sum, t4) 462EXC( STORE t5, UNIT(5)(dst), .Ls_exc) 463 ADDC(sum, t5) 464EXC( STORE t6, UNIT(6)(dst), .Ls_exc) 465 ADDC(sum, t6) 466EXC( STORE t7, UNIT(7)(dst), .Ls_exc) 467 ADDC(sum, t7) 468 .set reorder /* DADDI_WAR */ 469 ADD dst, dst, 8*NBYTES 470 bgez len, 1b 471 .set noreorder 472 ADD len, 8*NBYTES # revert len (see above) 473 474 /* 475 * len == the number of bytes left to copy < 8*NBYTES 476 */ 477.Lcleanup_both_aligned: 478#define rem t7 479 beqz len, .Ldone 480 sltu t0, len, 4*NBYTES 481 bnez t0, .Lless_than_4units 482 and rem, len, (NBYTES-1) # rem = len % NBYTES 483 /* 484 * len >= 4*NBYTES 485 */ 486EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 487EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 488EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 489EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 490 SUB len, len, 4*NBYTES 491 ADD src, src, 4*NBYTES 492EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 493 ADDC(sum, t0) 494EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 495 ADDC(sum, t1) 496EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 497 ADDC(sum, t2) 498EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 499 ADDC(sum, t3) 500 .set reorder /* DADDI_WAR */ 501 ADD dst, dst, 4*NBYTES 502 beqz len, .Ldone 503 .set noreorder 504.Lless_than_4units: 505 /* 506 * rem = len % NBYTES 507 */ 508 beq rem, len, .Lcopy_bytes 509 nop 5101: 511EXC( LOAD t0, 0(src), .Ll_exc) 512 ADD src, src, NBYTES 513 SUB len, len, NBYTES 514EXC( STORE t0, 0(dst), .Ls_exc) 515 ADDC(sum, t0) 516 .set reorder /* DADDI_WAR */ 517 ADD dst, dst, NBYTES 518 bne rem, len, 1b 519 .set noreorder 520 521 /* 522 * src and dst are aligned, need to copy rem bytes (rem < NBYTES) 523 * A loop would do only a byte at a time with possible branch 524 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE 525 * because can't assume read-access to dst. Instead, use 526 * STREST dst, which doesn't require read access to dst. 527 * 528 * This code should perform better than a simple loop on modern, 529 * wide-issue mips processors because the code has fewer branches and 530 * more instruction-level parallelism. 531 */ 532#define bits t2 533 beqz len, .Ldone 534 ADD t1, dst, len # t1 is just past last byte of dst 535 li bits, 8*NBYTES 536 SLL rem, len, 3 # rem = number of bits to keep 537EXC( LOAD t0, 0(src), .Ll_exc) 538 SUB bits, bits, rem # bits = number of bits to discard 539 SHIFT_DISCARD t0, t0, bits 540EXC( STREST t0, -1(t1), .Ls_exc) 541 SHIFT_DISCARD_REVERT t0, t0, bits 542 .set reorder 543 ADDC(sum, t0) 544 b .Ldone 545 .set noreorder 546.Ldst_unaligned: 547 /* 548 * dst is unaligned 549 * t0 = src & ADDRMASK 550 * t1 = dst & ADDRMASK; T1 > 0 551 * len >= NBYTES 552 * 553 * Copy enough bytes to align dst 554 * Set match = (src and dst have same alignment) 555 */ 556#define match rem 557EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) 558 ADD t2, zero, NBYTES 559EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) 560 SUB t2, t2, t1 # t2 = number of bytes copied 561 xor match, t0, t1 562EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 563 SLL t4, t1, 3 # t4 = number of bits to discard 564 SHIFT_DISCARD t3, t3, t4 565 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ 566 ADDC(sum, t3) 567 beq len, t2, .Ldone 568 SUB len, len, t2 569 ADD dst, dst, t2 570 beqz match, .Lboth_aligned 571 ADD src, src, t2 572 573.Lsrc_unaligned_dst_aligned: 574 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter 575 beqz t0, .Lcleanup_src_unaligned 576 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES 5771: 578/* 579 * Avoid consecutive LD*'s to the same register since some mips 580 * implementations can't issue them in the same cycle. 581 * It's OK to load FIRST(N+1) before REST(N) because the two addresses 582 * are to the same unit (unless src is aligned, but it's not). 583 */ 584EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 585EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) 586 SUB len, len, 4*NBYTES 587EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 588EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) 589EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) 590EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) 591EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) 592EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) 593 ADD src, src, 4*NBYTES 594#ifdef CONFIG_CPU_SB1 595 nop # improves slotting 596#endif 597EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 598 ADDC(sum, t0) 599EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 600 ADDC(sum, t1) 601EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 602 ADDC(sum, t2) 603EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 604 ADDC(sum, t3) 605 .set reorder /* DADDI_WAR */ 606 ADD dst, dst, 4*NBYTES 607 bne len, rem, 1b 608 .set noreorder 609 610.Lcleanup_src_unaligned: 611 beqz len, .Ldone 612 and rem, len, NBYTES-1 # rem = len % NBYTES 613 beq rem, len, .Lcopy_bytes 614 nop 6151: 616EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 617EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 618 ADD src, src, NBYTES 619 SUB len, len, NBYTES 620EXC( STORE t0, 0(dst), .Ls_exc) 621 ADDC(sum, t0) 622 .set reorder /* DADDI_WAR */ 623 ADD dst, dst, NBYTES 624 bne len, rem, 1b 625 .set noreorder 626 627.Lcopy_bytes_checklen: 628 beqz len, .Ldone 629 nop 630.Lcopy_bytes: 631 /* 0 < len < NBYTES */ 632#ifdef CONFIG_CPU_LITTLE_ENDIAN 633#define SHIFT_START 0 634#define SHIFT_INC 8 635#else 636#define SHIFT_START 8*(NBYTES-1) 637#define SHIFT_INC -8 638#endif 639 move t2, zero # partial word 640 li t3, SHIFT_START # shift 641/* use .Ll_exc_copy here to return correct sum on fault */ 642#define COPY_BYTE(N) \ 643EXC( lbu t0, N(src), .Ll_exc_copy); \ 644 SUB len, len, 1; \ 645EXC( sb t0, N(dst), .Ls_exc); \ 646 SLLV t0, t0, t3; \ 647 addu t3, SHIFT_INC; \ 648 beqz len, .Lcopy_bytes_done; \ 649 or t2, t0 650 651 COPY_BYTE(0) 652 COPY_BYTE(1) 653#ifdef USE_DOUBLE 654 COPY_BYTE(2) 655 COPY_BYTE(3) 656 COPY_BYTE(4) 657 COPY_BYTE(5) 658#endif 659EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy) 660 SUB len, len, 1 661EXC( sb t0, NBYTES-2(dst), .Ls_exc) 662 SLLV t0, t0, t3 663 or t2, t0 664.Lcopy_bytes_done: 665 ADDC(sum, t2) 666.Ldone: 667 /* fold checksum */ 668#ifdef USE_DOUBLE 669 dsll32 v1, sum, 0 670 daddu sum, v1 671 sltu v1, sum, v1 672 dsra32 sum, sum, 0 673 addu sum, v1 674#endif 675 676#ifdef CONFIG_CPU_MIPSR2 677 wsbh v1, sum 678 movn sum, v1, odd 679#else 680 beqz odd, 1f /* odd buffer alignment? */ 681 lui v1, 0x00ff 682 addu v1, 0x00ff 683 and t0, sum, v1 684 sll t0, t0, 8 685 srl sum, sum, 8 686 and sum, sum, v1 687 or sum, sum, t0 6881: 689#endif 690 .set reorder 691 ADDC32(sum, psum) 692 jr ra 693 .set noreorder 694 695.Ll_exc_copy: 696 /* 697 * Copy bytes from src until faulting load address (or until a 698 * lb faults) 699 * 700 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28) 701 * may be more than a byte beyond the last address. 702 * Hence, the lb below may get an exception. 703 * 704 * Assumes src < THREAD_BUADDR($28) 705 */ 706 LOAD t0, TI_TASK($28) 707 li t2, SHIFT_START 708 LOAD t0, THREAD_BUADDR(t0) 7091: 710EXC( lbu t1, 0(src), .Ll_exc) 711 ADD src, src, 1 712 sb t1, 0(dst) # can't fault -- we're copy_from_user 713 SLLV t1, t1, t2 714 addu t2, SHIFT_INC 715 ADDC(sum, t1) 716 .set reorder /* DADDI_WAR */ 717 ADD dst, dst, 1 718 bne src, t0, 1b 719 .set noreorder 720.Ll_exc: 721 LOAD t0, TI_TASK($28) 722 nop 723 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address 724 nop 725 SUB len, AT, t0 # len number of uncopied bytes 726 /* 727 * Here's where we rely on src and dst being incremented in tandem, 728 * See (3) above. 729 * dst += (fault addr - src) to put dst at first byte to clear 730 */ 731 ADD dst, t0 # compute start address in a1 732 SUB dst, src 733 /* 734 * Clear len bytes starting at dst. Can't call __bzero because it 735 * might modify len. An inefficient loop for these rare times... 736 */ 737 .set reorder /* DADDI_WAR */ 738 SUB src, len, 1 739 beqz len, .Ldone 740 .set noreorder 7411: sb zero, 0(dst) 742 ADD dst, dst, 1 743 .set push 744 .set noat 745#ifndef CONFIG_CPU_DADDI_WORKAROUNDS 746 bnez src, 1b 747 SUB src, src, 1 748#else 749 li v1, 1 750 bnez src, 1b 751 SUB src, src, v1 752#endif 753 li v1, -EFAULT 754 b .Ldone 755 sw v1, (errptr) 756 757.Ls_exc: 758 li v0, -1 /* invalid checksum */ 759 li v1, -EFAULT 760 jr ra 761 sw v1, (errptr) 762 .set pop 763 END(__csum_partial_copy_kernel) 764