1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Quick'n'dirty IP checksum ... 7 * 8 * Copyright (C) 1998, 1999 Ralf Baechle 9 * Copyright (C) 1999 Silicon Graphics, Inc. 10 * Copyright (C) 2007 Maciej W. Rozycki 11 */ 12#include <linux/errno.h> 13#include <asm/asm.h> 14#include <asm/asm-offsets.h> 15#include <asm/regdef.h> 16 17#ifdef CONFIG_64BIT 18/* 19 * As we are sharing code base with the mips32 tree (which use the o32 ABI 20 * register definitions). We need to redefine the register definitions from 21 * the n64 ABI register naming to the o32 ABI register naming. 22 */ 23#undef t0 24#undef t1 25#undef t2 26#undef t3 27#define t0 $8 28#define t1 $9 29#define t2 $10 30#define t3 $11 31#define t4 $12 32#define t5 $13 33#define t6 $14 34#define t7 $15 35 36#define USE_DOUBLE 37#endif 38 39#ifdef USE_DOUBLE 40 41#define LOAD ld 42#define LOAD32 lwu 43#define ADD daddu 44#define NBYTES 8 45 46#else 47 48#define LOAD lw 49#define LOAD32 lw 50#define ADD addu 51#define NBYTES 4 52 53#endif /* USE_DOUBLE */ 54 55#define UNIT(unit) ((unit)*NBYTES) 56 57#define ADDC(sum,reg) \ 58 ADD sum, reg; \ 59 sltu v1, sum, reg; \ 60 ADD sum, v1; \ 61 62#define ADDC32(sum,reg) \ 63 addu sum, reg; \ 64 sltu v1, sum, reg; \ 65 addu sum, v1; \ 66 67#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \ 68 LOAD _t0, (offset + UNIT(0))(src); \ 69 LOAD _t1, (offset + UNIT(1))(src); \ 70 LOAD _t2, (offset + UNIT(2))(src); \ 71 LOAD _t3, (offset + UNIT(3))(src); \ 72 ADDC(sum, _t0); \ 73 ADDC(sum, _t1); \ 74 ADDC(sum, _t2); \ 75 ADDC(sum, _t3) 76 77#ifdef USE_DOUBLE 78#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \ 79 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) 80#else 81#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \ 82 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \ 83 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3) 84#endif 85 86/* 87 * a0: source address 88 * a1: length of the area to checksum 89 * a2: partial checksum 90 */ 91 92#define src a0 93#define sum v0 94 95 .text 96 .set noreorder 97 .align 5 98LEAF(csum_partial) 99 move sum, zero 100 move t7, zero 101 102 sltiu t8, a1, 0x8 103 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */ 104 move t2, a1 105 106 andi t7, src, 0x1 /* odd buffer? */ 107 108.Lhword_align: 109 beqz t7, .Lword_align 110 andi t8, src, 0x2 111 112 lbu t0, (src) 113 LONG_SUBU a1, a1, 0x1 114#ifdef __MIPSEL__ 115 sll t0, t0, 8 116#endif 117 ADDC(sum, t0) 118 PTR_ADDU src, src, 0x1 119 andi t8, src, 0x2 120 121.Lword_align: 122 beqz t8, .Ldword_align 123 sltiu t8, a1, 56 124 125 lhu t0, (src) 126 LONG_SUBU a1, a1, 0x2 127 ADDC(sum, t0) 128 sltiu t8, a1, 56 129 PTR_ADDU src, src, 0x2 130 131.Ldword_align: 132 bnez t8, .Ldo_end_words 133 move t8, a1 134 135 andi t8, src, 0x4 136 beqz t8, .Lqword_align 137 andi t8, src, 0x8 138 139 LOAD32 t0, 0x00(src) 140 LONG_SUBU a1, a1, 0x4 141 ADDC(sum, t0) 142 PTR_ADDU src, src, 0x4 143 andi t8, src, 0x8 144 145.Lqword_align: 146 beqz t8, .Loword_align 147 andi t8, src, 0x10 148 149#ifdef USE_DOUBLE 150 ld t0, 0x00(src) 151 LONG_SUBU a1, a1, 0x8 152 ADDC(sum, t0) 153#else 154 lw t0, 0x00(src) 155 lw t1, 0x04(src) 156 LONG_SUBU a1, a1, 0x8 157 ADDC(sum, t0) 158 ADDC(sum, t1) 159#endif 160 PTR_ADDU src, src, 0x8 161 andi t8, src, 0x10 162 163.Loword_align: 164 beqz t8, .Lbegin_movement 165 LONG_SRL t8, a1, 0x7 166 167#ifdef USE_DOUBLE 168 ld t0, 0x00(src) 169 ld t1, 0x08(src) 170 ADDC(sum, t0) 171 ADDC(sum, t1) 172#else 173 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4) 174#endif 175 LONG_SUBU a1, a1, 0x10 176 PTR_ADDU src, src, 0x10 177 LONG_SRL t8, a1, 0x7 178 179.Lbegin_movement: 180 beqz t8, 1f 181 andi t2, a1, 0x40 182 183.Lmove_128bytes: 184 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) 185 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4) 186 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4) 187 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4) 188 LONG_SUBU t8, t8, 0x01 189 .set reorder /* DADDI_WAR */ 190 PTR_ADDU src, src, 0x80 191 bnez t8, .Lmove_128bytes 192 .set noreorder 193 1941: 195 beqz t2, 1f 196 andi t2, a1, 0x20 197 198.Lmove_64bytes: 199 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) 200 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4) 201 PTR_ADDU src, src, 0x40 202 2031: 204 beqz t2, .Ldo_end_words 205 andi t8, a1, 0x1c 206 207.Lmove_32bytes: 208 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) 209 andi t8, a1, 0x1c 210 PTR_ADDU src, src, 0x20 211 212.Ldo_end_words: 213 beqz t8, .Lsmall_csumcpy 214 andi t2, a1, 0x3 215 LONG_SRL t8, t8, 0x2 216 217.Lend_words: 218 LOAD32 t0, (src) 219 LONG_SUBU t8, t8, 0x1 220 ADDC(sum, t0) 221 .set reorder /* DADDI_WAR */ 222 PTR_ADDU src, src, 0x4 223 bnez t8, .Lend_words 224 .set noreorder 225 226/* unknown src alignment and < 8 bytes to go */ 227.Lsmall_csumcpy: 228 move a1, t2 229 230 andi t0, a1, 4 231 beqz t0, 1f 232 andi t0, a1, 2 233 234 /* Still a full word to go */ 235 ulw t1, (src) 236 PTR_ADDIU src, 4 237#ifdef USE_DOUBLE 238 dsll t1, t1, 32 /* clear lower 32bit */ 239#endif 240 ADDC(sum, t1) 241 2421: move t1, zero 243 beqz t0, 1f 244 andi t0, a1, 1 245 246 /* Still a halfword to go */ 247 ulhu t1, (src) 248 PTR_ADDIU src, 2 249 2501: beqz t0, 1f 251 sll t1, t1, 16 252 253 lbu t2, (src) 254 nop 255 256#ifdef __MIPSEB__ 257 sll t2, t2, 8 258#endif 259 or t1, t2 260 2611: ADDC(sum, t1) 262 263 /* fold checksum */ 264#ifdef USE_DOUBLE 265 dsll32 v1, sum, 0 266 daddu sum, v1 267 sltu v1, sum, v1 268 dsra32 sum, sum, 0 269 addu sum, v1 270#endif 271 272 /* odd buffer alignment? */ 273 beqz t7, 1f 274 nop 275 sll v1, sum, 8 276 srl sum, sum, 8 277 or sum, v1 278 andi sum, 0xffff 2791: 280 .set reorder 281 /* Add the passed partial csum. */ 282 ADDC32(sum, a2) 283 jr ra 284 .set noreorder 285 END(csum_partial) 286 287 288/* 289 * checksum and copy routines based on memcpy.S 290 * 291 * csum_partial_copy_nocheck(src, dst, len, sum) 292 * __csum_partial_copy_user(src, dst, len, sum, errp) 293 * 294 * See "Spec" in memcpy.S for details. Unlike __copy_user, all 295 * function in this file use the standard calling convention. 296 */ 297 298#define src a0 299#define dst a1 300#define len a2 301#define psum a3 302#define sum v0 303#define odd t8 304#define errptr t9 305 306/* 307 * The exception handler for loads requires that: 308 * 1- AT contain the address of the byte just past the end of the source 309 * of the copy, 310 * 2- src_entry <= src < AT, and 311 * 3- (dst - src) == (dst_entry - src_entry), 312 * The _entry suffix denotes values when __copy_user was called. 313 * 314 * (1) is set up up by __csum_partial_copy_from_user and maintained by 315 * not writing AT in __csum_partial_copy 316 * (2) is met by incrementing src by the number of bytes copied 317 * (3) is met by not doing loads between a pair of increments of dst and src 318 * 319 * The exception handlers for stores stores -EFAULT to errptr and return. 320 * These handlers do not need to overwrite any data. 321 */ 322 323#define EXC(inst_reg,addr,handler) \ 3249: inst_reg, addr; \ 325 .section __ex_table,"a"; \ 326 PTR 9b, handler; \ 327 .previous 328 329#ifdef USE_DOUBLE 330 331#define LOAD ld 332#define LOADL ldl 333#define LOADR ldr 334#define STOREL sdl 335#define STORER sdr 336#define STORE sd 337#define ADD daddu 338#define SUB dsubu 339#define SRL dsrl 340#define SLL dsll 341#define SLLV dsllv 342#define SRLV dsrlv 343#define NBYTES 8 344#define LOG_NBYTES 3 345 346#else 347 348#define LOAD lw 349#define LOADL lwl 350#define LOADR lwr 351#define STOREL swl 352#define STORER swr 353#define STORE sw 354#define ADD addu 355#define SUB subu 356#define SRL srl 357#define SLL sll 358#define SLLV sllv 359#define SRLV srlv 360#define NBYTES 4 361#define LOG_NBYTES 2 362 363#endif /* USE_DOUBLE */ 364 365#ifdef CONFIG_CPU_LITTLE_ENDIAN 366#define LDFIRST LOADR 367#define LDREST LOADL 368#define STFIRST STORER 369#define STREST STOREL 370#define SHIFT_DISCARD SLLV 371#define SHIFT_DISCARD_REVERT SRLV 372#else 373#define LDFIRST LOADL 374#define LDREST LOADR 375#define STFIRST STOREL 376#define STREST STORER 377#define SHIFT_DISCARD SRLV 378#define SHIFT_DISCARD_REVERT SLLV 379#endif 380 381#define FIRST(unit) ((unit)*NBYTES) 382#define REST(unit) (FIRST(unit)+NBYTES-1) 383 384#define ADDRMASK (NBYTES-1) 385 386#ifndef CONFIG_CPU_DADDI_WORKAROUNDS 387 .set noat 388#else 389 .set at=v1 390#endif 391 392LEAF(__csum_partial_copy_user) 393 PTR_ADDU AT, src, len /* See (1) above. */ 394#ifdef CONFIG_64BIT 395 move errptr, a4 396#else 397 lw errptr, 16(sp) 398#endif 399FEXPORT(csum_partial_copy_nocheck) 400 move sum, zero 401 move odd, zero 402 /* 403 * Note: dst & src may be unaligned, len may be 0 404 * Temps 405 */ 406 /* 407 * The "issue break"s below are very approximate. 408 * Issue delays for dcache fills will perturb the schedule, as will 409 * load queue full replay traps, etc. 410 * 411 * If len < NBYTES use byte operations. 412 */ 413 sltu t2, len, NBYTES 414 and t1, dst, ADDRMASK 415 bnez t2, .Lcopy_bytes_checklen 416 and t0, src, ADDRMASK 417 andi odd, dst, 0x1 /* odd buffer? */ 418 bnez t1, .Ldst_unaligned 419 nop 420 bnez t0, .Lsrc_unaligned_dst_aligned 421 /* 422 * use delay slot for fall-through 423 * src and dst are aligned; need to compute rem 424 */ 425.Lboth_aligned: 426 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter 427 beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES 428 nop 429 SUB len, 8*NBYTES # subtract here for bgez loop 430 .align 4 4311: 432EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 433EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 434EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 435EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 436EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) 437EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy) 438EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy) 439EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy) 440 SUB len, len, 8*NBYTES 441 ADD src, src, 8*NBYTES 442EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 443 ADDC(sum, t0) 444EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 445 ADDC(sum, t1) 446EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 447 ADDC(sum, t2) 448EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 449 ADDC(sum, t3) 450EXC( STORE t4, UNIT(4)(dst), .Ls_exc) 451 ADDC(sum, t4) 452EXC( STORE t5, UNIT(5)(dst), .Ls_exc) 453 ADDC(sum, t5) 454EXC( STORE t6, UNIT(6)(dst), .Ls_exc) 455 ADDC(sum, t6) 456EXC( STORE t7, UNIT(7)(dst), .Ls_exc) 457 ADDC(sum, t7) 458 .set reorder /* DADDI_WAR */ 459 ADD dst, dst, 8*NBYTES 460 bgez len, 1b 461 .set noreorder 462 ADD len, 8*NBYTES # revert len (see above) 463 464 /* 465 * len == the number of bytes left to copy < 8*NBYTES 466 */ 467.Lcleanup_both_aligned: 468#define rem t7 469 beqz len, .Ldone 470 sltu t0, len, 4*NBYTES 471 bnez t0, .Lless_than_4units 472 and rem, len, (NBYTES-1) # rem = len % NBYTES 473 /* 474 * len >= 4*NBYTES 475 */ 476EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 477EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 478EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 479EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 480 SUB len, len, 4*NBYTES 481 ADD src, src, 4*NBYTES 482EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 483 ADDC(sum, t0) 484EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 485 ADDC(sum, t1) 486EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 487 ADDC(sum, t2) 488EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 489 ADDC(sum, t3) 490 .set reorder /* DADDI_WAR */ 491 ADD dst, dst, 4*NBYTES 492 beqz len, .Ldone 493 .set noreorder 494.Lless_than_4units: 495 /* 496 * rem = len % NBYTES 497 */ 498 beq rem, len, .Lcopy_bytes 499 nop 5001: 501EXC( LOAD t0, 0(src), .Ll_exc) 502 ADD src, src, NBYTES 503 SUB len, len, NBYTES 504EXC( STORE t0, 0(dst), .Ls_exc) 505 ADDC(sum, t0) 506 .set reorder /* DADDI_WAR */ 507 ADD dst, dst, NBYTES 508 bne rem, len, 1b 509 .set noreorder 510 511 /* 512 * src and dst are aligned, need to copy rem bytes (rem < NBYTES) 513 * A loop would do only a byte at a time with possible branch 514 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE 515 * because can't assume read-access to dst. Instead, use 516 * STREST dst, which doesn't require read access to dst. 517 * 518 * This code should perform better than a simple loop on modern, 519 * wide-issue mips processors because the code has fewer branches and 520 * more instruction-level parallelism. 521 */ 522#define bits t2 523 beqz len, .Ldone 524 ADD t1, dst, len # t1 is just past last byte of dst 525 li bits, 8*NBYTES 526 SLL rem, len, 3 # rem = number of bits to keep 527EXC( LOAD t0, 0(src), .Ll_exc) 528 SUB bits, bits, rem # bits = number of bits to discard 529 SHIFT_DISCARD t0, t0, bits 530EXC( STREST t0, -1(t1), .Ls_exc) 531 SHIFT_DISCARD_REVERT t0, t0, bits 532 .set reorder 533 ADDC(sum, t0) 534 b .Ldone 535 .set noreorder 536.Ldst_unaligned: 537 /* 538 * dst is unaligned 539 * t0 = src & ADDRMASK 540 * t1 = dst & ADDRMASK; T1 > 0 541 * len >= NBYTES 542 * 543 * Copy enough bytes to align dst 544 * Set match = (src and dst have same alignment) 545 */ 546#define match rem 547EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) 548 ADD t2, zero, NBYTES 549EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) 550 SUB t2, t2, t1 # t2 = number of bytes copied 551 xor match, t0, t1 552EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 553 SLL t4, t1, 3 # t4 = number of bits to discard 554 SHIFT_DISCARD t3, t3, t4 555 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ 556 ADDC(sum, t3) 557 beq len, t2, .Ldone 558 SUB len, len, t2 559 ADD dst, dst, t2 560 beqz match, .Lboth_aligned 561 ADD src, src, t2 562 563.Lsrc_unaligned_dst_aligned: 564 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter 565 beqz t0, .Lcleanup_src_unaligned 566 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES 5671: 568/* 569 * Avoid consecutive LD*'s to the same register since some mips 570 * implementations can't issue them in the same cycle. 571 * It's OK to load FIRST(N+1) before REST(N) because the two addresses 572 * are to the same unit (unless src is aligned, but it's not). 573 */ 574EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 575EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) 576 SUB len, len, 4*NBYTES 577EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 578EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) 579EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) 580EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) 581EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) 582EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) 583 ADD src, src, 4*NBYTES 584#ifdef CONFIG_CPU_SB1 585 nop # improves slotting 586#endif 587EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 588 ADDC(sum, t0) 589EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 590 ADDC(sum, t1) 591EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 592 ADDC(sum, t2) 593EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 594 ADDC(sum, t3) 595 .set reorder /* DADDI_WAR */ 596 ADD dst, dst, 4*NBYTES 597 bne len, rem, 1b 598 .set noreorder 599 600.Lcleanup_src_unaligned: 601 beqz len, .Ldone 602 and rem, len, NBYTES-1 # rem = len % NBYTES 603 beq rem, len, .Lcopy_bytes 604 nop 6051: 606EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 607EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 608 ADD src, src, NBYTES 609 SUB len, len, NBYTES 610EXC( STORE t0, 0(dst), .Ls_exc) 611 ADDC(sum, t0) 612 .set reorder /* DADDI_WAR */ 613 ADD dst, dst, NBYTES 614 bne len, rem, 1b 615 .set noreorder 616 617.Lcopy_bytes_checklen: 618 beqz len, .Ldone 619 nop 620.Lcopy_bytes: 621 /* 0 < len < NBYTES */ 622#ifdef CONFIG_CPU_LITTLE_ENDIAN 623#define SHIFT_START 0 624#define SHIFT_INC 8 625#else 626#define SHIFT_START 8*(NBYTES-1) 627#define SHIFT_INC -8 628#endif 629 move t2, zero # partial word 630 li t3, SHIFT_START # shift 631/* use .Ll_exc_copy here to return correct sum on fault */ 632#define COPY_BYTE(N) \ 633EXC( lbu t0, N(src), .Ll_exc_copy); \ 634 SUB len, len, 1; \ 635EXC( sb t0, N(dst), .Ls_exc); \ 636 SLLV t0, t0, t3; \ 637 addu t3, SHIFT_INC; \ 638 beqz len, .Lcopy_bytes_done; \ 639 or t2, t0 640 641 COPY_BYTE(0) 642 COPY_BYTE(1) 643#ifdef USE_DOUBLE 644 COPY_BYTE(2) 645 COPY_BYTE(3) 646 COPY_BYTE(4) 647 COPY_BYTE(5) 648#endif 649EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy) 650 SUB len, len, 1 651EXC( sb t0, NBYTES-2(dst), .Ls_exc) 652 SLLV t0, t0, t3 653 or t2, t0 654.Lcopy_bytes_done: 655 ADDC(sum, t2) 656.Ldone: 657 /* fold checksum */ 658#ifdef USE_DOUBLE 659 dsll32 v1, sum, 0 660 daddu sum, v1 661 sltu v1, sum, v1 662 dsra32 sum, sum, 0 663 addu sum, v1 664#endif 665 666 /* odd buffer alignment? */ 667 beqz odd, 1f 668 nop 669 sll v1, sum, 8 670 srl sum, sum, 8 671 or sum, v1 672 andi sum, 0xffff 6731: 674 .set reorder 675 ADDC32(sum, psum) 676 jr ra 677 .set noreorder 678 679.Ll_exc_copy: 680 /* 681 * Copy bytes from src until faulting load address (or until a 682 * lb faults) 683 * 684 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28) 685 * may be more than a byte beyond the last address. 686 * Hence, the lb below may get an exception. 687 * 688 * Assumes src < THREAD_BUADDR($28) 689 */ 690 LOAD t0, TI_TASK($28) 691 li t2, SHIFT_START 692 LOAD t0, THREAD_BUADDR(t0) 6931: 694EXC( lbu t1, 0(src), .Ll_exc) 695 ADD src, src, 1 696 sb t1, 0(dst) # can't fault -- we're copy_from_user 697 SLLV t1, t1, t2 698 addu t2, SHIFT_INC 699 ADDC(sum, t1) 700 .set reorder /* DADDI_WAR */ 701 ADD dst, dst, 1 702 bne src, t0, 1b 703 .set noreorder 704.Ll_exc: 705 LOAD t0, TI_TASK($28) 706 nop 707 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address 708 nop 709 SUB len, AT, t0 # len number of uncopied bytes 710 /* 711 * Here's where we rely on src and dst being incremented in tandem, 712 * See (3) above. 713 * dst += (fault addr - src) to put dst at first byte to clear 714 */ 715 ADD dst, t0 # compute start address in a1 716 SUB dst, src 717 /* 718 * Clear len bytes starting at dst. Can't call __bzero because it 719 * might modify len. An inefficient loop for these rare times... 720 */ 721 .set reorder /* DADDI_WAR */ 722 SUB src, len, 1 723 beqz len, .Ldone 724 .set noreorder 7251: sb zero, 0(dst) 726 ADD dst, dst, 1 727 .set push 728 .set noat 729#ifndef CONFIG_CPU_DADDI_WORKAROUNDS 730 bnez src, 1b 731 SUB src, src, 1 732#else 733 li v1, 1 734 bnez src, 1b 735 SUB src, src, v1 736#endif 737 li v1, -EFAULT 738 b .Ldone 739 sw v1, (errptr) 740 741.Ls_exc: 742 li v0, -1 /* invalid checksum */ 743 li v1, -EFAULT 744 jr ra 745 sw v1, (errptr) 746 .set pop 747 END(__csum_partial_copy_user) 748