1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions 4 * 5 * Module based on arm64/crypto/crc32-arm.c 6 * 7 * Copyright (C) 2014 Linaro Ltd <[email protected]> 8 * Copyright (C) 2018 MIPS Tech, LLC 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <linux/crc32.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <asm/mipsregs.h> 17 #include <linux/unaligned.h> 18 19 #ifndef TOOLCHAIN_SUPPORTS_CRC 20 #define _ASM_SET_CRC(OP, SZ, TYPE) \ 21 _ASM_MACRO_3R(OP, rt, rs, rt2, \ 22 ".ifnc \\rt, \\rt2\n\t" \ 23 ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \ 24 ".endif\n\t" \ 25 _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \ 26 ((SZ) << 6) | ((TYPE) << 8)) \ 27 _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \ 28 ((SZ) << 14) | ((TYPE) << 3))) 29 #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t" 30 #else /* !TOOLCHAIN_SUPPORTS_CRC */ 31 #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t" 32 #define _ASM_UNSET_CRC(op, SZ, TYPE) 33 #endif 34 35 #define __CRC32(crc, value, op, SZ, TYPE) \ 36 do { \ 37 __asm__ __volatile__( \ 38 ".set push\n\t" \ 39 _ASM_SET_CRC(op, SZ, TYPE) \ 40 #op " %0, %1, %0\n\t" \ 41 _ASM_UNSET_CRC(op, SZ, TYPE) \ 42 ".set pop" \ 43 : "+r" (crc) \ 44 : "r" (value)); \ 45 } while (0) 46 47 #define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0) 48 #define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0) 49 #define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0) 50 #define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0) 51 #define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1) 52 #define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1) 53 #define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1) 54 #define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1) 55 56 #define _CRC32(crc, value, size, op) \ 57 _CRC32_##op##size(crc, value) 58 59 #define CRC32(crc, value, size) \ 60 _CRC32(crc, value, size, crc32) 61 62 #define CRC32C(crc, value, size) \ 63 _CRC32(crc, value, size, crc32c) 64 65 static DEFINE_STATIC_KEY_FALSE(have_crc32); 66 67 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 68 { 69 if (!static_branch_likely(&have_crc32)) 70 return crc32_le_base(crc, p, len); 71 72 if (IS_ENABLED(CONFIG_64BIT)) { 73 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 74 u64 value = get_unaligned_le64(p); 75 76 CRC32(crc, value, d); 77 } 78 79 if (len & sizeof(u32)) { 80 u32 value = get_unaligned_le32(p); 81 82 CRC32(crc, value, w); 83 p += sizeof(u32); 84 } 85 } else { 86 for (; len >= sizeof(u32); len -= sizeof(u32)) { 87 u32 value = get_unaligned_le32(p); 88 89 CRC32(crc, value, w); 90 p += sizeof(u32); 91 } 92 } 93 94 if (len & sizeof(u16)) { 95 u16 value = get_unaligned_le16(p); 96 97 CRC32(crc, value, h); 98 p += sizeof(u16); 99 } 100 101 if (len & sizeof(u8)) { 102 u8 value = *p++; 103 104 CRC32(crc, value, b); 105 } 106 107 return crc; 108 } 109 EXPORT_SYMBOL(crc32_le_arch); 110 111 u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 112 { 113 if (!static_branch_likely(&have_crc32)) 114 return crc32c_base(crc, p, len); 115 116 if (IS_ENABLED(CONFIG_64BIT)) { 117 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 118 u64 value = get_unaligned_le64(p); 119 120 CRC32C(crc, value, d); 121 } 122 123 if (len & sizeof(u32)) { 124 u32 value = get_unaligned_le32(p); 125 126 CRC32C(crc, value, w); 127 p += sizeof(u32); 128 } 129 } else { 130 for (; len >= sizeof(u32); len -= sizeof(u32)) { 131 u32 value = get_unaligned_le32(p); 132 133 CRC32C(crc, value, w); 134 p += sizeof(u32); 135 } 136 } 137 138 if (len & sizeof(u16)) { 139 u16 value = get_unaligned_le16(p); 140 141 CRC32C(crc, value, h); 142 p += sizeof(u16); 143 } 144 145 if (len & sizeof(u8)) { 146 u8 value = *p++; 147 148 CRC32C(crc, value, b); 149 } 150 return crc; 151 } 152 EXPORT_SYMBOL(crc32c_arch); 153 154 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 155 { 156 return crc32_be_base(crc, p, len); 157 } 158 EXPORT_SYMBOL(crc32_be_arch); 159 160 static int __init crc32_mips_init(void) 161 { 162 if (cpu_have_feature(cpu_feature(MIPS_CRC32))) 163 static_branch_enable(&have_crc32); 164 return 0; 165 } 166 arch_initcall(crc32_mips_init); 167 168 static void __exit crc32_mips_exit(void) 169 { 170 } 171 module_exit(crc32_mips_exit); 172 173 u32 crc32_optimizations(void) 174 { 175 if (static_key_enabled(&have_crc32)) 176 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 177 return 0; 178 } 179 EXPORT_SYMBOL(crc32_optimizations); 180 181 MODULE_AUTHOR("Marcin Nowakowski <[email protected]"); 182 MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions"); 183 MODULE_LICENSE("GPL v2"); 184