1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Re-map IO memory to kernel address space so that we can access it. 4 * This is needed for high PCI addresses that aren't mapped in the 5 * 640k-1MB IO memory area on PC's 6 * 7 * (C) Copyright 1995 1996 Linus Torvalds 8 */ 9 #include <linux/vmalloc.h> 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/io.h> 13 #include <linux/export.h> 14 #include <asm/cacheflush.h> 15 16 #include "pgalloc-track.h" 17 18 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 19 static int __read_mostly ioremap_p4d_capable; 20 static int __read_mostly ioremap_pud_capable; 21 static int __read_mostly ioremap_pmd_capable; 22 static int __read_mostly ioremap_huge_disabled; 23 24 static int __init set_nohugeiomap(char *str) 25 { 26 ioremap_huge_disabled = 1; 27 return 0; 28 } 29 early_param("nohugeiomap", set_nohugeiomap); 30 31 void __init ioremap_huge_init(void) 32 { 33 if (!ioremap_huge_disabled) { 34 if (arch_ioremap_p4d_supported()) 35 ioremap_p4d_capable = 1; 36 if (arch_ioremap_pud_supported()) 37 ioremap_pud_capable = 1; 38 if (arch_ioremap_pmd_supported()) 39 ioremap_pmd_capable = 1; 40 } 41 } 42 43 static inline int ioremap_p4d_enabled(void) 44 { 45 return ioremap_p4d_capable; 46 } 47 48 static inline int ioremap_pud_enabled(void) 49 { 50 return ioremap_pud_capable; 51 } 52 53 static inline int ioremap_pmd_enabled(void) 54 { 55 return ioremap_pmd_capable; 56 } 57 58 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 59 static inline int ioremap_p4d_enabled(void) { return 0; } 60 static inline int ioremap_pud_enabled(void) { return 0; } 61 static inline int ioremap_pmd_enabled(void) { return 0; } 62 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 63 64 static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 65 phys_addr_t phys_addr, pgprot_t prot, 66 pgtbl_mod_mask *mask) 67 { 68 pte_t *pte; 69 u64 pfn; 70 71 pfn = phys_addr >> PAGE_SHIFT; 72 pte = pte_alloc_kernel_track(pmd, addr, mask); 73 if (!pte) 74 return -ENOMEM; 75 do { 76 BUG_ON(!pte_none(*pte)); 77 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 78 pfn++; 79 } while (pte++, addr += PAGE_SIZE, addr != end); 80 *mask |= PGTBL_PTE_MODIFIED; 81 return 0; 82 } 83 84 static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 85 phys_addr_t phys_addr, pgprot_t prot) 86 { 87 if (!ioremap_pmd_enabled()) 88 return 0; 89 90 if ((end - addr) != PMD_SIZE) 91 return 0; 92 93 if (!IS_ALIGNED(addr, PMD_SIZE)) 94 return 0; 95 96 if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 97 return 0; 98 99 if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 100 return 0; 101 102 return pmd_set_huge(pmd, phys_addr, prot); 103 } 104 105 static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 106 phys_addr_t phys_addr, pgprot_t prot, 107 pgtbl_mod_mask *mask) 108 { 109 pmd_t *pmd; 110 unsigned long next; 111 112 pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 113 if (!pmd) 114 return -ENOMEM; 115 do { 116 next = pmd_addr_end(addr, end); 117 118 if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) { 119 *mask |= PGTBL_PMD_MODIFIED; 120 continue; 121 } 122 123 if (vmap_pte_range(pmd, addr, next, phys_addr, prot, mask)) 124 return -ENOMEM; 125 } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 126 return 0; 127 } 128 129 static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 130 phys_addr_t phys_addr, pgprot_t prot) 131 { 132 if (!ioremap_pud_enabled()) 133 return 0; 134 135 if ((end - addr) != PUD_SIZE) 136 return 0; 137 138 if (!IS_ALIGNED(addr, PUD_SIZE)) 139 return 0; 140 141 if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 142 return 0; 143 144 if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 145 return 0; 146 147 return pud_set_huge(pud, phys_addr, prot); 148 } 149 150 static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 151 phys_addr_t phys_addr, pgprot_t prot, 152 pgtbl_mod_mask *mask) 153 { 154 pud_t *pud; 155 unsigned long next; 156 157 pud = pud_alloc_track(&init_mm, p4d, addr, mask); 158 if (!pud) 159 return -ENOMEM; 160 do { 161 next = pud_addr_end(addr, end); 162 163 if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot)) { 164 *mask |= PGTBL_PUD_MODIFIED; 165 continue; 166 } 167 168 if (vmap_pmd_range(pud, addr, next, phys_addr, prot, mask)) 169 return -ENOMEM; 170 } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 171 return 0; 172 } 173 174 static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 175 phys_addr_t phys_addr, pgprot_t prot) 176 { 177 if (!ioremap_p4d_enabled()) 178 return 0; 179 180 if ((end - addr) != P4D_SIZE) 181 return 0; 182 183 if (!IS_ALIGNED(addr, P4D_SIZE)) 184 return 0; 185 186 if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 187 return 0; 188 189 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 190 return 0; 191 192 return p4d_set_huge(p4d, phys_addr, prot); 193 } 194 195 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 196 phys_addr_t phys_addr, pgprot_t prot, 197 pgtbl_mod_mask *mask) 198 { 199 p4d_t *p4d; 200 unsigned long next; 201 202 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 203 if (!p4d) 204 return -ENOMEM; 205 do { 206 next = p4d_addr_end(addr, end); 207 208 if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) { 209 *mask |= PGTBL_P4D_MODIFIED; 210 continue; 211 } 212 213 if (vmap_pud_range(p4d, addr, next, phys_addr, prot, mask)) 214 return -ENOMEM; 215 } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 216 return 0; 217 } 218 219 static int vmap_range(unsigned long addr, unsigned long end, 220 phys_addr_t phys_addr, pgprot_t prot) 221 { 222 pgd_t *pgd; 223 unsigned long start; 224 unsigned long next; 225 int err; 226 pgtbl_mod_mask mask = 0; 227 228 might_sleep(); 229 BUG_ON(addr >= end); 230 231 start = addr; 232 pgd = pgd_offset_k(addr); 233 do { 234 next = pgd_addr_end(addr, end); 235 err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, &mask); 236 if (err) 237 break; 238 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 239 240 flush_cache_vmap(start, end); 241 242 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 243 arch_sync_kernel_mappings(start, end); 244 245 return err; 246 } 247 248 int ioremap_page_range(unsigned long addr, 249 unsigned long end, phys_addr_t phys_addr, pgprot_t prot) 250 { 251 return vmap_range(addr, end, phys_addr, prot); 252 } 253 254 #ifdef CONFIG_GENERIC_IOREMAP 255 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) 256 { 257 unsigned long offset, vaddr; 258 phys_addr_t last_addr; 259 struct vm_struct *area; 260 261 /* Disallow wrap-around or zero size */ 262 last_addr = addr + size - 1; 263 if (!size || last_addr < addr) 264 return NULL; 265 266 /* Page-align mappings */ 267 offset = addr & (~PAGE_MASK); 268 addr -= offset; 269 size = PAGE_ALIGN(size + offset); 270 271 area = get_vm_area_caller(size, VM_IOREMAP, 272 __builtin_return_address(0)); 273 if (!area) 274 return NULL; 275 vaddr = (unsigned long)area->addr; 276 277 if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { 278 free_vm_area(area); 279 return NULL; 280 } 281 282 return (void __iomem *)(vaddr + offset); 283 } 284 EXPORT_SYMBOL(ioremap_prot); 285 286 void iounmap(volatile void __iomem *addr) 287 { 288 vunmap((void *)((unsigned long)addr & PAGE_MASK)); 289 } 290 EXPORT_SYMBOL(iounmap); 291 #endif /* CONFIG_GENERIC_IOREMAP */ 292