1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Re-map IO memory to kernel address space so that we can access it. 4 * This is needed for high PCI addresses that aren't mapped in the 5 * 640k-1MB IO memory area on PC's 6 * 7 * (C) Copyright 1995 1996 Linus Torvalds 8 */ 9 #include <linux/vmalloc.h> 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/io.h> 13 #include <linux/export.h> 14 #include <asm/cacheflush.h> 15 16 #include "pgalloc-track.h" 17 18 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 19 static bool __ro_after_init iomap_max_page_shift = PAGE_SHIFT; 20 21 static int __init set_nohugeiomap(char *str) 22 { 23 iomap_max_page_shift = P4D_SHIFT; 24 return 0; 25 } 26 early_param("nohugeiomap", set_nohugeiomap); 27 #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 28 static const bool iomap_max_page_shift = PAGE_SHIFT; 29 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 30 31 static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 32 phys_addr_t phys_addr, pgprot_t prot, 33 pgtbl_mod_mask *mask) 34 { 35 pte_t *pte; 36 u64 pfn; 37 38 pfn = phys_addr >> PAGE_SHIFT; 39 pte = pte_alloc_kernel_track(pmd, addr, mask); 40 if (!pte) 41 return -ENOMEM; 42 do { 43 BUG_ON(!pte_none(*pte)); 44 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 45 pfn++; 46 } while (pte++, addr += PAGE_SIZE, addr != end); 47 *mask |= PGTBL_PTE_MODIFIED; 48 return 0; 49 } 50 51 static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 52 phys_addr_t phys_addr, pgprot_t prot, 53 unsigned int max_page_shift) 54 { 55 if (max_page_shift < PMD_SHIFT) 56 return 0; 57 58 if (!arch_vmap_pmd_supported(prot)) 59 return 0; 60 61 if ((end - addr) != PMD_SIZE) 62 return 0; 63 64 if (!IS_ALIGNED(addr, PMD_SIZE)) 65 return 0; 66 67 if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 68 return 0; 69 70 if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 71 return 0; 72 73 return pmd_set_huge(pmd, phys_addr, prot); 74 } 75 76 static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 77 phys_addr_t phys_addr, pgprot_t prot, 78 unsigned int max_page_shift, pgtbl_mod_mask *mask) 79 { 80 pmd_t *pmd; 81 unsigned long next; 82 83 pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 84 if (!pmd) 85 return -ENOMEM; 86 do { 87 next = pmd_addr_end(addr, end); 88 89 if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, 90 max_page_shift)) { 91 *mask |= PGTBL_PMD_MODIFIED; 92 continue; 93 } 94 95 if (vmap_pte_range(pmd, addr, next, phys_addr, prot, mask)) 96 return -ENOMEM; 97 } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 98 return 0; 99 } 100 101 static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 102 phys_addr_t phys_addr, pgprot_t prot, 103 unsigned int max_page_shift) 104 { 105 if (max_page_shift < PUD_SHIFT) 106 return 0; 107 108 if (!arch_vmap_pud_supported(prot)) 109 return 0; 110 111 if ((end - addr) != PUD_SIZE) 112 return 0; 113 114 if (!IS_ALIGNED(addr, PUD_SIZE)) 115 return 0; 116 117 if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 118 return 0; 119 120 if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 121 return 0; 122 123 return pud_set_huge(pud, phys_addr, prot); 124 } 125 126 static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 127 phys_addr_t phys_addr, pgprot_t prot, 128 unsigned int max_page_shift, pgtbl_mod_mask *mask) 129 { 130 pud_t *pud; 131 unsigned long next; 132 133 pud = pud_alloc_track(&init_mm, p4d, addr, mask); 134 if (!pud) 135 return -ENOMEM; 136 do { 137 next = pud_addr_end(addr, end); 138 139 if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, 140 max_page_shift)) { 141 *mask |= PGTBL_PUD_MODIFIED; 142 continue; 143 } 144 145 if (vmap_pmd_range(pud, addr, next, phys_addr, prot, 146 max_page_shift, mask)) 147 return -ENOMEM; 148 } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 149 return 0; 150 } 151 152 static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 153 phys_addr_t phys_addr, pgprot_t prot, 154 unsigned int max_page_shift) 155 { 156 if (max_page_shift < P4D_SHIFT) 157 return 0; 158 159 if (!arch_vmap_p4d_supported(prot)) 160 return 0; 161 162 if ((end - addr) != P4D_SIZE) 163 return 0; 164 165 if (!IS_ALIGNED(addr, P4D_SIZE)) 166 return 0; 167 168 if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 169 return 0; 170 171 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 172 return 0; 173 174 return p4d_set_huge(p4d, phys_addr, prot); 175 } 176 177 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 178 phys_addr_t phys_addr, pgprot_t prot, 179 unsigned int max_page_shift, pgtbl_mod_mask *mask) 180 { 181 p4d_t *p4d; 182 unsigned long next; 183 184 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 185 if (!p4d) 186 return -ENOMEM; 187 do { 188 next = p4d_addr_end(addr, end); 189 190 if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, 191 max_page_shift)) { 192 *mask |= PGTBL_P4D_MODIFIED; 193 continue; 194 } 195 196 if (vmap_pud_range(p4d, addr, next, phys_addr, prot, 197 max_page_shift, mask)) 198 return -ENOMEM; 199 } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 200 return 0; 201 } 202 203 static int vmap_range(unsigned long addr, unsigned long end, 204 phys_addr_t phys_addr, pgprot_t prot, 205 unsigned int max_page_shift) 206 { 207 pgd_t *pgd; 208 unsigned long start; 209 unsigned long next; 210 int err; 211 pgtbl_mod_mask mask = 0; 212 213 might_sleep(); 214 BUG_ON(addr >= end); 215 216 start = addr; 217 pgd = pgd_offset_k(addr); 218 do { 219 next = pgd_addr_end(addr, end); 220 err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, 221 max_page_shift, &mask); 222 if (err) 223 break; 224 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 225 226 flush_cache_vmap(start, end); 227 228 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 229 arch_sync_kernel_mappings(start, end); 230 231 return err; 232 } 233 234 int ioremap_page_range(unsigned long addr, 235 unsigned long end, phys_addr_t phys_addr, pgprot_t prot) 236 { 237 return vmap_range(addr, end, phys_addr, prot, iomap_max_page_shift); 238 } 239 240 #ifdef CONFIG_GENERIC_IOREMAP 241 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) 242 { 243 unsigned long offset, vaddr; 244 phys_addr_t last_addr; 245 struct vm_struct *area; 246 247 /* Disallow wrap-around or zero size */ 248 last_addr = addr + size - 1; 249 if (!size || last_addr < addr) 250 return NULL; 251 252 /* Page-align mappings */ 253 offset = addr & (~PAGE_MASK); 254 addr -= offset; 255 size = PAGE_ALIGN(size + offset); 256 257 area = get_vm_area_caller(size, VM_IOREMAP, 258 __builtin_return_address(0)); 259 if (!area) 260 return NULL; 261 vaddr = (unsigned long)area->addr; 262 263 if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { 264 free_vm_area(area); 265 return NULL; 266 } 267 268 return (void __iomem *)(vaddr + offset); 269 } 270 EXPORT_SYMBOL(ioremap_prot); 271 272 void iounmap(volatile void __iomem *addr) 273 { 274 vunmap((void *)((unsigned long)addr & PAGE_MASK)); 275 } 276 EXPORT_SYMBOL(iounmap); 277 #endif /* CONFIG_GENERIC_IOREMAP */ 278