xref: /linux-6.15/include/linux/bitmap.h (revision 4e23eeeb)
1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
21da177e4SLinus Torvalds #ifndef __LINUX_BITMAP_H
31da177e4SLinus Torvalds #define __LINUX_BITMAP_H
41da177e4SLinus Torvalds 
51da177e4SLinus Torvalds #ifndef __ASSEMBLY__
61da177e4SLinus Torvalds 
708c5188eSAndy Shevchenko #include <linux/align.h>
81da177e4SLinus Torvalds #include <linux/bitops.h>
947d8c156SYury Norov #include <linux/find.h>
1008c5188eSAndy Shevchenko #include <linux/limits.h>
11c13656b9SBartosz Golaszewski #include <linux/string.h>
12c13656b9SBartosz Golaszewski #include <linux/types.h>
131da177e4SLinus Torvalds 
14e829c2e4SBartosz Golaszewski struct device;
15e829c2e4SBartosz Golaszewski 
161da177e4SLinus Torvalds /*
171da177e4SLinus Torvalds  * bitmaps provide bit arrays that consume one or more unsigned
181da177e4SLinus Torvalds  * longs.  The bitmap interface and available operations are listed
191da177e4SLinus Torvalds  * here, in bitmap.h
201da177e4SLinus Torvalds  *
211da177e4SLinus Torvalds  * Function implementations generic to all architectures are in
221da177e4SLinus Torvalds  * lib/bitmap.c.  Functions implementations that are architecture
231da177e4SLinus Torvalds  * specific are in various include/asm-<arch>/bitops.h headers
241da177e4SLinus Torvalds  * and other arch/<arch> specific files.
251da177e4SLinus Torvalds  *
261da177e4SLinus Torvalds  * See lib/bitmap.c for more details.
271da177e4SLinus Torvalds  */
281da177e4SLinus Torvalds 
297d7363e4SRandy Dunlap /**
307d7363e4SRandy Dunlap  * DOC: bitmap overview
317d7363e4SRandy Dunlap  *
321da177e4SLinus Torvalds  * The available bitmap operations and their rough meaning in the
331da177e4SLinus Torvalds  * case that the bitmap is a single unsigned long are thus:
341da177e4SLinus Torvalds  *
3541e7b166SRasmus Villemoes  * The generated code is more efficient when nbits is known at
3641e7b166SRasmus Villemoes  * compile-time and at most BITS_PER_LONG.
3708cd3657SAndi Kleen  *
387d7363e4SRandy Dunlap  * ::
397d7363e4SRandy Dunlap  *
401da177e4SLinus Torvalds  *  bitmap_zero(dst, nbits)                     *dst = 0UL
411da177e4SLinus Torvalds  *  bitmap_fill(dst, nbits)                     *dst = ~0UL
421da177e4SLinus Torvalds  *  bitmap_copy(dst, src, nbits)                *dst = *src
431da177e4SLinus Torvalds  *  bitmap_and(dst, src1, src2, nbits)          *dst = *src1 & *src2
441da177e4SLinus Torvalds  *  bitmap_or(dst, src1, src2, nbits)           *dst = *src1 | *src2
451da177e4SLinus Torvalds  *  bitmap_xor(dst, src1, src2, nbits)          *dst = *src1 ^ *src2
461da177e4SLinus Torvalds  *  bitmap_andnot(dst, src1, src2, nbits)       *dst = *src1 & ~(*src2)
471da177e4SLinus Torvalds  *  bitmap_complement(dst, src, nbits)          *dst = ~(*src)
481da177e4SLinus Torvalds  *  bitmap_equal(src1, src2, nbits)             Are *src1 and *src2 equal?
491da177e4SLinus Torvalds  *  bitmap_intersects(src1, src2, nbits)        Do *src1 and *src2 overlap?
501da177e4SLinus Torvalds  *  bitmap_subset(src1, src2, nbits)            Is *src1 a subset of *src2?
511da177e4SLinus Torvalds  *  bitmap_empty(src, nbits)                    Are all bits zero in *src?
521da177e4SLinus Torvalds  *  bitmap_full(src, nbits)                     Are all bits set in *src?
531da177e4SLinus Torvalds  *  bitmap_weight(src, nbits)                   Hamming Weight: number set bits
54c1a2a962SAkinobu Mita  *  bitmap_set(dst, pos, nbits)                 Set specified bit area
55c1a2a962SAkinobu Mita  *  bitmap_clear(dst, pos, nbits)               Clear specified bit area
56c1a2a962SAkinobu Mita  *  bitmap_find_next_zero_area(buf, len, pos, n, mask)  Find bit free area
57780d2a9cSWolfram Sang  *  bitmap_find_next_zero_area_off(buf, len, pos, n, mask, mask_off)  as above
581da177e4SLinus Torvalds  *  bitmap_shift_right(dst, src, n, nbits)      *dst = *src >> n
591da177e4SLinus Torvalds  *  bitmap_shift_left(dst, src, n, nbits)       *dst = *src << n
6020927671SStefano Brivio  *  bitmap_cut(dst, src, first, n, nbits)       Cut n bits from first, copy rest
6130544ed5SAndy Shevchenko  *  bitmap_replace(dst, old, new, mask, nbits)  *dst = (*old & ~(*mask)) | (*new & *mask)
62fb5eeeeeSPaul Jackson  *  bitmap_remap(dst, src, old, new, nbits)     *dst = map(old, new)(src)
63fb5eeeeeSPaul Jackson  *  bitmap_bitremap(oldbit, old, new, nbits)    newbit = map(old, new)(oldbit)
647ea931c9SPaul Jackson  *  bitmap_onto(dst, orig, relmap, nbits)       *dst = orig relative to relmap
657ea931c9SPaul Jackson  *  bitmap_fold(dst, orig, sz, nbits)           dst bits = orig bits mod sz
6601a3ee2bSReinette Chatre  *  bitmap_parse(buf, buflen, dst, nbits)       Parse bitmap dst from kernel buf
6701a3ee2bSReinette Chatre  *  bitmap_parse_user(ubuf, ulen, dst, nbits)   Parse bitmap dst from user buf
684b060420SMike Travis  *  bitmap_parselist(buf, dst, nbits)           Parse bitmap dst from kernel buf
694b060420SMike Travis  *  bitmap_parselist_user(buf, dst, nbits)      Parse bitmap dst from user buf
7087e24802SPaul Jackson  *  bitmap_find_free_region(bitmap, bits, order)  Find and allocate bit region
7187e24802SPaul Jackson  *  bitmap_release_region(bitmap, pos, order)   Free specified bit region
7287e24802SPaul Jackson  *  bitmap_allocate_region(bitmap, pos, order)  Allocate specified bit region
73c724f193SYury Norov  *  bitmap_from_arr32(dst, buf, nbits)          Copy nbits from u32[] buf to dst
74ba1afa67SQu Wenruo  *  bitmap_from_arr64(dst, buf, nbits)          Copy nbits from u64[] buf to dst
75c724f193SYury Norov  *  bitmap_to_arr32(buf, src, nbits)            Copy nbits from buf to u32[] dst
760a97953fSYury Norov  *  bitmap_to_arr64(buf, src, nbits)            Copy nbits from buf to u64[] dst
77169c474fSWilliam Breathitt Gray  *  bitmap_get_value8(map, start)               Get 8bit value from map at start
78169c474fSWilliam Breathitt Gray  *  bitmap_set_value8(map, value, start)        Set 8bit value to map at start
797d7363e4SRandy Dunlap  *
80334cfa48SAndy Shevchenko  * Note, bitmap_zero() and bitmap_fill() operate over the region of
81334cfa48SAndy Shevchenko  * unsigned longs, that is, bits behind bitmap till the unsigned long
82334cfa48SAndy Shevchenko  * boundary will be zeroed or filled as well. Consider to use
83334cfa48SAndy Shevchenko  * bitmap_clear() or bitmap_set() to make explicit zeroing or filling
84334cfa48SAndy Shevchenko  * respectively.
851da177e4SLinus Torvalds  */
861da177e4SLinus Torvalds 
877d7363e4SRandy Dunlap /**
887d7363e4SRandy Dunlap  * DOC: bitmap bitops
897d7363e4SRandy Dunlap  *
907d7363e4SRandy Dunlap  * Also the following operations in asm/bitops.h apply to bitmaps.::
911da177e4SLinus Torvalds  *
921da177e4SLinus Torvalds  *  set_bit(bit, addr)                  *addr |= bit
931da177e4SLinus Torvalds  *  clear_bit(bit, addr)                *addr &= ~bit
941da177e4SLinus Torvalds  *  change_bit(bit, addr)               *addr ^= bit
951da177e4SLinus Torvalds  *  test_bit(bit, addr)                 Is bit set in *addr?
961da177e4SLinus Torvalds  *  test_and_set_bit(bit, addr)         Set bit and return old value
971da177e4SLinus Torvalds  *  test_and_clear_bit(bit, addr)       Clear bit and return old value
981da177e4SLinus Torvalds  *  test_and_change_bit(bit, addr)      Change bit and return old value
991da177e4SLinus Torvalds  *  find_first_zero_bit(addr, nbits)    Position first zero bit in *addr
1001da177e4SLinus Torvalds  *  find_first_bit(addr, nbits)         Position first set bit in *addr
1010ade34c3SClement Courbet  *  find_next_zero_bit(addr, nbits, bit)
1020ade34c3SClement Courbet  *                                      Position next zero bit in *addr >= bit
1031da177e4SLinus Torvalds  *  find_next_bit(addr, nbits, bit)     Position next set bit in *addr >= bit
1040ade34c3SClement Courbet  *  find_next_and_bit(addr1, addr2, nbits, bit)
1050ade34c3SClement Courbet  *                                      Same as find_next_bit, but in
1060ade34c3SClement Courbet  *                                      (*addr1 & *addr2)
1077d7363e4SRandy Dunlap  *
1081da177e4SLinus Torvalds  */
1091da177e4SLinus Torvalds 
1107d7363e4SRandy Dunlap /**
1117d7363e4SRandy Dunlap  * DOC: declare bitmap
1121da177e4SLinus Torvalds  * The DECLARE_BITMAP(name,bits) macro, in linux/types.h, can be used
1131da177e4SLinus Torvalds  * to declare an array named 'name' of just enough unsigned longs to
1141da177e4SLinus Torvalds  * contain all bit positions from 0 to 'bits' - 1.
1151da177e4SLinus Torvalds  */
1161da177e4SLinus Torvalds 
1171da177e4SLinus Torvalds /*
118c42b65e3SAndy Shevchenko  * Allocation and deallocation of bitmap.
119c42b65e3SAndy Shevchenko  * Provided in lib/bitmap.c to avoid circular dependency.
120c42b65e3SAndy Shevchenko  */
12198635b29SBartosz Golaszewski unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags);
12298635b29SBartosz Golaszewski unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags);
1237529cc7fSTariq Toukan unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node);
1247529cc7fSTariq Toukan unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node);
12598635b29SBartosz Golaszewski void bitmap_free(const unsigned long *bitmap);
126c42b65e3SAndy Shevchenko 
127e829c2e4SBartosz Golaszewski /* Managed variants of the above. */
128e829c2e4SBartosz Golaszewski unsigned long *devm_bitmap_alloc(struct device *dev,
129e829c2e4SBartosz Golaszewski 				 unsigned int nbits, gfp_t flags);
130e829c2e4SBartosz Golaszewski unsigned long *devm_bitmap_zalloc(struct device *dev,
131e829c2e4SBartosz Golaszewski 				  unsigned int nbits, gfp_t flags);
132e829c2e4SBartosz Golaszewski 
133c42b65e3SAndy Shevchenko /*
1341da177e4SLinus Torvalds  * lib/bitmap.c provides these functions:
1351da177e4SLinus Torvalds  */
1361da177e4SLinus Torvalds 
137005f1700SKees Cook bool __bitmap_equal(const unsigned long *bitmap1,
1385e068069SRasmus Villemoes 		    const unsigned long *bitmap2, unsigned int nbits);
13998635b29SBartosz Golaszewski bool __pure __bitmap_or_equal(const unsigned long *src1,
140b9fa6442SThomas Gleixner 			      const unsigned long *src2,
141b9fa6442SThomas Gleixner 			      const unsigned long *src3,
142b9fa6442SThomas Gleixner 			      unsigned int nbits);
14398635b29SBartosz Golaszewski void __bitmap_complement(unsigned long *dst, const unsigned long *src,
1443d6684f4SRasmus Villemoes 			 unsigned int nbits);
14598635b29SBartosz Golaszewski void __bitmap_shift_right(unsigned long *dst, const unsigned long *src,
1462fbad299SRasmus Villemoes 			  unsigned int shift, unsigned int nbits);
14798635b29SBartosz Golaszewski void __bitmap_shift_left(unsigned long *dst, const unsigned long *src,
148dba94c25SRasmus Villemoes 			 unsigned int shift, unsigned int nbits);
14998635b29SBartosz Golaszewski void bitmap_cut(unsigned long *dst, const unsigned long *src,
15098635b29SBartosz Golaszewski 		unsigned int first, unsigned int cut, unsigned int nbits);
151e2863a78SYury Norov bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
1522f9305ebSRasmus Villemoes 		 const unsigned long *bitmap2, unsigned int nbits);
15398635b29SBartosz Golaszewski void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
1542f9305ebSRasmus Villemoes 		 const unsigned long *bitmap2, unsigned int nbits);
15598635b29SBartosz Golaszewski void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
1562f9305ebSRasmus Villemoes 		  const unsigned long *bitmap2, unsigned int nbits);
157e2863a78SYury Norov bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
1582f9305ebSRasmus Villemoes 		    const unsigned long *bitmap2, unsigned int nbits);
15998635b29SBartosz Golaszewski void __bitmap_replace(unsigned long *dst,
16030544ed5SAndy Shevchenko 		      const unsigned long *old, const unsigned long *new,
16130544ed5SAndy Shevchenko 		      const unsigned long *mask, unsigned int nbits);
162005f1700SKees Cook bool __bitmap_intersects(const unsigned long *bitmap1,
1636dfe9799SRasmus Villemoes 			 const unsigned long *bitmap2, unsigned int nbits);
164005f1700SKees Cook bool __bitmap_subset(const unsigned long *bitmap1,
1655be20213SRasmus Villemoes 		     const unsigned long *bitmap2, unsigned int nbits);
1664e23eeebSLinus Torvalds unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);
16798635b29SBartosz Golaszewski void __bitmap_set(unsigned long *map, unsigned int start, int len);
16898635b29SBartosz Golaszewski void __bitmap_clear(unsigned long *map, unsigned int start, int len);
1695e19b013SMichal Nazarewicz 
17098635b29SBartosz Golaszewski unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
171c1a2a962SAkinobu Mita 					     unsigned long size,
172c1a2a962SAkinobu Mita 					     unsigned long start,
173c1a2a962SAkinobu Mita 					     unsigned int nr,
1745e19b013SMichal Nazarewicz 					     unsigned long align_mask,
1755e19b013SMichal Nazarewicz 					     unsigned long align_offset);
1765e19b013SMichal Nazarewicz 
1775e19b013SMichal Nazarewicz /**
1785e19b013SMichal Nazarewicz  * bitmap_find_next_zero_area - find a contiguous aligned zero area
1795e19b013SMichal Nazarewicz  * @map: The address to base the search on
1805e19b013SMichal Nazarewicz  * @size: The bitmap size in bits
1815e19b013SMichal Nazarewicz  * @start: The bitnumber to start searching at
1825e19b013SMichal Nazarewicz  * @nr: The number of zeroed bits we're looking for
1835e19b013SMichal Nazarewicz  * @align_mask: Alignment mask for zero area
1845e19b013SMichal Nazarewicz  *
1855e19b013SMichal Nazarewicz  * The @align_mask should be one less than a power of 2; the effect is that
1865e19b013SMichal Nazarewicz  * the bit offset of all zero areas this function finds is multiples of that
1875e19b013SMichal Nazarewicz  * power of 2. A @align_mask of 0 means no alignment is required.
1885e19b013SMichal Nazarewicz  */
1895e19b013SMichal Nazarewicz static inline unsigned long
1905e19b013SMichal Nazarewicz bitmap_find_next_zero_area(unsigned long *map,
1915e19b013SMichal Nazarewicz 			   unsigned long size,
1925e19b013SMichal Nazarewicz 			   unsigned long start,
1935e19b013SMichal Nazarewicz 			   unsigned int nr,
1945e19b013SMichal Nazarewicz 			   unsigned long align_mask)
1955e19b013SMichal Nazarewicz {
1965e19b013SMichal Nazarewicz 	return bitmap_find_next_zero_area_off(map, size, start, nr,
1975e19b013SMichal Nazarewicz 					      align_mask, 0);
1985e19b013SMichal Nazarewicz }
199c1a2a962SAkinobu Mita 
20098635b29SBartosz Golaszewski int bitmap_parse(const char *buf, unsigned int buflen,
20101a3ee2bSReinette Chatre 			unsigned long *dst, int nbits);
20298635b29SBartosz Golaszewski int bitmap_parse_user(const char __user *ubuf, unsigned int ulen,
2031da177e4SLinus Torvalds 			unsigned long *dst, int nbits);
20498635b29SBartosz Golaszewski int bitmap_parselist(const char *buf, unsigned long *maskp,
2051da177e4SLinus Torvalds 			int nmaskbits);
20698635b29SBartosz Golaszewski int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen,
2074b060420SMike Travis 			unsigned long *dst, int nbits);
20898635b29SBartosz Golaszewski void bitmap_remap(unsigned long *dst, const unsigned long *src,
2099814ec13SRasmus Villemoes 		const unsigned long *old, const unsigned long *new, unsigned int nbits);
21098635b29SBartosz Golaszewski int bitmap_bitremap(int oldbit,
211fb5eeeeeSPaul Jackson 		const unsigned long *old, const unsigned long *new, int bits);
21298635b29SBartosz Golaszewski void bitmap_onto(unsigned long *dst, const unsigned long *orig,
213eb569883SRasmus Villemoes 		const unsigned long *relmap, unsigned int bits);
21498635b29SBartosz Golaszewski void bitmap_fold(unsigned long *dst, const unsigned long *orig,
215b26ad583SRasmus Villemoes 		unsigned int sz, unsigned int nbits);
21698635b29SBartosz Golaszewski int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order);
21798635b29SBartosz Golaszewski void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order);
21898635b29SBartosz Golaszewski int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order);
2193aa56885SYury Norov 
220e8f24278SRasmus Villemoes #ifdef __BIG_ENDIAN
22198635b29SBartosz Golaszewski void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int nbits);
222e8f24278SRasmus Villemoes #else
223e8f24278SRasmus Villemoes #define bitmap_copy_le bitmap_copy
224e8f24278SRasmus Villemoes #endif
22598635b29SBartosz Golaszewski unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits);
22698635b29SBartosz Golaszewski int bitmap_print_to_pagebuf(bool list, char *buf,
2275aaba363SSudeep Holla 				   const unsigned long *maskp, int nmaskbits);
2281da177e4SLinus Torvalds 
2291fae5629STian Tao extern int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp,
2301fae5629STian Tao 				      int nmaskbits, loff_t off, size_t count);
2311fae5629STian Tao 
2321fae5629STian Tao extern int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp,
2331fae5629STian Tao 				      int nmaskbits, loff_t off, size_t count);
2341fae5629STian Tao 
23589c1e79eSRasmus Villemoes #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1)))
23689c1e79eSRasmus Villemoes #define BITMAP_LAST_WORD_MASK(nbits) (~0UL >> (-(nbits) & (BITS_PER_LONG - 1)))
2371da177e4SLinus Torvalds 
2388b4daad5SRasmus Villemoes static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
2391da177e4SLinus Torvalds {
2408b4daad5SRasmus Villemoes 	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
2413e7e5baaSAlexander Lobakin 
2423e7e5baaSAlexander Lobakin 	if (small_const_nbits(nbits))
2433e7e5baaSAlexander Lobakin 		*dst = 0;
2443e7e5baaSAlexander Lobakin 	else
2451da177e4SLinus Torvalds 		memset(dst, 0, len);
2461da177e4SLinus Torvalds }
2471da177e4SLinus Torvalds 
2488b4daad5SRasmus Villemoes static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
2491da177e4SLinus Torvalds {
250334cfa48SAndy Shevchenko 	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
2513e7e5baaSAlexander Lobakin 
2523e7e5baaSAlexander Lobakin 	if (small_const_nbits(nbits))
2533e7e5baaSAlexander Lobakin 		*dst = ~0UL;
2543e7e5baaSAlexander Lobakin 	else
2551da177e4SLinus Torvalds 		memset(dst, 0xff, len);
2561da177e4SLinus Torvalds }
2571da177e4SLinus Torvalds 
2581da177e4SLinus Torvalds static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
2598b4daad5SRasmus Villemoes 			unsigned int nbits)
2601da177e4SLinus Torvalds {
2618b4daad5SRasmus Villemoes 	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
2623e7e5baaSAlexander Lobakin 
2633e7e5baaSAlexander Lobakin 	if (small_const_nbits(nbits))
2643e7e5baaSAlexander Lobakin 		*dst = *src;
2653e7e5baaSAlexander Lobakin 	else
2661da177e4SLinus Torvalds 		memcpy(dst, src, len);
2671da177e4SLinus Torvalds }
2681da177e4SLinus Torvalds 
269c724f193SYury Norov /*
270c724f193SYury Norov  * Copy bitmap and clear tail bits in last word.
271c724f193SYury Norov  */
272c724f193SYury Norov static inline void bitmap_copy_clear_tail(unsigned long *dst,
273c724f193SYury Norov 		const unsigned long *src, unsigned int nbits)
274c724f193SYury Norov {
275c724f193SYury Norov 	bitmap_copy(dst, src, nbits);
276c724f193SYury Norov 	if (nbits % BITS_PER_LONG)
277c724f193SYury Norov 		dst[nbits / BITS_PER_LONG] &= BITMAP_LAST_WORD_MASK(nbits);
278c724f193SYury Norov }
279c724f193SYury Norov 
280c724f193SYury Norov /*
281e041e0acSYury Norov  * On 32-bit systems bitmaps are represented as u32 arrays internally. On LE64
282e041e0acSYury Norov  * machines the order of hi and lo parts of numbers match the bitmap structure.
283e041e0acSYury Norov  * In both cases conversion is not needed when copying data from/to arrays of
284e041e0acSYury Norov  * u32. But in LE64 case, typecast in bitmap_copy_clear_tail() may lead
285e041e0acSYury Norov  * to out-of-bound access. To avoid that, both LE and BE variants of 64-bit
286e041e0acSYury Norov  * architectures are not using bitmap_copy_clear_tail().
287c724f193SYury Norov  */
288c724f193SYury Norov #if BITS_PER_LONG == 64
28998635b29SBartosz Golaszewski void bitmap_from_arr32(unsigned long *bitmap, const u32 *buf,
290c724f193SYury Norov 							unsigned int nbits);
29198635b29SBartosz Golaszewski void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap,
292c724f193SYury Norov 							unsigned int nbits);
293c724f193SYury Norov #else
294c724f193SYury Norov #define bitmap_from_arr32(bitmap, buf, nbits)			\
295c724f193SYury Norov 	bitmap_copy_clear_tail((unsigned long *) (bitmap),	\
296c724f193SYury Norov 			(const unsigned long *) (buf), (nbits))
297c724f193SYury Norov #define bitmap_to_arr32(buf, bitmap, nbits)			\
298c724f193SYury Norov 	bitmap_copy_clear_tail((unsigned long *) (buf),		\
299c724f193SYury Norov 			(const unsigned long *) (bitmap), (nbits))
300c724f193SYury Norov #endif
301c724f193SYury Norov 
3020a97953fSYury Norov /*
3030a97953fSYury Norov  * On 64-bit systems bitmaps are represented as u64 arrays internally. On LE32
3040a97953fSYury Norov  * machines the order of hi and lo parts of numbers match the bitmap structure.
3050a97953fSYury Norov  * In both cases conversion is not needed when copying data from/to arrays of
3060a97953fSYury Norov  * u64.
3070a97953fSYury Norov  */
3080a97953fSYury Norov #if (BITS_PER_LONG == 32) && defined(__BIG_ENDIAN)
3090a97953fSYury Norov void bitmap_from_arr64(unsigned long *bitmap, const u64 *buf, unsigned int nbits);
3100a97953fSYury Norov void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits);
3110a97953fSYury Norov #else
3120a97953fSYury Norov #define bitmap_from_arr64(bitmap, buf, nbits)			\
3130a97953fSYury Norov 	bitmap_copy_clear_tail((unsigned long *)(bitmap), (const unsigned long *)(buf), (nbits))
3140a97953fSYury Norov #define bitmap_to_arr64(buf, bitmap, nbits)			\
3150a97953fSYury Norov 	bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
3160a97953fSYury Norov #endif
3170a97953fSYury Norov 
318e2863a78SYury Norov static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,
3192f9305ebSRasmus Villemoes 			const unsigned long *src2, unsigned int nbits)
3201da177e4SLinus Torvalds {
3214b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
3227e5f97d1SRasmus Villemoes 		return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
323f4b0373bSLinus Torvalds 	return __bitmap_and(dst, src1, src2, nbits);
3241da177e4SLinus Torvalds }
3251da177e4SLinus Torvalds 
3261da177e4SLinus Torvalds static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
3272f9305ebSRasmus Villemoes 			const unsigned long *src2, unsigned int nbits)
3281da177e4SLinus Torvalds {
3294b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
3301da177e4SLinus Torvalds 		*dst = *src1 | *src2;
3311da177e4SLinus Torvalds 	else
3321da177e4SLinus Torvalds 		__bitmap_or(dst, src1, src2, nbits);
3331da177e4SLinus Torvalds }
3341da177e4SLinus Torvalds 
3351da177e4SLinus Torvalds static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
3362f9305ebSRasmus Villemoes 			const unsigned long *src2, unsigned int nbits)
3371da177e4SLinus Torvalds {
3384b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
3391da177e4SLinus Torvalds 		*dst = *src1 ^ *src2;
3401da177e4SLinus Torvalds 	else
3411da177e4SLinus Torvalds 		__bitmap_xor(dst, src1, src2, nbits);
3421da177e4SLinus Torvalds }
3431da177e4SLinus Torvalds 
344e2863a78SYury Norov static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
3452f9305ebSRasmus Villemoes 			const unsigned long *src2, unsigned int nbits)
3461da177e4SLinus Torvalds {
3474b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
34874e76531SRasmus Villemoes 		return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
349f4b0373bSLinus Torvalds 	return __bitmap_andnot(dst, src1, src2, nbits);
3501da177e4SLinus Torvalds }
3511da177e4SLinus Torvalds 
3521da177e4SLinus Torvalds static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
3533d6684f4SRasmus Villemoes 			unsigned int nbits)
3541da177e4SLinus Torvalds {
3554b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
35665b4ee62SRasmus Villemoes 		*dst = ~(*src);
3571da177e4SLinus Torvalds 	else
3581da177e4SLinus Torvalds 		__bitmap_complement(dst, src, nbits);
3591da177e4SLinus Torvalds }
3601da177e4SLinus Torvalds 
36121035965SOmar Sandoval #ifdef __LITTLE_ENDIAN
36221035965SOmar Sandoval #define BITMAP_MEM_ALIGNMENT 8
36321035965SOmar Sandoval #else
36421035965SOmar Sandoval #define BITMAP_MEM_ALIGNMENT (8 * sizeof(unsigned long))
36521035965SOmar Sandoval #endif
36621035965SOmar Sandoval #define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
36721035965SOmar Sandoval 
368005f1700SKees Cook static inline bool bitmap_equal(const unsigned long *src1,
3693d6684f4SRasmus Villemoes 				const unsigned long *src2, unsigned int nbits)
3701da177e4SLinus Torvalds {
3714b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
3721da177e4SLinus Torvalds 		return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
37321035965SOmar Sandoval 	if (__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
37421035965SOmar Sandoval 	    IS_ALIGNED(nbits, BITMAP_MEM_ALIGNMENT))
3757dd96816SMartin Schwidefsky 		return !memcmp(src1, src2, nbits / 8);
3761da177e4SLinus Torvalds 	return __bitmap_equal(src1, src2, nbits);
3771da177e4SLinus Torvalds }
3781da177e4SLinus Torvalds 
379b9fa6442SThomas Gleixner /**
3802a7e582fSRandy Dunlap  * bitmap_or_equal - Check whether the or of two bitmaps is equal to a third
381b9fa6442SThomas Gleixner  * @src1:	Pointer to bitmap 1
382b9fa6442SThomas Gleixner  * @src2:	Pointer to bitmap 2 will be or'ed with bitmap 1
383b9fa6442SThomas Gleixner  * @src3:	Pointer to bitmap 3. Compare to the result of *@src1 | *@src2
3842a7e582fSRandy Dunlap  * @nbits:	number of bits in each of these bitmaps
385b9fa6442SThomas Gleixner  *
386b9fa6442SThomas Gleixner  * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
387b9fa6442SThomas Gleixner  */
388b9fa6442SThomas Gleixner static inline bool bitmap_or_equal(const unsigned long *src1,
389b9fa6442SThomas Gleixner 				   const unsigned long *src2,
390b9fa6442SThomas Gleixner 				   const unsigned long *src3,
391b9fa6442SThomas Gleixner 				   unsigned int nbits)
392b9fa6442SThomas Gleixner {
393b9fa6442SThomas Gleixner 	if (!small_const_nbits(nbits))
394b9fa6442SThomas Gleixner 		return __bitmap_or_equal(src1, src2, src3, nbits);
395b9fa6442SThomas Gleixner 
396b9fa6442SThomas Gleixner 	return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
397b9fa6442SThomas Gleixner }
398b9fa6442SThomas Gleixner 
399005f1700SKees Cook static inline bool bitmap_intersects(const unsigned long *src1,
400005f1700SKees Cook 				     const unsigned long *src2,
401005f1700SKees Cook 				     unsigned int nbits)
4021da177e4SLinus Torvalds {
4034b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
4041da177e4SLinus Torvalds 		return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
4051da177e4SLinus Torvalds 	else
4061da177e4SLinus Torvalds 		return __bitmap_intersects(src1, src2, nbits);
4071da177e4SLinus Torvalds }
4081da177e4SLinus Torvalds 
409005f1700SKees Cook static inline bool bitmap_subset(const unsigned long *src1,
4105be20213SRasmus Villemoes 				 const unsigned long *src2, unsigned int nbits)
4111da177e4SLinus Torvalds {
4124b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
4131da177e4SLinus Torvalds 		return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
4141da177e4SLinus Torvalds 	else
4151da177e4SLinus Torvalds 		return __bitmap_subset(src1, src2, nbits);
4161da177e4SLinus Torvalds }
4171da177e4SLinus Torvalds 
4180bb86779SAndy Shevchenko static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
4191da177e4SLinus Torvalds {
4204b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
4211da177e4SLinus Torvalds 		return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
4222afe27c7SYury Norov 
4232afe27c7SYury Norov 	return find_first_bit(src, nbits) == nbits;
4241da177e4SLinus Torvalds }
4251da177e4SLinus Torvalds 
4260bb86779SAndy Shevchenko static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)
4271da177e4SLinus Torvalds {
4284b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
4291da177e4SLinus Torvalds 		return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
4302afe27c7SYury Norov 
4312afe27c7SYury Norov 	return find_first_zero_bit(src, nbits) == nbits;
4321da177e4SLinus Torvalds }
4331da177e4SLinus Torvalds 
434*4dea97f8SYury Norov static __always_inline
4354e23eeebSLinus Torvalds unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits)
4361da177e4SLinus Torvalds {
4374b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
43808cd3657SAndi Kleen 		return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));
4391da177e4SLinus Torvalds 	return __bitmap_weight(src, nbits);
4401da177e4SLinus Torvalds }
4411da177e4SLinus Torvalds 
442e5af323cSMatthew Wilcox static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
443e5af323cSMatthew Wilcox 		unsigned int nbits)
444e5af323cSMatthew Wilcox {
445e5af323cSMatthew Wilcox 	if (__builtin_constant_p(nbits) && nbits == 1)
446e5af323cSMatthew Wilcox 		__set_bit(start, map);
4473e7e5baaSAlexander Lobakin 	else if (small_const_nbits(start + nbits))
4483e7e5baaSAlexander Lobakin 		*map |= GENMASK(start + nbits - 1, start);
44921035965SOmar Sandoval 	else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&
45021035965SOmar Sandoval 		 IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&
45121035965SOmar Sandoval 		 __builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
45221035965SOmar Sandoval 		 IS_ALIGNED(nbits, BITMAP_MEM_ALIGNMENT))
4532a98dc02SMatthew Wilcox 		memset((char *)map + start / 8, 0xff, nbits / 8);
454e5af323cSMatthew Wilcox 	else
455e5af323cSMatthew Wilcox 		__bitmap_set(map, start, nbits);
456e5af323cSMatthew Wilcox }
457e5af323cSMatthew Wilcox 
458e5af323cSMatthew Wilcox static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
459e5af323cSMatthew Wilcox 		unsigned int nbits)
460e5af323cSMatthew Wilcox {
461e5af323cSMatthew Wilcox 	if (__builtin_constant_p(nbits) && nbits == 1)
462e5af323cSMatthew Wilcox 		__clear_bit(start, map);
4633e7e5baaSAlexander Lobakin 	else if (small_const_nbits(start + nbits))
4643e7e5baaSAlexander Lobakin 		*map &= ~GENMASK(start + nbits - 1, start);
46521035965SOmar Sandoval 	else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&
46621035965SOmar Sandoval 		 IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&
46721035965SOmar Sandoval 		 __builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
46821035965SOmar Sandoval 		 IS_ALIGNED(nbits, BITMAP_MEM_ALIGNMENT))
4692a98dc02SMatthew Wilcox 		memset((char *)map + start / 8, 0, nbits / 8);
470e5af323cSMatthew Wilcox 	else
471e5af323cSMatthew Wilcox 		__bitmap_clear(map, start, nbits);
472e5af323cSMatthew Wilcox }
473e5af323cSMatthew Wilcox 
4742fbad299SRasmus Villemoes static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
475d9873969SRasmus Villemoes 				unsigned int shift, unsigned int nbits)
4761da177e4SLinus Torvalds {
4774b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
4782fbad299SRasmus Villemoes 		*dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
4791da177e4SLinus Torvalds 	else
4802fbad299SRasmus Villemoes 		__bitmap_shift_right(dst, src, shift, nbits);
4811da177e4SLinus Torvalds }
4821da177e4SLinus Torvalds 
483dba94c25SRasmus Villemoes static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
484dba94c25SRasmus Villemoes 				unsigned int shift, unsigned int nbits)
4851da177e4SLinus Torvalds {
4864b0bc0bcSRusty Russell 	if (small_const_nbits(nbits))
487dba94c25SRasmus Villemoes 		*dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
4881da177e4SLinus Torvalds 	else
489dba94c25SRasmus Villemoes 		__bitmap_shift_left(dst, src, shift, nbits);
4901da177e4SLinus Torvalds }
4911da177e4SLinus Torvalds 
49230544ed5SAndy Shevchenko static inline void bitmap_replace(unsigned long *dst,
49330544ed5SAndy Shevchenko 				  const unsigned long *old,
49430544ed5SAndy Shevchenko 				  const unsigned long *new,
49530544ed5SAndy Shevchenko 				  const unsigned long *mask,
49630544ed5SAndy Shevchenko 				  unsigned int nbits)
49730544ed5SAndy Shevchenko {
49830544ed5SAndy Shevchenko 	if (small_const_nbits(nbits))
49930544ed5SAndy Shevchenko 		*dst = (*old & ~(*mask)) | (*new & *mask);
50030544ed5SAndy Shevchenko 	else
50130544ed5SAndy Shevchenko 		__bitmap_replace(dst, old, new, mask, nbits);
50230544ed5SAndy Shevchenko }
50330544ed5SAndy Shevchenko 
504e837dfdeSDennis Zhou static inline void bitmap_next_set_region(unsigned long *bitmap,
505e837dfdeSDennis Zhou 					  unsigned int *rs, unsigned int *re,
506e837dfdeSDennis Zhou 					  unsigned int end)
507e837dfdeSDennis Zhou {
508e837dfdeSDennis Zhou 	*rs = find_next_bit(bitmap, end, *rs);
509e837dfdeSDennis Zhou 	*re = find_next_zero_bit(bitmap, end, *rs + 1);
510e837dfdeSDennis Zhou }
511e837dfdeSDennis Zhou 
512404376afSRandy Dunlap /**
51360ef6900SYury Norov  * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap.
514404376afSRandy Dunlap  * @n: u64 value
51560ef6900SYury Norov  *
51660ef6900SYury Norov  * Linux bitmaps are internally arrays of unsigned longs, i.e. 32-bit
51760ef6900SYury Norov  * integers in 32-bit environment, and 64-bit integers in 64-bit one.
51860ef6900SYury Norov  *
51960ef6900SYury Norov  * There are four combinations of endianness and length of the word in linux
52060ef6900SYury Norov  * ABIs: LE64, BE64, LE32 and BE32.
52160ef6900SYury Norov  *
52260ef6900SYury Norov  * On 64-bit kernels 64-bit LE and BE numbers are naturally ordered in
52360ef6900SYury Norov  * bitmaps and therefore don't require any special handling.
52460ef6900SYury Norov  *
52560ef6900SYury Norov  * On 32-bit kernels 32-bit LE ABI orders lo word of 64-bit number in memory
52660ef6900SYury Norov  * prior to hi, and 32-bit BE orders hi word prior to lo. The bitmap on the
52760ef6900SYury Norov  * other hand is represented as an array of 32-bit words and the position of
52860ef6900SYury Norov  * bit N may therefore be calculated as: word #(N/32) and bit #(N%32) in that
52960ef6900SYury Norov  * word.  For example, bit #42 is located at 10th position of 2nd word.
53060ef6900SYury Norov  * It matches 32-bit LE ABI, and we can simply let the compiler store 64-bit
53160ef6900SYury Norov  * values in memory as it usually does. But for BE we need to swap hi and lo
53260ef6900SYury Norov  * words manually.
53360ef6900SYury Norov  *
53460ef6900SYury Norov  * With all that, the macro BITMAP_FROM_U64() does explicit reordering of hi and
53560ef6900SYury Norov  * lo parts of u64.  For LE32 it does nothing, and for BE environment it swaps
53660ef6900SYury Norov  * hi and lo words, as is expected by bitmap.
53760ef6900SYury Norov  */
53860ef6900SYury Norov #if __BITS_PER_LONG == 64
53960ef6900SYury Norov #define BITMAP_FROM_U64(n) (n)
54060ef6900SYury Norov #else
54160ef6900SYury Norov #define BITMAP_FROM_U64(n) ((unsigned long) ((u64)(n) & ULONG_MAX)), \
54260ef6900SYury Norov 				((unsigned long) ((u64)(n) >> 32))
54360ef6900SYury Norov #endif
54460ef6900SYury Norov 
545404376afSRandy Dunlap /**
54629dd3288SMadhavan Srinivasan  * bitmap_from_u64 - Check and swap words within u64.
54729dd3288SMadhavan Srinivasan  *  @mask: source bitmap
54829dd3288SMadhavan Srinivasan  *  @dst:  destination bitmap
54929dd3288SMadhavan Srinivasan  *
550404376afSRandy Dunlap  * In 32-bit Big Endian kernel, when using ``(u32 *)(&val)[*]``
55129dd3288SMadhavan Srinivasan  * to read u64 mask, we will get the wrong word.
552404376afSRandy Dunlap  * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
55329dd3288SMadhavan Srinivasan  * but we expect the lower 32-bits of u64.
55429dd3288SMadhavan Srinivasan  */
55529dd3288SMadhavan Srinivasan static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
55629dd3288SMadhavan Srinivasan {
5570a97953fSYury Norov 	bitmap_from_arr64(dst, &mask, 64);
55829dd3288SMadhavan Srinivasan }
55929dd3288SMadhavan Srinivasan 
560169c474fSWilliam Breathitt Gray /**
561169c474fSWilliam Breathitt Gray  * bitmap_get_value8 - get an 8-bit value within a memory region
562169c474fSWilliam Breathitt Gray  * @map: address to the bitmap memory region
563169c474fSWilliam Breathitt Gray  * @start: bit offset of the 8-bit value; must be a multiple of 8
564169c474fSWilliam Breathitt Gray  *
565169c474fSWilliam Breathitt Gray  * Returns the 8-bit value located at the @start bit offset within the @src
566169c474fSWilliam Breathitt Gray  * memory region.
567169c474fSWilliam Breathitt Gray  */
568169c474fSWilliam Breathitt Gray static inline unsigned long bitmap_get_value8(const unsigned long *map,
569169c474fSWilliam Breathitt Gray 					      unsigned long start)
570169c474fSWilliam Breathitt Gray {
571169c474fSWilliam Breathitt Gray 	const size_t index = BIT_WORD(start);
572169c474fSWilliam Breathitt Gray 	const unsigned long offset = start % BITS_PER_LONG;
573169c474fSWilliam Breathitt Gray 
574169c474fSWilliam Breathitt Gray 	return (map[index] >> offset) & 0xFF;
575169c474fSWilliam Breathitt Gray }
576169c474fSWilliam Breathitt Gray 
577169c474fSWilliam Breathitt Gray /**
578169c474fSWilliam Breathitt Gray  * bitmap_set_value8 - set an 8-bit value within a memory region
579169c474fSWilliam Breathitt Gray  * @map: address to the bitmap memory region
580169c474fSWilliam Breathitt Gray  * @value: the 8-bit value; values wider than 8 bits may clobber bitmap
581169c474fSWilliam Breathitt Gray  * @start: bit offset of the 8-bit value; must be a multiple of 8
582169c474fSWilliam Breathitt Gray  */
583169c474fSWilliam Breathitt Gray static inline void bitmap_set_value8(unsigned long *map, unsigned long value,
584169c474fSWilliam Breathitt Gray 				     unsigned long start)
585169c474fSWilliam Breathitt Gray {
586169c474fSWilliam Breathitt Gray 	const size_t index = BIT_WORD(start);
587169c474fSWilliam Breathitt Gray 	const unsigned long offset = start % BITS_PER_LONG;
588169c474fSWilliam Breathitt Gray 
589169c474fSWilliam Breathitt Gray 	map[index] &= ~(0xFFUL << offset);
590169c474fSWilliam Breathitt Gray 	map[index] |= value << offset;
591169c474fSWilliam Breathitt Gray }
592169c474fSWilliam Breathitt Gray 
5931da177e4SLinus Torvalds #endif /* __ASSEMBLY__ */
5941da177e4SLinus Torvalds 
5951da177e4SLinus Torvalds #endif /* __LINUX_BITMAP_H */
596