Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef __LINUX_CPUMASK_H
3 : #define __LINUX_CPUMASK_H
4 :
5 : /*
6 : * Cpumasks provide a bitmap suitable for representing the
7 : * set of CPU's in a system, one bit position per CPU number. In general,
8 : * only nr_cpu_ids (<= NR_CPUS) bits are valid.
9 : */
10 : #include <linux/kernel.h>
11 : #include <linux/threads.h>
12 : #include <linux/bitmap.h>
13 : #include <linux/atomic.h>
14 : #include <linux/bug.h>
15 : #include <linux/gfp_types.h>
16 : #include <linux/numa.h>
17 :
18 : /* Don't assign or return these: may not be this big! */
19 : typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
20 :
21 : /**
22 : * cpumask_bits - get the bits in a cpumask
23 : * @maskp: the struct cpumask *
24 : *
25 : * You should only assume nr_cpu_ids bits of this mask are valid. This is
26 : * a macro so it's const-correct.
27 : */
28 : #define cpumask_bits(maskp) ((maskp)->bits)
29 :
30 : /**
31 : * cpumask_pr_args - printf args to output a cpumask
32 : * @maskp: cpumask to be printed
33 : *
34 : * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
35 : */
36 : #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
37 :
38 : #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
39 : #define nr_cpu_ids ((unsigned int)NR_CPUS)
40 : #else
41 : extern unsigned int nr_cpu_ids;
42 : #endif
43 :
44 : static inline void set_nr_cpu_ids(unsigned int nr)
45 : {
46 : #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
47 : WARN_ON(nr != nr_cpu_ids);
48 : #else
49 : nr_cpu_ids = nr;
50 : #endif
51 : }
52 :
53 : /*
54 : * We have several different "preferred sizes" for the cpumask
55 : * operations, depending on operation.
56 : *
57 : * For example, the bitmap scanning and operating operations have
58 : * optimized routines that work for the single-word case, but only when
59 : * the size is constant. So if NR_CPUS fits in one single word, we are
60 : * better off using that small constant, in order to trigger the
61 : * optimized bit finding. That is 'small_cpumask_size'.
62 : *
63 : * The clearing and copying operations will similarly perform better
64 : * with a constant size, but we limit that size arbitrarily to four
65 : * words. We call this 'large_cpumask_size'.
66 : *
67 : * Finally, some operations just want the exact limit, either because
68 : * they set bits or just don't have any faster fixed-sized versions. We
69 : * call this just 'nr_cpumask_bits'.
70 : *
71 : * Note that these optional constants are always guaranteed to be at
72 : * least as big as 'nr_cpu_ids' itself is, and all our cpumask
73 : * allocations are at least that size (see cpumask_size()). The
74 : * optimization comes from being able to potentially use a compile-time
75 : * constant instead of a run-time generated exact number of CPUs.
76 : */
77 : #if NR_CPUS <= BITS_PER_LONG
78 : #define small_cpumask_bits ((unsigned int)NR_CPUS)
79 : #define large_cpumask_bits ((unsigned int)NR_CPUS)
80 : #elif NR_CPUS <= 4*BITS_PER_LONG
81 : #define small_cpumask_bits nr_cpu_ids
82 : #define large_cpumask_bits ((unsigned int)NR_CPUS)
83 : #else
84 : #define small_cpumask_bits nr_cpu_ids
85 : #define large_cpumask_bits nr_cpu_ids
86 : #endif
87 : #define nr_cpumask_bits nr_cpu_ids
88 :
89 : /*
90 : * The following particular system cpumasks and operations manage
91 : * possible, present, active and online cpus.
92 : *
93 : * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
94 : * cpu_present_mask - has bit 'cpu' set iff cpu is populated
95 : * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
96 : * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
97 : *
98 : * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
99 : *
100 : * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
101 : * that it is possible might ever be plugged in at anytime during the
102 : * life of that system boot. The cpu_present_mask is dynamic(*),
103 : * representing which CPUs are currently plugged in. And
104 : * cpu_online_mask is the dynamic subset of cpu_present_mask,
105 : * indicating those CPUs available for scheduling.
106 : *
107 : * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
108 : * depending on what ACPI reports as currently plugged in, otherwise
109 : * cpu_present_mask is just a copy of cpu_possible_mask.
110 : *
111 : * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
112 : * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
113 : *
114 : * Subtleties:
115 : * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
116 : * assumption that their single CPU is online. The UP
117 : * cpu_{online,possible,present}_masks are placebos. Changing them
118 : * will have no useful affect on the following num_*_cpus()
119 : * and cpu_*() macros in the UP case. This ugliness is a UP
120 : * optimization - don't waste any instructions or memory references
121 : * asking if you're online or how many CPUs there are if there is
122 : * only one CPU.
123 : */
124 :
125 : extern struct cpumask __cpu_possible_mask;
126 : extern struct cpumask __cpu_online_mask;
127 : extern struct cpumask __cpu_present_mask;
128 : extern struct cpumask __cpu_active_mask;
129 : extern struct cpumask __cpu_dying_mask;
130 : #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
131 : #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
132 : #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
133 : #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
134 : #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
135 :
136 : extern atomic_t __num_online_cpus;
137 :
138 : extern cpumask_t cpus_booted_once_mask;
139 :
140 : static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
141 : {
142 : #ifdef CONFIG_DEBUG_PER_CPU_MAPS
143 : WARN_ON_ONCE(cpu >= bits);
144 : #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
145 : }
146 :
147 : /* verify cpu argument to cpumask_* operators */
148 : static __always_inline unsigned int cpumask_check(unsigned int cpu)
149 : {
150 : cpu_max_bits_warn(cpu, small_cpumask_bits);
151 : return cpu;
152 : }
153 :
154 : /**
155 : * cpumask_first - get the first cpu in a cpumask
156 : * @srcp: the cpumask pointer
157 : *
158 : * Returns >= nr_cpu_ids if no cpus set.
159 : */
160 : static inline unsigned int cpumask_first(const struct cpumask *srcp)
161 : {
162 : return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
163 : }
164 :
165 : /**
166 : * cpumask_first_zero - get the first unset cpu in a cpumask
167 : * @srcp: the cpumask pointer
168 : *
169 : * Returns >= nr_cpu_ids if all cpus are set.
170 : */
171 : static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
172 : {
173 : return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
174 : }
175 :
176 : /**
177 : * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
178 : * @src1p: the first input
179 : * @src2p: the second input
180 : *
181 : * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
182 : */
183 : static inline
184 : unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
185 : {
186 : return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
187 : }
188 :
189 : /**
190 : * cpumask_last - get the last CPU in a cpumask
191 : * @srcp: - the cpumask pointer
192 : *
193 : * Returns >= nr_cpumask_bits if no CPUs set.
194 : */
195 : static inline unsigned int cpumask_last(const struct cpumask *srcp)
196 : {
197 : return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
198 : }
199 :
200 : /**
201 : * cpumask_next - get the next cpu in a cpumask
202 : * @n: the cpu prior to the place to search (ie. return will be > @n)
203 : * @srcp: the cpumask pointer
204 : *
205 : * Returns >= nr_cpu_ids if no further cpus set.
206 : */
207 : static inline
208 : unsigned int cpumask_next(int n, const struct cpumask *srcp)
209 : {
210 : /* -1 is a legal arg here. */
211 8 : if (n != -1)
212 : cpumask_check(n);
213 8 : return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);
214 : }
215 :
216 : /**
217 : * cpumask_next_zero - get the next unset cpu in a cpumask
218 : * @n: the cpu prior to the place to search (ie. return will be > @n)
219 : * @srcp: the cpumask pointer
220 : *
221 : * Returns >= nr_cpu_ids if no further cpus unset.
222 : */
223 : static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
224 : {
225 : /* -1 is a legal arg here. */
226 : if (n != -1)
227 : cpumask_check(n);
228 : return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);
229 : }
230 :
231 : #if NR_CPUS == 1
232 : /* Uniprocessor: there is only one valid CPU */
233 : static inline unsigned int cpumask_local_spread(unsigned int i, int node)
234 : {
235 : return 0;
236 : }
237 :
238 : static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
239 : const struct cpumask *src2p)
240 : {
241 : return cpumask_first_and(src1p, src2p);
242 : }
243 :
244 : static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
245 : {
246 : return cpumask_first(srcp);
247 : }
248 : #else
249 : unsigned int cpumask_local_spread(unsigned int i, int node);
250 : unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
251 : const struct cpumask *src2p);
252 : unsigned int cpumask_any_distribute(const struct cpumask *srcp);
253 : #endif /* NR_CPUS */
254 :
255 : /**
256 : * cpumask_next_and - get the next cpu in *src1p & *src2p
257 : * @n: the cpu prior to the place to search (ie. return will be > @n)
258 : * @src1p: the first cpumask pointer
259 : * @src2p: the second cpumask pointer
260 : *
261 : * Returns >= nr_cpu_ids if no further cpus set in both.
262 : */
263 : static inline
264 : unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
265 : const struct cpumask *src2p)
266 : {
267 : /* -1 is a legal arg here. */
268 : if (n != -1)
269 : cpumask_check(n);
270 : return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
271 : small_cpumask_bits, n + 1);
272 : }
273 :
274 : /**
275 : * for_each_cpu - iterate over every cpu in a mask
276 : * @cpu: the (optionally unsigned) integer iterator
277 : * @mask: the cpumask pointer
278 : *
279 : * After the loop, cpu is >= nr_cpu_ids.
280 : */
281 : #define for_each_cpu(cpu, mask) \
282 : for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
283 :
284 : #if NR_CPUS == 1
285 : static inline
286 : unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
287 : {
288 : cpumask_check(start);
289 : if (n != -1)
290 : cpumask_check(n);
291 :
292 : /*
293 : * Return the first available CPU when wrapping, or when starting before cpu0,
294 : * since there is only one valid option.
295 : */
296 : if (wrap && n >= 0)
297 : return nr_cpumask_bits;
298 :
299 : return cpumask_first(mask);
300 : }
301 : #else
302 : unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
303 : #endif
304 :
305 : /**
306 : * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
307 : * @cpu: the (optionally unsigned) integer iterator
308 : * @mask: the cpumask pointer
309 : * @start: the start location
310 : *
311 : * The implementation does not assume any bit in @mask is set (including @start).
312 : *
313 : * After the loop, cpu is >= nr_cpu_ids.
314 : */
315 : #define for_each_cpu_wrap(cpu, mask, start) \
316 : for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
317 :
318 : /**
319 : * for_each_cpu_and - iterate over every cpu in both masks
320 : * @cpu: the (optionally unsigned) integer iterator
321 : * @mask1: the first cpumask pointer
322 : * @mask2: the second cpumask pointer
323 : *
324 : * This saves a temporary CPU mask in many places. It is equivalent to:
325 : * struct cpumask tmp;
326 : * cpumask_and(&tmp, &mask1, &mask2);
327 : * for_each_cpu(cpu, &tmp)
328 : * ...
329 : *
330 : * After the loop, cpu is >= nr_cpu_ids.
331 : */
332 : #define for_each_cpu_and(cpu, mask1, mask2) \
333 : for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
334 :
335 : /**
336 : * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
337 : * those present in another.
338 : * @cpu: the (optionally unsigned) integer iterator
339 : * @mask1: the first cpumask pointer
340 : * @mask2: the second cpumask pointer
341 : *
342 : * This saves a temporary CPU mask in many places. It is equivalent to:
343 : * struct cpumask tmp;
344 : * cpumask_andnot(&tmp, &mask1, &mask2);
345 : * for_each_cpu(cpu, &tmp)
346 : * ...
347 : *
348 : * After the loop, cpu is >= nr_cpu_ids.
349 : */
350 : #define for_each_cpu_andnot(cpu, mask1, mask2) \
351 : for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
352 :
353 : /**
354 : * for_each_cpu_or - iterate over every cpu present in either mask
355 : * @cpu: the (optionally unsigned) integer iterator
356 : * @mask1: the first cpumask pointer
357 : * @mask2: the second cpumask pointer
358 : *
359 : * This saves a temporary CPU mask in many places. It is equivalent to:
360 : * struct cpumask tmp;
361 : * cpumask_or(&tmp, &mask1, &mask2);
362 : * for_each_cpu(cpu, &tmp)
363 : * ...
364 : *
365 : * After the loop, cpu is >= nr_cpu_ids.
366 : */
367 : #define for_each_cpu_or(cpu, mask1, mask2) \
368 : for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
369 :
370 : /**
371 : * cpumask_any_but - return a "random" in a cpumask, but not this one.
372 : * @mask: the cpumask to search
373 : * @cpu: the cpu to ignore.
374 : *
375 : * Often used to find any cpu but smp_processor_id() in a mask.
376 : * Returns >= nr_cpu_ids if no cpus set.
377 : */
378 : static inline
379 : unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
380 : {
381 : unsigned int i;
382 :
383 : cpumask_check(cpu);
384 : for_each_cpu(i, mask)
385 : if (i != cpu)
386 : break;
387 : return i;
388 : }
389 :
390 : /**
391 : * cpumask_nth - get the first cpu in a cpumask
392 : * @srcp: the cpumask pointer
393 : * @cpu: the N'th cpu to find, starting from 0
394 : *
395 : * Returns >= nr_cpu_ids if such cpu doesn't exist.
396 : */
397 : static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
398 : {
399 : return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
400 : }
401 :
402 : /**
403 : * cpumask_nth_and - get the first cpu in 2 cpumasks
404 : * @srcp1: the cpumask pointer
405 : * @srcp2: the cpumask pointer
406 : * @cpu: the N'th cpu to find, starting from 0
407 : *
408 : * Returns >= nr_cpu_ids if such cpu doesn't exist.
409 : */
410 : static inline
411 : unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
412 : const struct cpumask *srcp2)
413 : {
414 : return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
415 : small_cpumask_bits, cpumask_check(cpu));
416 : }
417 :
418 : /**
419 : * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd.
420 : * @srcp1: the cpumask pointer
421 : * @srcp2: the cpumask pointer
422 : * @cpu: the N'th cpu to find, starting from 0
423 : *
424 : * Returns >= nr_cpu_ids if such cpu doesn't exist.
425 : */
426 : static inline
427 : unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
428 : const struct cpumask *srcp2)
429 : {
430 : return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
431 : small_cpumask_bits, cpumask_check(cpu));
432 : }
433 :
434 : /**
435 : * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
436 : * @srcp1: the cpumask pointer
437 : * @srcp2: the cpumask pointer
438 : * @srcp3: the cpumask pointer
439 : * @cpu: the N'th cpu to find, starting from 0
440 : *
441 : * Returns >= nr_cpu_ids if such cpu doesn't exist.
442 : */
443 : static __always_inline
444 : unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,
445 : const struct cpumask *srcp2,
446 : const struct cpumask *srcp3)
447 : {
448 : return find_nth_and_andnot_bit(cpumask_bits(srcp1),
449 : cpumask_bits(srcp2),
450 : cpumask_bits(srcp3),
451 : small_cpumask_bits, cpumask_check(cpu));
452 : }
453 :
454 : #define CPU_BITS_NONE \
455 : { \
456 : [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
457 : }
458 :
459 : #define CPU_BITS_CPU0 \
460 : { \
461 : [0] = 1UL \
462 : }
463 :
464 : /**
465 : * cpumask_set_cpu - set a cpu in a cpumask
466 : * @cpu: cpu number (< nr_cpu_ids)
467 : * @dstp: the cpumask pointer
468 : */
469 : static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
470 : {
471 : set_bit(cpumask_check(cpu), cpumask_bits(dstp));
472 : }
473 :
474 : static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
475 : {
476 : __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
477 : }
478 :
479 :
480 : /**
481 : * cpumask_clear_cpu - clear a cpu in a cpumask
482 : * @cpu: cpu number (< nr_cpu_ids)
483 : * @dstp: the cpumask pointer
484 : */
485 : static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
486 : {
487 : clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
488 : }
489 :
490 : static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
491 : {
492 : __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
493 : }
494 :
495 : /**
496 : * cpumask_test_cpu - test for a cpu in a cpumask
497 : * @cpu: cpu number (< nr_cpu_ids)
498 : * @cpumask: the cpumask pointer
499 : *
500 : * Returns true if @cpu is set in @cpumask, else returns false
501 : */
502 : static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
503 : {
504 4 : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
505 : }
506 :
507 : /**
508 : * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
509 : * @cpu: cpu number (< nr_cpu_ids)
510 : * @cpumask: the cpumask pointer
511 : *
512 : * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
513 : *
514 : * test_and_set_bit wrapper for cpumasks.
515 : */
516 : static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
517 : {
518 : return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
519 : }
520 :
521 : /**
522 : * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
523 : * @cpu: cpu number (< nr_cpu_ids)
524 : * @cpumask: the cpumask pointer
525 : *
526 : * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
527 : *
528 : * test_and_clear_bit wrapper for cpumasks.
529 : */
530 : static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
531 : {
532 : return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
533 : }
534 :
535 : /**
536 : * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
537 : * @dstp: the cpumask pointer
538 : */
539 : static inline void cpumask_setall(struct cpumask *dstp)
540 : {
541 : if (small_const_nbits(small_cpumask_bits)) {
542 : cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
543 : return;
544 : }
545 : bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
546 : }
547 :
548 : /**
549 : * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
550 : * @dstp: the cpumask pointer
551 : */
552 : static inline void cpumask_clear(struct cpumask *dstp)
553 : {
554 : bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
555 : }
556 :
557 : /**
558 : * cpumask_and - *dstp = *src1p & *src2p
559 : * @dstp: the cpumask result
560 : * @src1p: the first input
561 : * @src2p: the second input
562 : *
563 : * If *@dstp is empty, returns false, else returns true
564 : */
565 : static inline bool cpumask_and(struct cpumask *dstp,
566 : const struct cpumask *src1p,
567 : const struct cpumask *src2p)
568 : {
569 : return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
570 : cpumask_bits(src2p), small_cpumask_bits);
571 : }
572 :
573 : /**
574 : * cpumask_or - *dstp = *src1p | *src2p
575 : * @dstp: the cpumask result
576 : * @src1p: the first input
577 : * @src2p: the second input
578 : */
579 : static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
580 : const struct cpumask *src2p)
581 : {
582 : bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
583 : cpumask_bits(src2p), small_cpumask_bits);
584 : }
585 :
586 : /**
587 : * cpumask_xor - *dstp = *src1p ^ *src2p
588 : * @dstp: the cpumask result
589 : * @src1p: the first input
590 : * @src2p: the second input
591 : */
592 : static inline void cpumask_xor(struct cpumask *dstp,
593 : const struct cpumask *src1p,
594 : const struct cpumask *src2p)
595 : {
596 : bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
597 : cpumask_bits(src2p), small_cpumask_bits);
598 : }
599 :
600 : /**
601 : * cpumask_andnot - *dstp = *src1p & ~*src2p
602 : * @dstp: the cpumask result
603 : * @src1p: the first input
604 : * @src2p: the second input
605 : *
606 : * If *@dstp is empty, returns false, else returns true
607 : */
608 : static inline bool cpumask_andnot(struct cpumask *dstp,
609 : const struct cpumask *src1p,
610 : const struct cpumask *src2p)
611 : {
612 : return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
613 : cpumask_bits(src2p), small_cpumask_bits);
614 : }
615 :
616 : /**
617 : * cpumask_equal - *src1p == *src2p
618 : * @src1p: the first input
619 : * @src2p: the second input
620 : */
621 : static inline bool cpumask_equal(const struct cpumask *src1p,
622 : const struct cpumask *src2p)
623 : {
624 : return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
625 : small_cpumask_bits);
626 : }
627 :
628 : /**
629 : * cpumask_or_equal - *src1p | *src2p == *src3p
630 : * @src1p: the first input
631 : * @src2p: the second input
632 : * @src3p: the third input
633 : */
634 : static inline bool cpumask_or_equal(const struct cpumask *src1p,
635 : const struct cpumask *src2p,
636 : const struct cpumask *src3p)
637 : {
638 : return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
639 : cpumask_bits(src3p), small_cpumask_bits);
640 : }
641 :
642 : /**
643 : * cpumask_intersects - (*src1p & *src2p) != 0
644 : * @src1p: the first input
645 : * @src2p: the second input
646 : */
647 : static inline bool cpumask_intersects(const struct cpumask *src1p,
648 : const struct cpumask *src2p)
649 : {
650 : return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
651 : small_cpumask_bits);
652 : }
653 :
654 : /**
655 : * cpumask_subset - (*src1p & ~*src2p) == 0
656 : * @src1p: the first input
657 : * @src2p: the second input
658 : *
659 : * Returns true if *@src1p is a subset of *@src2p, else returns false
660 : */
661 : static inline bool cpumask_subset(const struct cpumask *src1p,
662 : const struct cpumask *src2p)
663 : {
664 : return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
665 : small_cpumask_bits);
666 : }
667 :
668 : /**
669 : * cpumask_empty - *srcp == 0
670 : * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
671 : */
672 : static inline bool cpumask_empty(const struct cpumask *srcp)
673 : {
674 : return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
675 : }
676 :
677 : /**
678 : * cpumask_full - *srcp == 0xFFFFFFFF...
679 : * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
680 : */
681 : static inline bool cpumask_full(const struct cpumask *srcp)
682 : {
683 : return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
684 : }
685 :
686 : /**
687 : * cpumask_weight - Count of bits in *srcp
688 : * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
689 : */
690 958233 : static inline unsigned int cpumask_weight(const struct cpumask *srcp)
691 : {
692 958233 : return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
693 : }
694 :
695 : /**
696 : * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
697 : * @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
698 : * @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
699 : */
700 : static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
701 : const struct cpumask *srcp2)
702 : {
703 : return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
704 : }
705 :
706 : /**
707 : * cpumask_shift_right - *dstp = *srcp >> n
708 : * @dstp: the cpumask result
709 : * @srcp: the input to shift
710 : * @n: the number of bits to shift by
711 : */
712 : static inline void cpumask_shift_right(struct cpumask *dstp,
713 : const struct cpumask *srcp, int n)
714 : {
715 : bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
716 : small_cpumask_bits);
717 : }
718 :
719 : /**
720 : * cpumask_shift_left - *dstp = *srcp << n
721 : * @dstp: the cpumask result
722 : * @srcp: the input to shift
723 : * @n: the number of bits to shift by
724 : */
725 : static inline void cpumask_shift_left(struct cpumask *dstp,
726 : const struct cpumask *srcp, int n)
727 : {
728 : bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
729 : nr_cpumask_bits);
730 : }
731 :
732 : /**
733 : * cpumask_copy - *dstp = *srcp
734 : * @dstp: the result
735 : * @srcp: the input cpumask
736 : */
737 : static inline void cpumask_copy(struct cpumask *dstp,
738 : const struct cpumask *srcp)
739 : {
740 : bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
741 : }
742 :
743 : /**
744 : * cpumask_any - pick a "random" cpu from *srcp
745 : * @srcp: the input cpumask
746 : *
747 : * Returns >= nr_cpu_ids if no cpus set.
748 : */
749 : #define cpumask_any(srcp) cpumask_first(srcp)
750 :
751 : /**
752 : * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
753 : * @mask1: the first input cpumask
754 : * @mask2: the second input cpumask
755 : *
756 : * Returns >= nr_cpu_ids if no cpus set.
757 : */
758 : #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
759 :
760 : /**
761 : * cpumask_of - the cpumask containing just a given cpu
762 : * @cpu: the cpu (<= nr_cpu_ids)
763 : */
764 : #define cpumask_of(cpu) (get_cpu_mask(cpu))
765 :
766 : /**
767 : * cpumask_parse_user - extract a cpumask from a user string
768 : * @buf: the buffer to extract from
769 : * @len: the length of the buffer
770 : * @dstp: the cpumask to set.
771 : *
772 : * Returns -errno, or 0 for success.
773 : */
774 : static inline int cpumask_parse_user(const char __user *buf, int len,
775 : struct cpumask *dstp)
776 : {
777 : return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
778 : }
779 :
780 : /**
781 : * cpumask_parselist_user - extract a cpumask from a user string
782 : * @buf: the buffer to extract from
783 : * @len: the length of the buffer
784 : * @dstp: the cpumask to set.
785 : *
786 : * Returns -errno, or 0 for success.
787 : */
788 : static inline int cpumask_parselist_user(const char __user *buf, int len,
789 : struct cpumask *dstp)
790 : {
791 : return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
792 : nr_cpumask_bits);
793 : }
794 :
795 : /**
796 : * cpumask_parse - extract a cpumask from a string
797 : * @buf: the buffer to extract from
798 : * @dstp: the cpumask to set.
799 : *
800 : * Returns -errno, or 0 for success.
801 : */
802 : static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
803 : {
804 : return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
805 : }
806 :
807 : /**
808 : * cpulist_parse - extract a cpumask from a user string of ranges
809 : * @buf: the buffer to extract from
810 : * @dstp: the cpumask to set.
811 : *
812 : * Returns -errno, or 0 for success.
813 : */
814 : static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
815 : {
816 : return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
817 : }
818 :
819 : /**
820 : * cpumask_size - size to allocate for a 'struct cpumask' in bytes
821 : */
822 : static inline unsigned int cpumask_size(void)
823 : {
824 : return BITS_TO_LONGS(large_cpumask_bits) * sizeof(long);
825 : }
826 :
827 : /*
828 : * cpumask_var_t: struct cpumask for stack usage.
829 : *
830 : * Oh, the wicked games we play! In order to make kernel coding a
831 : * little more difficult, we typedef cpumask_var_t to an array or a
832 : * pointer: doing &mask on an array is a noop, so it still works.
833 : *
834 : * ie.
835 : * cpumask_var_t tmpmask;
836 : * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
837 : * return -ENOMEM;
838 : *
839 : * ... use 'tmpmask' like a normal struct cpumask * ...
840 : *
841 : * free_cpumask_var(tmpmask);
842 : *
843 : *
844 : * However, one notable exception is there. alloc_cpumask_var() allocates
845 : * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
846 : * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
847 : *
848 : * cpumask_var_t tmpmask;
849 : * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
850 : * return -ENOMEM;
851 : *
852 : * var = *tmpmask;
853 : *
854 : * This code makes NR_CPUS length memcopy and brings to a memory corruption.
855 : * cpumask_copy() provide safe copy functionality.
856 : *
857 : * Note that there is another evil here: If you define a cpumask_var_t
858 : * as a percpu variable then the way to obtain the address of the cpumask
859 : * structure differently influences what this_cpu_* operation needs to be
860 : * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
861 : * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
862 : * other type of cpumask_var_t implementation is configured.
863 : *
864 : * Please also note that __cpumask_var_read_mostly can be used to declare
865 : * a cpumask_var_t variable itself (not its content) as read mostly.
866 : */
867 : #ifdef CONFIG_CPUMASK_OFFSTACK
868 : typedef struct cpumask *cpumask_var_t;
869 :
870 : #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
871 : #define __cpumask_var_read_mostly __read_mostly
872 :
873 : bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
874 :
875 : static inline
876 : bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
877 : {
878 : return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
879 : }
880 :
881 : /**
882 : * alloc_cpumask_var - allocate a struct cpumask
883 : * @mask: pointer to cpumask_var_t where the cpumask is returned
884 : * @flags: GFP_ flags
885 : *
886 : * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
887 : * a nop returning a constant 1 (in <linux/cpumask.h>).
888 : *
889 : * See alloc_cpumask_var_node.
890 : */
891 : static inline
892 : bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
893 : {
894 : return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
895 : }
896 :
897 : static inline
898 : bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
899 : {
900 : return alloc_cpumask_var(mask, flags | __GFP_ZERO);
901 : }
902 :
903 : void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
904 : void free_cpumask_var(cpumask_var_t mask);
905 : void free_bootmem_cpumask_var(cpumask_var_t mask);
906 :
907 : static inline bool cpumask_available(cpumask_var_t mask)
908 : {
909 : return mask != NULL;
910 : }
911 :
912 : #else
913 : typedef struct cpumask cpumask_var_t[1];
914 :
915 : #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
916 : #define __cpumask_var_read_mostly
917 :
918 : static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
919 : {
920 : return true;
921 : }
922 :
923 : static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
924 : int node)
925 : {
926 : return true;
927 : }
928 :
929 : static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
930 : {
931 : cpumask_clear(*mask);
932 : return true;
933 : }
934 :
935 : static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
936 : int node)
937 : {
938 : cpumask_clear(*mask);
939 : return true;
940 : }
941 :
942 : static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
943 : {
944 : }
945 :
946 : static inline void free_cpumask_var(cpumask_var_t mask)
947 : {
948 : }
949 :
950 : static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
951 : {
952 : }
953 :
954 : static inline bool cpumask_available(cpumask_var_t mask)
955 : {
956 : return true;
957 : }
958 : #endif /* CONFIG_CPUMASK_OFFSTACK */
959 :
960 : /* It's common to want to use cpu_all_mask in struct member initializers,
961 : * so it has to refer to an address rather than a pointer. */
962 : extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
963 : #define cpu_all_mask to_cpumask(cpu_all_bits)
964 :
965 : /* First bits of cpu_bit_bitmap are in fact unset. */
966 : #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
967 :
968 : #if NR_CPUS == 1
969 : /* Uniprocessor: the possible/online/present masks are always "1" */
970 : #define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
971 : #define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
972 : #define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
973 : #else
974 : #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
975 : #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
976 : #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
977 : #endif
978 :
979 : /* Wrappers for arch boot code to manipulate normally-constant masks */
980 : void init_cpu_present(const struct cpumask *src);
981 : void init_cpu_possible(const struct cpumask *src);
982 : void init_cpu_online(const struct cpumask *src);
983 :
984 : static inline void reset_cpu_possible_mask(void)
985 : {
986 : bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
987 : }
988 :
989 : static inline void
990 : set_cpu_possible(unsigned int cpu, bool possible)
991 : {
992 : if (possible)
993 : cpumask_set_cpu(cpu, &__cpu_possible_mask);
994 : else
995 : cpumask_clear_cpu(cpu, &__cpu_possible_mask);
996 : }
997 :
998 : static inline void
999 : set_cpu_present(unsigned int cpu, bool present)
1000 : {
1001 : if (present)
1002 : cpumask_set_cpu(cpu, &__cpu_present_mask);
1003 : else
1004 : cpumask_clear_cpu(cpu, &__cpu_present_mask);
1005 : }
1006 :
1007 : void set_cpu_online(unsigned int cpu, bool online);
1008 :
1009 : static inline void
1010 : set_cpu_active(unsigned int cpu, bool active)
1011 : {
1012 : if (active)
1013 : cpumask_set_cpu(cpu, &__cpu_active_mask);
1014 : else
1015 : cpumask_clear_cpu(cpu, &__cpu_active_mask);
1016 : }
1017 :
1018 : static inline void
1019 : set_cpu_dying(unsigned int cpu, bool dying)
1020 : {
1021 : if (dying)
1022 : cpumask_set_cpu(cpu, &__cpu_dying_mask);
1023 : else
1024 : cpumask_clear_cpu(cpu, &__cpu_dying_mask);
1025 : }
1026 :
1027 : /**
1028 : * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
1029 : * @bitmap: the bitmap
1030 : *
1031 : * There are a few places where cpumask_var_t isn't appropriate and
1032 : * static cpumasks must be used (eg. very early boot), yet we don't
1033 : * expose the definition of 'struct cpumask'.
1034 : *
1035 : * This does the conversion, and can be used as a constant initializer.
1036 : */
1037 : #define to_cpumask(bitmap) \
1038 : ((struct cpumask *)(1 ? (bitmap) \
1039 : : (void *)sizeof(__check_is_bitmap(bitmap))))
1040 :
1041 : static inline int __check_is_bitmap(const unsigned long *bitmap)
1042 : {
1043 : return 1;
1044 : }
1045 :
1046 : /*
1047 : * Special-case data structure for "single bit set only" constant CPU masks.
1048 : *
1049 : * We pre-generate all the 64 (or 32) possible bit positions, with enough
1050 : * padding to the left and the right, and return the constant pointer
1051 : * appropriately offset.
1052 : */
1053 : extern const unsigned long
1054 : cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1055 :
1056 : static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
1057 : {
1058 : const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
1059 : p -= cpu / BITS_PER_LONG;
1060 : return to_cpumask(p);
1061 : }
1062 :
1063 : #if NR_CPUS > 1
1064 : /**
1065 : * num_online_cpus() - Read the number of online CPUs
1066 : *
1067 : * Despite the fact that __num_online_cpus is of type atomic_t, this
1068 : * interface gives only a momentary snapshot and is not protected against
1069 : * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1070 : * region.
1071 : */
1072 : static __always_inline unsigned int num_online_cpus(void)
1073 : {
1074 1774933042 : return raw_atomic_read(&__num_online_cpus);
1075 : }
1076 : #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
1077 : #define num_present_cpus() cpumask_weight(cpu_present_mask)
1078 : #define num_active_cpus() cpumask_weight(cpu_active_mask)
1079 :
1080 : static inline bool cpu_online(unsigned int cpu)
1081 : {
1082 4 : return cpumask_test_cpu(cpu, cpu_online_mask);
1083 : }
1084 :
1085 : static inline bool cpu_possible(unsigned int cpu)
1086 : {
1087 : return cpumask_test_cpu(cpu, cpu_possible_mask);
1088 : }
1089 :
1090 : static inline bool cpu_present(unsigned int cpu)
1091 : {
1092 : return cpumask_test_cpu(cpu, cpu_present_mask);
1093 : }
1094 :
1095 : static inline bool cpu_active(unsigned int cpu)
1096 : {
1097 : return cpumask_test_cpu(cpu, cpu_active_mask);
1098 : }
1099 :
1100 : static inline bool cpu_dying(unsigned int cpu)
1101 : {
1102 : return cpumask_test_cpu(cpu, cpu_dying_mask);
1103 : }
1104 :
1105 : #else
1106 :
1107 : #define num_online_cpus() 1U
1108 : #define num_possible_cpus() 1U
1109 : #define num_present_cpus() 1U
1110 : #define num_active_cpus() 1U
1111 :
1112 : static inline bool cpu_online(unsigned int cpu)
1113 : {
1114 : return cpu == 0;
1115 : }
1116 :
1117 : static inline bool cpu_possible(unsigned int cpu)
1118 : {
1119 : return cpu == 0;
1120 : }
1121 :
1122 : static inline bool cpu_present(unsigned int cpu)
1123 : {
1124 : return cpu == 0;
1125 : }
1126 :
1127 : static inline bool cpu_active(unsigned int cpu)
1128 : {
1129 : return cpu == 0;
1130 : }
1131 :
1132 : static inline bool cpu_dying(unsigned int cpu)
1133 : {
1134 : return false;
1135 : }
1136 :
1137 : #endif /* NR_CPUS > 1 */
1138 :
1139 : #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
1140 :
1141 : #if NR_CPUS <= BITS_PER_LONG
1142 : #define CPU_BITS_ALL \
1143 : { \
1144 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1145 : }
1146 :
1147 : #else /* NR_CPUS > BITS_PER_LONG */
1148 :
1149 : #define CPU_BITS_ALL \
1150 : { \
1151 : [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1152 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1153 : }
1154 : #endif /* NR_CPUS > BITS_PER_LONG */
1155 :
1156 : /**
1157 : * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
1158 : * as comma-separated list of cpus or hex values of cpumask
1159 : * @list: indicates whether the cpumap must be list
1160 : * @mask: the cpumask to copy
1161 : * @buf: the buffer to copy into
1162 : *
1163 : * Returns the length of the (null-terminated) @buf string, zero if
1164 : * nothing is copied.
1165 : */
1166 : static inline ssize_t
1167 : cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
1168 : {
1169 : return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
1170 : nr_cpu_ids);
1171 : }
1172 :
1173 : /**
1174 : * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
1175 : * hex values of cpumask
1176 : *
1177 : * @buf: the buffer to copy into
1178 : * @mask: the cpumask to copy
1179 : * @off: in the string from which we are copying, we copy to @buf
1180 : * @count: the maximum number of bytes to print
1181 : *
1182 : * The function prints the cpumask into the buffer as hex values of
1183 : * cpumask; Typically used by bin_attribute to export cpumask bitmask
1184 : * ABI.
1185 : *
1186 : * Returns the length of how many bytes have been copied, excluding
1187 : * terminating '\0'.
1188 : */
1189 : static inline ssize_t
1190 : cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
1191 : loff_t off, size_t count)
1192 : {
1193 : return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1194 : nr_cpu_ids, off, count) - 1;
1195 : }
1196 :
1197 : /**
1198 : * cpumap_print_list_to_buf - copies the cpumask into the buffer as
1199 : * comma-separated list of cpus
1200 : *
1201 : * Everything is same with the above cpumap_print_bitmask_to_buf()
1202 : * except the print format.
1203 : */
1204 : static inline ssize_t
1205 : cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
1206 : loff_t off, size_t count)
1207 : {
1208 : return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1209 : nr_cpu_ids, off, count) - 1;
1210 : }
1211 :
1212 : #if NR_CPUS <= BITS_PER_LONG
1213 : #define CPU_MASK_ALL \
1214 : (cpumask_t) { { \
1215 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1216 : } }
1217 : #else
1218 : #define CPU_MASK_ALL \
1219 : (cpumask_t) { { \
1220 : [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1221 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1222 : } }
1223 : #endif /* NR_CPUS > BITS_PER_LONG */
1224 :
1225 : #define CPU_MASK_NONE \
1226 : (cpumask_t) { { \
1227 : [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
1228 : } }
1229 :
1230 : #define CPU_MASK_CPU0 \
1231 : (cpumask_t) { { \
1232 : [0] = 1UL \
1233 : } }
1234 :
1235 : /*
1236 : * Provide a valid theoretical max size for cpumap and cpulist sysfs files
1237 : * to avoid breaking userspace which may allocate a buffer based on the size
1238 : * reported by e.g. fstat.
1239 : *
1240 : * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1241 : *
1242 : * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1243 : * to 2 orders of magnitude larger than 8192. And then we divide by 2 to
1244 : * cover a worst-case of every other cpu being on one of two nodes for a
1245 : * very large NR_CPUS.
1246 : *
1247 : * Use PAGE_SIZE as a minimum for smaller configurations while avoiding
1248 : * unsigned comparison to -1.
1249 : */
1250 : #define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \
1251 : ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
1252 : #define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
1253 :
1254 : #endif /* __LINUX_CPUMASK_H */
|
