// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2023 Meta, Inc */#include <linux/bpf.h>#include <linux/bpf_mem_alloc.h>#include <linux/btf.h>#include <linux/btf_ids.h>#include <linux/cpumask.h> /** * struct bpf_cpumask - refcounted BPF cpumask wrapper structure * @cpumask: The actual cpumask embedded in the struct. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. * * Note that we explicitly embed a cpumask_t rather than a cpumask_var_t. This * is done to avoid confusing the verifier due to the typedef of cpumask_var_t * changing depending on whether CONFIG_CPUMASK_OFFSTACK is defined or not. See * the details in <linux/cpumask.h>. The consequence is that this structure is * likely a bit larger than it needs to be when CONFIG_CPUMASK_OFFSTACK is * defined due to embedding the whole NR_CPUS-size bitmap, but the extra memory * overhead is minimal. For the more typical case of CONFIG_CPUMASK_OFFSTACK * not being defined, the structure is the same size regardless. */struct bpf_cpumask { cpumask_t cpumask; refcount_t usage;}; static struct bpf_mem_alloc bpf_cpumask_ma; static bool cpu_valid(u32 cpu){ return cpu < nr_cpu_ids;} __bpf_kfunc_start_defs(); /** * bpf_cpumask_create() - Create a mutable BPF cpumask. * * Allocates a cpumask that can be queried, mutated, acquired, and released by * a BPF program. The cpumask returned by this function must either be embedded * in a map as a kptr, or freed with bpf_cpumask_release(). * * bpf_cpumask_create() allocates memory using the BPF memory allocator, and * will not block. It may return NULL if no memory is available. * * Return: * * A pointer to a new struct bpf_cpumask instance on success. * * NULL if the BPF memory allocator is out of memory. */__bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void){ struct bpf_cpumask *cpumask; /* cpumask must be the first element so struct bpf_cpumask be cast to struct cpumask. */ BUILD_BUG_ON(offsetof(struct bpf_cpumask, cpumask) != 0); cpumask = bpf_mem_cache_alloc(&bpf_cpumask_ma); if (!cpumask) return NULL; memset(cpumask, 0, sizeof(*cpumask)); refcount_set(&cpumask->usage, 1); return cpumask;} /** * bpf_cpumask_acquire() - Acquire a reference to a BPF cpumask. * @cpumask: The BPF cpumask being acquired. The cpumask must be a trusted * pointer. * * Acquires a reference to a BPF cpumask. The cpumask returned by this function * must either be embedded in a map as a kptr, or freed with * bpf_cpumask_release(). * * Return: * * The struct bpf_cpumask pointer passed to the function. * */__bpf_kfunc struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask){ refcount_inc(&cpumask->usage); return cpumask;} /** * bpf_cpumask_release() - Release a previously acquired BPF cpumask. * @cpumask: The cpumask being released. * * Releases a previously acquired reference to a BPF cpumask. When the final * reference of the BPF cpumask has been released, it is subsequently freed in * an RCU callback in the BPF memory allocator. */__bpf_kfunc void bpf_cpumask_release(struct bpf_cpumask *cpumask){ if (!refcount_dec_and_test(&cpumask->usage)) return; bpf_mem_cache_free_rcu(&bpf_cpumask_ma, cpumask);} __bpf_kfunc void bpf_cpumask_release_dtor(void *cpumask){ bpf_cpumask_release(cpumask);}CFI_NOSEAL(bpf_cpumask_release_dtor); /** * bpf_cpumask_first() - Get the index of the first nonzero bit in the cpumask. * @cpumask: The cpumask being queried. * * Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask * pointer may be safely passed to this function. * * Return: * * The index of the first nonzero bit in the struct cpumask. */__bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask){ return cpumask_first(cpumask);} /** * bpf_cpumask_first_zero() - Get the index of the first unset bit in the * cpumask. * @cpumask: The cpumask being queried. * * Find the index of the first unset bit of the cpumask. A struct bpf_cpumask * pointer may be safely passed to this function. * * Return: * * The index of the first zero bit in the struct cpumask. */__bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask){ return cpumask_first_zero(cpumask);} /** * bpf_cpumask_first_and() - Return the index of the first nonzero bit from the * AND of two cpumasks. * @src1: The first cpumask. * @src2: The second cpumask. * * Find the index of the first nonzero bit of the AND of two cpumasks. * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. * * Return: * * The index of the first bit that is nonzero in both cpumask instances. */__bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1, const struct cpumask *src2){ return cpumask_first_and(src1, src2);} /** * bpf_cpumask_set_cpu() - Set a bit for a CPU in a BPF cpumask. * @cpu: The CPU to be set in the cpumask. * @cpumask: The BPF cpumask in which a bit is being set. */__bpf_kfunc void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask){ if (!cpu_valid(cpu)) return; cpumask_set_cpu(cpu, (struct cpumask *)cpumask);} /** * bpf_cpumask_clear_cpu() - Clear a bit for a CPU in a BPF cpumask. * @cpu: The CPU to be cleared from the cpumask. * @cpumask: The BPF cpumask in which a bit is being cleared. */__bpf_kfunc void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask){ if (!cpu_valid(cpu)) return; cpumask_clear_cpu(cpu, (struct cpumask *)cpumask);} /** * bpf_cpumask_test_cpu() - Test whether a CPU is set in a cpumask. * @cpu: The CPU being queried for. * @cpumask: The cpumask being queried for containing a CPU. * * Return: * * true - @cpu is set in the cpumask * * false - @cpu was not set in the cpumask, or @cpu is an invalid cpu. */__bpf_kfunc bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask){ if (!cpu_valid(cpu)) return false; return cpumask_test_cpu(cpu, (struct cpumask *)cpumask);} /** * bpf_cpumask_test_and_set_cpu() - Atomically test and set a CPU in a BPF cpumask. * @cpu: The CPU being set and queried for. * @cpumask: The BPF cpumask being set and queried for containing a CPU. * * Return: * * true - @cpu is set in the cpumask * * false - @cpu was not set in the cpumask, or @cpu is invalid. */__bpf_kfunc bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask){ if (!cpu_valid(cpu)) return false; return cpumask_test_and_set_cpu(cpu, (struct cpumask *)cpumask);} /** * bpf_cpumask_test_and_clear_cpu() - Atomically test and clear a CPU in a BPF * cpumask. * @cpu: The CPU being cleared and queried for. * @cpumask: The BPF cpumask being cleared and queried for containing a CPU. * * Return: * * true - @cpu is set in the cpumask * * false - @cpu was not set in the cpumask, or @cpu is invalid. */__bpf_kfunc bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask){ if (!cpu_valid(cpu)) return false; return cpumask_test_and_clear_cpu(cpu, (struct cpumask *)cpumask);} /** * bpf_cpumask_setall() - Set all of the bits in a BPF cpumask. * @cpumask: The BPF cpumask having all of its bits set. */__bpf_kfunc void bpf_cpumask_setall(struct bpf_cpumask *cpumask){ cpumask_setall((struct cpumask *)cpumask);} /** * bpf_cpumask_clear() - Clear all of the bits in a BPF cpumask. * @cpumask: The BPF cpumask being cleared. */__bpf_kfunc void bpf_cpumask_clear(struct bpf_cpumask *cpumask){ cpumask_clear((struct cpumask *)cpumask);} /** * bpf_cpumask_and() - AND two cpumasks and store the result. * @dst: The BPF cpumask where the result is being stored. * @src1: The first input. * @src2: The second input. * * Return: * * true - @dst has at least one bit set following the operation * * false - @dst is empty following the operation * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, const struct cpumask *src2){ return cpumask_and((struct cpumask *)dst, src1, src2);} /** * bpf_cpumask_or() - OR two cpumasks and store the result. * @dst: The BPF cpumask where the result is being stored. * @src1: The first input. * @src2: The second input. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, const struct cpumask *src2){ cpumask_or((struct cpumask *)dst, src1, src2);} /** * bpf_cpumask_xor() - XOR two cpumasks and store the result. * @dst: The BPF cpumask where the result is being stored. * @src1: The first input. * @src2: The second input. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, const struct cpumask *src2){ cpumask_xor((struct cpumask *)dst, src1, src2);} /** * bpf_cpumask_equal() - Check two cpumasks for equality. * @src1: The first input. * @src2: The second input. * * Return: * * true - @src1 and @src2 have the same bits set. * * false - @src1 and @src2 differ in at least one bit. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2){ return cpumask_equal(src1, src2);} /** * bpf_cpumask_intersects() - Check two cpumasks for overlap. * @src1: The first input. * @src2: The second input. * * Return: * * true - @src1 and @src2 have at least one of the same bits set. * * false - @src1 and @src2 don't have any of the same bits set. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2){ return cpumask_intersects(src1, src2);} /** * bpf_cpumask_subset() - Check if a cpumask is a subset of another. * @src1: The first cpumask being checked as a subset. * @src2: The second cpumask being checked as a superset. * * Return: * * true - All of the bits of @src1 are set in @src2. * * false - At least one bit in @src1 is not set in @src2. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2){ return cpumask_subset(src1, src2);} /** * bpf_cpumask_empty() - Check if a cpumask is empty. * @cpumask: The cpumask being checked. * * Return: * * true - None of the bits in @cpumask are set. * * false - At least one bit in @cpumask is set. * * A struct bpf_cpumask pointer may be safely passed to @cpumask. */__bpf_kfunc bool bpf_cpumask_empty(const struct cpumask *cpumask){ return cpumask_empty(cpumask);} /** * bpf_cpumask_full() - Check if a cpumask has all bits set. * @cpumask: The cpumask being checked. * * Return: * * true - All of the bits in @cpumask are set. * * false - At least one bit in @cpumask is cleared. * * A struct bpf_cpumask pointer may be safely passed to @cpumask. */__bpf_kfunc bool bpf_cpumask_full(const struct cpumask *cpumask){ return cpumask_full(cpumask);} /** * bpf_cpumask_copy() - Copy the contents of a cpumask into a BPF cpumask. * @dst: The BPF cpumask being copied into. * @src: The cpumask being copied. * * A struct bpf_cpumask pointer may be safely passed to @src. */__bpf_kfunc void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src){ cpumask_copy((struct cpumask *)dst, src);} /** * bpf_cpumask_any_distribute() - Return a random set CPU from a cpumask. * @cpumask: The cpumask being queried. * * Return: * * A random set bit within [0, num_cpus) if at least one bit is set. * * >= num_cpus if no bit is set. * * A struct bpf_cpumask pointer may be safely passed to @src. */__bpf_kfunc u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask){ return cpumask_any_distribute(cpumask);} /** * bpf_cpumask_any_and_distribute() - Return a random set CPU from the AND of * two cpumasks. * @src1: The first cpumask. * @src2: The second cpumask. * * Return: * * A random set bit within [0, num_cpus) from the AND of two cpumasks, if at * least one bit is set. * * >= num_cpus if no bit is set. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */__bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, const struct cpumask *src2){ return cpumask_any_and_distribute(src1, src2);} /** * bpf_cpumask_weight() - Return the number of bits in @cpumask. * @cpumask: The cpumask being queried. * * Count the number of set bits in the given cpumask. * * Return: * * The number of bits set in the mask. */__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask){ return cpumask_weight(cpumask);} /** * bpf_cpumask_populate() - Populate the CPU mask from the contents of * a BPF memory region. * * @cpumask: The cpumask being populated. * @src: The BPF memory holding the bit pattern. * @src__sz: Length of the BPF memory region in bytes. * * Return: * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz){ unsigned long source = (unsigned long)src; /* The memory region must be large enough to populate the entire CPU mask. */ if (src__sz < bitmap_size(nr_cpu_ids)) return -EACCES; /* If avoiding unaligned accesses, the input region must be aligned to the nearest long. */ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && !IS_ALIGNED(source, sizeof(long))) return -EINVAL; bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0;} __bpf_kfunc_end_defs(); BTF_KFUNCS_START(cpumask_kfunc_btf_ids)BTF_ID_FLAGS(func, bpf_cpumask_create, KF_ACQUIRE | KF_RET_NULL)BTF_ID_FLAGS(func, bpf_cpumask_release, KF_RELEASE)BTF_ID_FLAGS(func, bpf_cpumask_acquire, KF_ACQUIRE)BTF_ID_FLAGS(func, bpf_cpumask_first, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_first_zero, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_first_and, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_set_cpu, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_clear_cpu, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_test_cpu, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_test_and_set_cpu, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_test_and_clear_cpu, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_setall, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_clear, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_and, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_or, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_xor, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_equal, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_intersects, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_subset, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_empty, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_full, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU)BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)Lines 1–500 of 534. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2023 Meta, Inc */ #include <linux/bpf.h> #include <linux/bpf_mem_alloc.h> #include <linux/btf.h> #include <linux/btf_ids.h> #include <linux/cpumask.h> /** * struct bpf_cpumask - refcounted BPF cpumask wrapper structure * @cpumask: The actual cpumask embedded in the struct. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. * * Note that we explicitly embed a cpumask_t rather than a cpumask_var_t. This * is done to avoid confusing the verifier due to the typedef of cpumask_var_t * changing depending on whether CONFIG_CPUMASK_OFFSTACK is defined or not. See * the details in <linux/cpumask.h>. The consequence is that this structure is * likely a bit larger than it needs to be when CONFIG_CPUMASK_OFFSTACK is * defined due to embedding the whole NR_CPUS-size bitmap, but the extra memory * overhead is minimal. For the more typical case of CONFIG_CPUMASK_OFFSTACK * not being defined, the structure is the same size regardless. */ struct bpf_cpumask { cpumask_t cpumask; refcount_t usage; }; static struct bpf_mem_alloc bpf_cpumask_ma; static bool cpu_valid(u32 cpu) { return cpu < nr_cpu_ids; } __bpf_kfunc_start_defs(); /** * bpf_cpumask_create() - Create a mutable BPF cpumask. * * Allocates a cpumask that can be queried, mutated, acquired, and released by * a BPF program. The cpumask returned by this function must either be embedded * in a map as a kptr, or freed with bpf_cpumask_release(). * * bpf_cpumask_create() allocates memory using the BPF memory allocator, and * will not block. It may return NULL if no memory is available. * * Return: * * A pointer to a new struct bpf_cpumask instance on success. * * NULL if the BPF memory allocator is out of memory. */ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void) { struct bpf_cpumask *cpumask; /* cpumask must be the first element so struct bpf_cpumask be cast to struct cpumask. */ BUILD_BUG_ON(offsetof(struct bpf_cpumask, cpumask) != 0); cpumask = bpf_mem_cache_alloc(&bpf_cpumask_ma); if (!cpumask) return NULL; memset(cpumask, 0, sizeof(*cpumask)); refcount_set(&cpumask->usage, 1); return cpumask; } /** * bpf_cpumask_acquire() - Acquire a reference to a BPF cpumask. * @cpumask: The BPF cpumask being acquired. The cpumask must be a trusted * pointer. * * Acquires a reference to a BPF cpumask. The cpumask returned by this function * must either be embedded in a map as a kptr, or freed with * bpf_cpumask_release(). * * Return: * * The struct bpf_cpumask pointer passed to the function. * */ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) { refcount_inc(&cpumask->usage); return cpumask; } /** * bpf_cpumask_release() - Release a previously acquired BPF cpumask. * @cpumask: The cpumask being released. * * Releases a previously acquired reference to a BPF cpumask. When the final * reference of the BPF cpumask has been released, it is subsequently freed in * an RCU callback in the BPF memory allocator. */ __bpf_kfunc void bpf_cpumask_release(struct bpf_cpumask *cpumask) { if (!refcount_dec_and_test(&cpumask->usage)) return; bpf_mem_cache_free_rcu(&bpf_cpumask_ma, cpumask); } __bpf_kfunc void bpf_cpumask_release_dtor(void *cpumask) { bpf_cpumask_release(cpumask); } CFI_NOSEAL(bpf_cpumask_release_dtor); /** * bpf_cpumask_first() - Get the index of the first nonzero bit in the cpumask. * @cpumask: The cpumask being queried. * * Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask * pointer may be safely passed to this function. * * Return: * * The index of the first nonzero bit in the struct cpumask. */ __bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask) { return cpumask_first(cpumask); } /** * bpf_cpumask_first_zero() - Get the index of the first unset bit in the * cpumask. * @cpumask: The cpumask being queried. * * Find the index of the first unset bit of the cpumask. A struct bpf_cpumask * pointer may be safely passed to this function. * * Return: * * The index of the first zero bit in the struct cpumask. */ __bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) { return cpumask_first_zero(cpumask); } /** * bpf_cpumask_first_and() - Return the index of the first nonzero bit from the * AND of two cpumasks. * @src1: The first cpumask. * @src2: The second cpumask. * * Find the index of the first nonzero bit of the AND of two cpumasks. * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. * * Return: * * The index of the first bit that is nonzero in both cpumask instances. */ __bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1, const struct cpumask *src2) { return cpumask_first_and(src1, src2); } /** * bpf_cpumask_set_cpu() - Set a bit for a CPU in a BPF cpumask. * @cpu: The CPU to be set in the cpumask. * @cpumask: The BPF cpumask in which a bit is being set. */ __bpf_kfunc void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) { if (!cpu_valid(cpu)) return; cpumask_set_cpu(cpu, (struct cpumask *)cpumask); } /** * bpf_cpumask_clear_cpu() - Clear a bit for a CPU in a BPF cpumask. * @cpu: The CPU to be cleared from the cpumask. * @cpumask: The BPF cpumask in which a bit is being cleared. */ __bpf_kfunc void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) { if (!cpu_valid(cpu)) return; cpumask_clear_cpu(cpu, (struct cpumask *)cpumask); } /** * bpf_cpumask_test_cpu() - Test whether a CPU is set in a cpumask. * @cpu: The CPU being queried for. * @cpumask: The cpumask being queried for containing a CPU. * * Return: * * true - @cpu is set in the cpumask * * false - @cpu was not set in the cpumask, or @cpu is an invalid cpu. */ __bpf_kfunc bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) { if (!cpu_valid(cpu)) return false; return cpumask_test_cpu(cpu, (struct cpumask *)cpumask); } /** * bpf_cpumask_test_and_set_cpu() - Atomically test and set a CPU in a BPF cpumask. * @cpu: The CPU being set and queried for. * @cpumask: The BPF cpumask being set and queried for containing a CPU. * * Return: * * true - @cpu is set in the cpumask * * false - @cpu was not set in the cpumask, or @cpu is invalid. */ __bpf_kfunc bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) { if (!cpu_valid(cpu)) return false; return cpumask_test_and_set_cpu(cpu, (struct cpumask *)cpumask); } /** * bpf_cpumask_test_and_clear_cpu() - Atomically test and clear a CPU in a BPF * cpumask. * @cpu: The CPU being cleared and queried for. * @cpumask: The BPF cpumask being cleared and queried for containing a CPU. * * Return: * * true - @cpu is set in the cpumask * * false - @cpu was not set in the cpumask, or @cpu is invalid. */ __bpf_kfunc bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) { if (!cpu_valid(cpu)) return false; return cpumask_test_and_clear_cpu(cpu, (struct cpumask *)cpumask); } /** * bpf_cpumask_setall() - Set all of the bits in a BPF cpumask. * @cpumask: The BPF cpumask having all of its bits set. */ __bpf_kfunc void bpf_cpumask_setall(struct bpf_cpumask *cpumask) { cpumask_setall((struct cpumask *)cpumask); } /** * bpf_cpumask_clear() - Clear all of the bits in a BPF cpumask. * @cpumask: The BPF cpumask being cleared. */ __bpf_kfunc void bpf_cpumask_clear(struct bpf_cpumask *cpumask) { cpumask_clear((struct cpumask *)cpumask); } /** * bpf_cpumask_and() - AND two cpumasks and store the result. * @dst: The BPF cpumask where the result is being stored. * @src1: The first input. * @src2: The second input. * * Return: * * true - @dst has at least one bit set following the operation * * false - @dst is empty following the operation * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, const struct cpumask *src2) { return cpumask_and((struct cpumask *)dst, src1, src2); } /** * bpf_cpumask_or() - OR two cpumasks and store the result. * @dst: The BPF cpumask where the result is being stored. * @src1: The first input. * @src2: The second input. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, const struct cpumask *src2) { cpumask_or((struct cpumask *)dst, src1, src2); } /** * bpf_cpumask_xor() - XOR two cpumasks and store the result. * @dst: The BPF cpumask where the result is being stored. * @src1: The first input. * @src2: The second input. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, const struct cpumask *src2) { cpumask_xor((struct cpumask *)dst, src1, src2); } /** * bpf_cpumask_equal() - Check two cpumasks for equality. * @src1: The first input. * @src2: The second input. * * Return: * * true - @src1 and @src2 have the same bits set. * * false - @src1 and @src2 differ in at least one bit. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) { return cpumask_equal(src1, src2); } /** * bpf_cpumask_intersects() - Check two cpumasks for overlap. * @src1: The first input. * @src2: The second input. * * Return: * * true - @src1 and @src2 have at least one of the same bits set. * * false - @src1 and @src2 don't have any of the same bits set. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) { return cpumask_intersects(src1, src2); } /** * bpf_cpumask_subset() - Check if a cpumask is a subset of another. * @src1: The first cpumask being checked as a subset. * @src2: The second cpumask being checked as a superset. * * Return: * * true - All of the bits of @src1 are set in @src2. * * false - At least one bit in @src1 is not set in @src2. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) { return cpumask_subset(src1, src2); } /** * bpf_cpumask_empty() - Check if a cpumask is empty. * @cpumask: The cpumask being checked. * * Return: * * true - None of the bits in @cpumask are set. * * false - At least one bit in @cpumask is set. * * A struct bpf_cpumask pointer may be safely passed to @cpumask. */ __bpf_kfunc bool bpf_cpumask_empty(const struct cpumask *cpumask) { return cpumask_empty(cpumask); } /** * bpf_cpumask_full() - Check if a cpumask has all bits set. * @cpumask: The cpumask being checked. * * Return: * * true - All of the bits in @cpumask are set. * * false - At least one bit in @cpumask is cleared. * * A struct bpf_cpumask pointer may be safely passed to @cpumask. */ __bpf_kfunc bool bpf_cpumask_full(const struct cpumask *cpumask) { return cpumask_full(cpumask); } /** * bpf_cpumask_copy() - Copy the contents of a cpumask into a BPF cpumask. * @dst: The BPF cpumask being copied into. * @src: The cpumask being copied. * * A struct bpf_cpumask pointer may be safely passed to @src. */ __bpf_kfunc void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) { cpumask_copy((struct cpumask *)dst, src); } /** * bpf_cpumask_any_distribute() - Return a random set CPU from a cpumask. * @cpumask: The cpumask being queried. * * Return: * * A random set bit within [0, num_cpus) if at least one bit is set. * * >= num_cpus if no bit is set. * * A struct bpf_cpumask pointer may be safely passed to @src. */ __bpf_kfunc u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) { return cpumask_any_distribute(cpumask); } /** * bpf_cpumask_any_and_distribute() - Return a random set CPU from the AND of * two cpumasks. * @src1: The first cpumask. * @src2: The second cpumask. * * Return: * * A random set bit within [0, num_cpus) from the AND of two cpumasks, if at * least one bit is set. * * >= num_cpus if no bit is set. * * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. */ __bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, const struct cpumask *src2) { return cpumask_any_and_distribute(src1, src2); } /** * bpf_cpumask_weight() - Return the number of bits in @cpumask. * @cpumask: The cpumask being queried. * * Count the number of set bits in the given cpumask. * * Return: * * The number of bits set in the mask. */ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) { return cpumask_weight(cpumask); } /** * bpf_cpumask_populate() - Populate the CPU mask from the contents of * a BPF memory region. * * @cpumask: The cpumask being populated. * @src: The BPF memory holding the bit pattern. * @src__sz: Length of the BPF memory region in bytes. * * Return: * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ __bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; /* The memory region must be large enough to populate the entire CPU mask. */ if (src__sz < bitmap_size(nr_cpu_ids)) return -EACCES; /* If avoiding unaligned accesses, the input region must be aligned to the nearest long. */ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && !IS_ALIGNED(source, sizeof(long))) return -EINVAL; bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(cpumask_kfunc_btf_ids) BTF_ID_FLAGS(func, bpf_cpumask_create, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_cpumask_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_cpumask_acquire, KF_ACQUIRE) BTF_ID_FLAGS(func, bpf_cpumask_first, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_first_zero, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_first_and, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_set_cpu, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_clear_cpu, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_test_cpu, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_test_and_set_cpu, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_test_and_clear_cpu, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_setall, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_clear, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_and, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_or, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_xor, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_equal, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_intersects, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_subset, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_empty, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_full, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)