| @@ -59,29 +59,32 @@ struct vm_area_struct; · 58 unmodified lines |
| 59 | 59 | #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ |
| 60 | 60 | #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) |
| 61 | 61 | |
| 62 | | −/* |
| 62 | +/** |
| 63 | + * DOC: Page mobility and placement hints |
| 64 | + * |
| 63 | 65 | * Page mobility and placement hints |
| 66 | + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| 64 | 67 | * |
| 65 | 68 | * These flags provide hints about how mobile the page is. Pages with similar |
| 66 | 69 | * mobility are placed within the same pageblocks to minimise problems due |
| 67 | 70 | * to external fragmentation. |
| 68 | 71 | * |
| 69 | | − * __GFP_MOVABLE (also a zone modifier) indicates that the page can be |
| 70 | | − * moved by page migration during memory compaction or can be reclaimed. |
| 72 | + * %__GFP_MOVABLE (also a zone modifier) indicates that the page can be |
| 73 | + * moved by page migration during memory compaction or can be reclaimed. |
| 71 | 74 | * |
| 72 | | − * __GFP_RECLAIMABLE is used for slab allocations that specify |
| 73 | | − * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. |
| 75 | + * %__GFP_RECLAIMABLE is used for slab allocations that specify |
| 76 | + * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. |
| 74 | 77 | * |
| 75 | | − * __GFP_WRITE indicates the caller intends to dirty the page. Where possible, |
| 76 | | − * these pages will be spread between local zones to avoid all the dirty |
| 77 | | − * pages being in one zone (fair zone allocation policy). |
| 78 | + * %__GFP_WRITE indicates the caller intends to dirty the page. Where possible, |
| 79 | + * these pages will be spread between local zones to avoid all the dirty |
| 80 | + * pages being in one zone (fair zone allocation policy). |
| 78 | 81 | * |
| 79 | | − * __GFP_HARDWALL enforces the cpuset memory allocation policy. |
| 82 | + * %__GFP_HARDWALL enforces the cpuset memory allocation policy. |
| 80 | 83 | * |
| 81 | | − * __GFP_THISNODE forces the allocation to be satisified from the requested |
| 82 | | − * node with no fallbacks or placement policy enforcements. |
| 84 | + * %__GFP_THISNODE forces the allocation to be satisified from the requested |
| 85 | + * node with no fallbacks or placement policy enforcements. |
| 83 | 86 | * |
| 84 | | − * __GFP_ACCOUNT causes the allocation to be accounted to kmemcg. |
| 87 | + * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. |
| 85 | 88 | */ |
| 86 | 89 | #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) |
| 87 | 90 | #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) |
| @@ -89,54 +92,60 @@ struct vm_area_struct; · 1 unmodified line |
| 89 | 92 | #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) |
| 90 | 93 | #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) |
| 91 | 94 | |
| 92 | | −/* |
| 95 | +/** |
| 96 | + * DOC: Watermark modifiers |
| 97 | + * |
| 93 | 98 | * Watermark modifiers -- controls access to emergency reserves |
| 99 | + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| 94 | 100 | * |
| 95 | | − * __GFP_HIGH indicates that the caller is high-priority and that granting |
| 96 | | − * the request is necessary before the system can make forward progress. |
| 97 | | − * For example, creating an IO context to clean pages. |
| 101 | + * %__GFP_HIGH indicates that the caller is high-priority and that granting |
| 102 | + * the request is necessary before the system can make forward progress. |
| 103 | + * For example, creating an IO context to clean pages. |
| 98 | 104 | * |
| 99 | | − * __GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is |
| 100 | | − * high priority. Users are typically interrupt handlers. This may be |
| 101 | | − * used in conjunction with __GFP_HIGH |
| 105 | + * %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is |
| 106 | + * high priority. Users are typically interrupt handlers. This may be |
| 107 | + * used in conjunction with %__GFP_HIGH |
| 102 | 108 | * |
| 103 | | − * __GFP_MEMALLOC allows access to all memory. This should only be used when |
| 104 | | − * the caller guarantees the allocation will allow more memory to be freed |
| 105 | | − * very shortly e.g. process exiting or swapping. Users either should |
| 106 | | − * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). |
| 109 | + * %__GFP_MEMALLOC allows access to all memory. This should only be used when |
| 110 | + * the caller guarantees the allocation will allow more memory to be freed |
| 111 | + * very shortly e.g. process exiting or swapping. Users either should |
| 112 | + * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). |
| 107 | 113 | * |
| 108 | | − * __GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. |
| 109 | | − * This takes precedence over the __GFP_MEMALLOC flag if both are set. |
| 114 | + * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. |
| 115 | + * This takes precedence over the %__GFP_MEMALLOC flag if both are set. |
| 110 | 116 | */ |
| 111 | 117 | #define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC) |
| 112 | 118 | #define __GFP_HIGH ((__force gfp_t)___GFP_HIGH) |
| 113 | 119 | #define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC) |
| 114 | 120 | #define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC) |
| 115 | 121 | |
| 116 | | −/* |
| 122 | +/** |
| 123 | + * DOC: Reclaim modifiers |
| 124 | + * |
| 117 | 125 | * Reclaim modifiers |
| 126 | + * ~~~~~~~~~~~~~~~~~ |
| 118 | 127 | * |
| 119 | | − * __GFP_IO can start physical IO. |
| 128 | + * %__GFP_IO can start physical IO. |
| 120 | 129 | * |
| 121 | | − * __GFP_FS can call down to the low-level FS. Clearing the flag avoids the |
| 122 | | − * allocator recursing into the filesystem which might already be holding |
| 123 | | − * locks. |
| 130 | + * %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the |
| 131 | + * allocator recursing into the filesystem which might already be holding |
| 132 | + * locks. |
| 124 | 133 | * |
| 125 | | − * __GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. |
| 126 | | − * This flag can be cleared to avoid unnecessary delays when a fallback |
| 127 | | − * option is available. |
| 134 | + * %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. |
| 135 | + * This flag can be cleared to avoid unnecessary delays when a fallback |
| 136 | + * option is available. |
| 128 | 137 | * |
| 129 | | − * __GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when |
| 130 | | − * the low watermark is reached and have it reclaim pages until the high |
| 131 | | − * watermark is reached. A caller may wish to clear this flag when fallback |
| 132 | | − * options are available and the reclaim is likely to disrupt the system. The |
| 133 | | − * canonical example is THP allocation where a fallback is cheap but |
| 134 | | − * reclaim/compaction may cause indirect stalls. |
| 138 | + * %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when |
| 139 | + * the low watermark is reached and have it reclaim pages until the high |
| 140 | + * watermark is reached. A caller may wish to clear this flag when fallback |
| 141 | + * options are available and the reclaim is likely to disrupt the system. The |
| 142 | + * canonical example is THP allocation where a fallback is cheap but |
| 143 | + * reclaim/compaction may cause indirect stalls. |
| 135 | 144 | * |
| 136 | | − * __GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. |
| 145 | + * %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. |
| 137 | 146 | * |
| 138 | 147 | * The default allocator behavior depends on the request size. We have a concept |
| 139 | | − * of so called costly allocations (with order > PAGE_ALLOC_COSTLY_ORDER). |
| 148 | + * of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER). |
| 140 | 149 | * !costly allocations are too essential to fail so they are implicitly |
| 141 | 150 | * non-failing by default (with some exceptions like OOM victims might fail so |
| 142 | 151 | * the caller still has to check for failures) while costly requests try to be |
| @@ -144,40 +153,40 @@ struct vm_area_struct; · 1 unmodified line |
| 144 | 153 | * The following three modifiers might be used to override some of these |
| 145 | 154 | * implicit rules |
| 146 | 155 | * |
| 147 | | − * __GFP_NORETRY: The VM implementation will try only very lightweight |
| 148 | | − * memory direct reclaim to get some memory under memory pressure (thus |
| 149 | | − * it can sleep). It will avoid disruptive actions like OOM killer. The |
| 150 | | − * caller must handle the failure which is quite likely to happen under |
| 151 | | − * heavy memory pressure. The flag is suitable when failure can easily be |
| 152 | | − * handled at small cost, such as reduced throughput |
| 153 | | − * |
| 154 | | − * __GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim |
| 155 | | − * procedures that have previously failed if there is some indication |
| 156 | | − * that progress has been made else where. It can wait for other |
| 157 | | − * tasks to attempt high level approaches to freeing memory such as |
| 158 | | − * compaction (which removes fragmentation) and page-out. |
| 159 | | − * There is still a definite limit to the number of retries, but it is |
| 160 | | − * a larger limit than with __GFP_NORETRY. |
| 161 | | − * Allocations with this flag may fail, but only when there is |
| 162 | | − * genuinely little unused memory. While these allocations do not |
| 163 | | − * directly trigger the OOM killer, their failure indicates that |
| 164 | | − * the system is likely to need to use the OOM killer soon. The |
| 165 | | − * caller must handle failure, but can reasonably do so by failing |
| 166 | | − * a higher-level request, or completing it only in a much less |
| 167 | | − * efficient manner. |
| 168 | | − * If the allocation does fail, and the caller is in a position to |
| 169 | | − * free some non-essential memory, doing so could benefit the system |
| 170 | | − * as a whole. |
| 171 | | − * |
| 172 | | − * __GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller |
| 173 | | − * cannot handle allocation failures. The allocation could block |
| 174 | | − * indefinitely but will never return with failure. Testing for |
| 175 | | − * failure is pointless. |
| 176 | | − * New users should be evaluated carefully (and the flag should be |
| 177 | | − * used only when there is no reasonable failure policy) but it is |
| 178 | | − * definitely preferable to use the flag rather than opencode endless |
| 179 | | − * loop around allocator. |
| 180 | | − * Using this flag for costly allocations is _highly_ discouraged. |
| 156 | + * %__GFP_NORETRY: The VM implementation will try only very lightweight |
| 157 | + * memory direct reclaim to get some memory under memory pressure (thus |
| 158 | + * it can sleep). It will avoid disruptive actions like OOM killer. The |
| 159 | + * caller must handle the failure which is quite likely to happen under |
| 160 | + * heavy memory pressure. The flag is suitable when failure can easily be |
| 161 | + * handled at small cost, such as reduced throughput |
| 162 | + * |
| 163 | + * %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim |
| 164 | + * procedures that have previously failed if there is some indication |
| 165 | + * that progress has been made else where. It can wait for other |
| 166 | + * tasks to attempt high level approaches to freeing memory such as |
| 167 | + * compaction (which removes fragmentation) and page-out. |
| 168 | + * There is still a definite limit to the number of retries, but it is |
| 169 | + * a larger limit than with %__GFP_NORETRY. |
| 170 | + * Allocations with this flag may fail, but only when there is |
| 171 | + * genuinely little unused memory. While these allocations do not |
| 172 | + * directly trigger the OOM killer, their failure indicates that |
| 173 | + * the system is likely to need to use the OOM killer soon. The |
| 174 | + * caller must handle failure, but can reasonably do so by failing |
| 175 | + * a higher-level request, or completing it only in a much less |
| 176 | + * efficient manner. |
| 177 | + * If the allocation does fail, and the caller is in a position to |
| 178 | + * free some non-essential memory, doing so could benefit the system |
| 179 | + * as a whole. |
| 180 | + * |
| 181 | + * %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller |
| 182 | + * cannot handle allocation failures. The allocation could block |
| 183 | + * indefinitely but will never return with failure. Testing for |
| 184 | + * failure is pointless. |
| 185 | + * New users should be evaluated carefully (and the flag should be |
| 186 | + * used only when there is no reasonable failure policy) but it is |
| 187 | + * definitely preferable to use the flag rather than opencode endless |
| 188 | + * loop around allocator. |
| 189 | + * Using this flag for costly allocations is _highly_ discouraged. |
| 181 | 190 | */ |
| 182 | 191 | #define __GFP_IO ((__force gfp_t)___GFP_IO) |
| 183 | 192 | #define __GFP_FS ((__force gfp_t)___GFP_FS) |
| @@ -188,14 +197,17 @@ struct vm_area_struct; · 4 unmodified lines |
| 188 | 197 | #define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL) |
| 189 | 198 | #define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY) |
| 190 | 199 | |
| 191 | | −/* |
| 200 | +/** |
| 201 | + * DOC: Action modifiers |
| 202 | + * |
| 192 | 203 | * Action modifiers |
| 204 | + * ~~~~~~~~~~~~~~~~ |
| 193 | 205 | * |
| 194 | | − * __GFP_NOWARN suppresses allocation failure reports. |
| 206 | + * %__GFP_NOWARN suppresses allocation failure reports. |
| 195 | 207 | * |
| 196 | | − * __GFP_COMP address compound page metadata. |
| 208 | + * %__GFP_COMP address compound page metadata. |
| 197 | 209 | * |
| 198 | | − * __GFP_ZERO returns a zeroed page on success. |
| 210 | + * %__GFP_ZERO returns a zeroed page on success. |
| 199 | 211 | */ |
| 200 | 212 | #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) |
| 201 | 213 | #define __GFP_COMP ((__force gfp_t)___GFP_COMP) |
| @@ -208,66 +220,71 @@ struct vm_area_struct; · 6 unmodified lines |
| 208 | 220 | #define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP)) |
| 209 | 221 | #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) |
| 210 | 222 | |
| 211 | | −/* |
| 223 | +/** |
| 224 | + * DOC: Useful GFP flag combinations |
| 225 | + * |
| 226 | + * Useful GFP flag combinations |
| 227 | + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| 228 | + * |
| 212 | 229 | * Useful GFP flag combinations that are commonly used. It is recommended |
| 213 | 230 | * that subsystems start with one of these combinations and then set/clear |
| 214 | | − * __GFP_FOO flags as necessary. |
| 215 | | − * |
| 216 | | − * GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower |
| 217 | | − * watermark is applied to allow access to "atomic reserves" |
| 218 | | − * |
| 219 | | − * GFP_KERNEL is typical for kernel-internal allocations. The caller requires |
| 220 | | − * ZONE_NORMAL or a lower zone for direct access but can direct reclaim. |
| 221 | | − * |
| 222 | | − * GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is |
| 223 | | − * accounted to kmemcg. |
| 224 | | − * |
| 225 | | − * GFP_NOWAIT is for kernel allocations that should not stall for direct |
| 226 | | − * reclaim, start physical IO or use any filesystem callback. |
| 227 | | − * |
| 228 | | − * GFP_NOIO will use direct reclaim to discard clean pages or slab pages |
| 229 | | − * that do not require the starting of any physical IO. |
| 230 | | − * Please try to avoid using this flag directly and instead use |
| 231 | | − * memalloc_noio_{save,restore} to mark the whole scope which cannot |
| 232 | | − * perform any IO with a short explanation why. All allocation requests |
| 233 | | − * will inherit GFP_NOIO implicitly. |
| 234 | | − * |
| 235 | | − * GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. |
| 236 | | − * Please try to avoid using this flag directly and instead use |
| 237 | | − * memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't |
| 238 | | − * recurse into the FS layer with a short explanation why. All allocation |
| 239 | | − * requests will inherit GFP_NOFS implicitly. |
| 240 | | − * |
| 241 | | − * GFP_USER is for userspace allocations that also need to be directly |
| 242 | | − * accessibly by the kernel or hardware. It is typically used by hardware |
| 243 | | − * for buffers that are mapped to userspace (e.g. graphics) that hardware |
| 244 | | − * still must DMA to. cpuset limits are enforced for these allocations. |
| 245 | | − * |
| 246 | | − * GFP_DMA exists for historical reasons and should be avoided where possible. |
| 247 | | − * The flags indicates that the caller requires that the lowest zone be |
| 248 | | − * used (ZONE_DMA or 16M on x86-64). Ideally, this would be removed but |
| 249 | | − * it would require careful auditing as some users really require it and |
| 250 | | − * others use the flag to avoid lowmem reserves in ZONE_DMA and treat the |
| 251 | | − * lowest zone as a type of emergency reserve. |
| 252 | | − * |
| 253 | | − * GFP_DMA32 is similar to GFP_DMA except that the caller requires a 32-bit |
| 254 | | − * address. |
| 255 | | − * |
| 256 | | − * GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, |
| 257 | | − * do not need to be directly accessible by the kernel but that cannot |
| 258 | | − * move once in use. An example may be a hardware allocation that maps |
| 259 | | − * data directly into userspace but has no addressing limitations. |
| 260 | | − * |
| 261 | | − * GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not |
| 262 | | − * need direct access to but can use kmap() when access is required. They |
| 263 | | − * are expected to be movable via page reclaim or page migration. Typically, |
| 264 | | − * pages on the LRU would also be allocated with GFP_HIGHUSER_MOVABLE. |
| 265 | | − * |
| 266 | | − * GFP_TRANSHUGE and GFP_TRANSHUGE_LIGHT are used for THP allocations. They are |
| 267 | | − * compound allocations that will generally fail quickly if memory is not |
| 268 | | − * available and will not wake kswapd/kcompactd on failure. The _LIGHT |
| 269 | | − * version does not attempt reclaim/compaction at all and is by default used |
| 270 | | − * in page fault path, while the non-light is used by khugepaged. |
| 231 | + * %__GFP_FOO flags as necessary. |
| 232 | + * |
| 233 | + * %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower |
| 234 | + * watermark is applied to allow access to "atomic reserves" |
| 235 | + * |
| 236 | + * %GFP_KERNEL is typical for kernel-internal allocations. The caller requires |
| 237 | + * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. |
| 238 | + * |
| 239 | + * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is |
| 240 | + * accounted to kmemcg. |
| 241 | + * |
| 242 | + * %GFP_NOWAIT is for kernel allocations that should not stall for direct |
| 243 | + * reclaim, start physical IO or use any filesystem callback. |
| 244 | + * |
| 245 | + * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages |
| 246 | + * that do not require the starting of any physical IO. |
| 247 | + * Please try to avoid using this flag directly and instead use |
| 248 | + * memalloc_noio_{save,restore} to mark the whole scope which cannot |
| 249 | + * perform any IO with a short explanation why. All allocation requests |
| 250 | + * will inherit GFP_NOIO implicitly. |
| 251 | + * |
| 252 | + * %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. |
| 253 | + * Please try to avoid using this flag directly and instead use |
| 254 | + * memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't |
| 255 | + * recurse into the FS layer with a short explanation why. All allocation |
| 256 | + * requests will inherit GFP_NOFS implicitly. |
| 257 | + * |
| 258 | + * %GFP_USER is for userspace allocations that also need to be directly |
| 259 | + * accessibly by the kernel or hardware. It is typically used by hardware |
| 260 | + * for buffers that are mapped to userspace (e.g. graphics) that hardware |
| 261 | + * still must DMA to. cpuset limits are enforced for these allocations. |
| 262 | + * |
| 263 | + * %GFP_DMA exists for historical reasons and should be avoided where possible. |
| 264 | + * The flags indicates that the caller requires that the lowest zone be |
| 265 | + * used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but |
| 266 | + * it would require careful auditing as some users really require it and |
| 267 | + * others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the |
| 268 | + * lowest zone as a type of emergency reserve. |
| 269 | + * |
| 270 | + * %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit |
| 271 | + * address. |
| 272 | + * |
| 273 | + * %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, |
| 274 | + * do not need to be directly accessible by the kernel but that cannot |
| 275 | + * move once in use. An example may be a hardware allocation that maps |
| 276 | + * data directly into userspace but has no addressing limitations. |
| 277 | + * |
| 278 | + * %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not |
| 279 | + * need direct access to but can use kmap() when access is required. They |
| 280 | + * are expected to be movable via page reclaim or page migration. Typically, |
| 281 | + * pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE. |
| 282 | + * |
| 283 | + * %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They |
| 284 | + * are compound allocations that will generally fail quickly if memory is not |
| 285 | + * available and will not wake kswapd/kcompactd on failure. The _LIGHT |
| 286 | + * version does not attempt reclaim/compaction at all and is by default used |
| 287 | + * in page fault path, while the non-light is used by khugepaged. |
| 271 | 288 | */ |
| 272 | 289 | #define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) |
| 273 | 290 | #define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) |