// SPDX-License-Identifier: GPL-2.0-only/* * fs/dax.c - Direct Access filesystem code * Copyright (c) 2013-2014 Intel Corporation * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> * Author: Ross Zwisler <ross.zwisler@linux.intel.com> */ #include <linux/atomic.h>#include <linux/blkdev.h>#include <linux/buffer_head.h>#include <linux/dax.h>#include <linux/fs.h>#include <linux/highmem.h>#include <linux/memcontrol.h>#include <linux/mm.h>#include <linux/mutex.h>#include <linux/sched.h>#include <linux/sched/signal.h>#include <linux/uio.h>#include <linux/vmstat.h>#include <linux/sizes.h>#include <linux/mmu_notifier.h>#include <linux/iomap.h>#include <linux/rmap.h>#include <linux/pgalloc.h> #define CREATE_TRACE_POINTS#include <trace/events/fs_dax.h> /* We choose 4096 entries - same as per-zone page wait tables */#define DAX_WAIT_TABLE_BITS 12#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) /* The 'colour' (ie low bits) within a PMD of a page offset. */#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT) static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; static int __init init_dax_wait_table(void){ int i; for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++) init_waitqueue_head(wait_table + i); return 0;}fs_initcall(init_dax_wait_table); /* * DAX pagecache entries use XArray value entries so they can't be mistaken * for pages. We use one bit for locking, one bit for the entry size (PMD) * and two more to tell us if the entry is a zero page or an empty entry that * is just used for locking. In total four special bits. * * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem * block allocation. */#define DAX_SHIFT (4)#define DAX_LOCKED (1UL << 0)#define DAX_PMD (1UL << 1)#define DAX_ZERO_PAGE (1UL << 2)#define DAX_EMPTY (1UL << 3) static unsigned long dax_to_pfn(void *entry){ return xa_to_value(entry) >> DAX_SHIFT;} static struct folio *dax_to_folio(void *entry){ return page_folio(pfn_to_page(dax_to_pfn(entry)));} static void *dax_make_entry(unsigned long pfn, unsigned long flags){ return xa_mk_value(flags | (pfn << DAX_SHIFT));} static bool dax_is_locked(void *entry){ return xa_to_value(entry) & DAX_LOCKED;} static unsigned int dax_entry_order(void *entry){ if (xa_to_value(entry) & DAX_PMD) return PMD_ORDER; return 0;} static unsigned long dax_is_pmd_entry(void *entry){ return xa_to_value(entry) & DAX_PMD;} static bool dax_is_pte_entry(void *entry){ return !(xa_to_value(entry) & DAX_PMD);} static int dax_is_zero_entry(void *entry){ return xa_to_value(entry) & DAX_ZERO_PAGE;} static int dax_is_empty_entry(void *entry){ return xa_to_value(entry) & DAX_EMPTY;} /* * true if the entry that was found is of a smaller order than the entry * we were looking for */static bool dax_is_conflict(void *entry){ return entry == XA_RETRY_ENTRY;} /* * DAX page cache entry locking */struct exceptional_entry_key { struct xarray *xa; pgoff_t entry_start;}; struct wait_exceptional_entry_queue { wait_queue_entry_t wait; struct exceptional_entry_key key;}; /** * enum dax_wake_mode: waitqueue wakeup behaviour * @WAKE_ALL: wake all waiters in the waitqueue * @WAKE_NEXT: wake only the first waiter in the waitqueue */enum dax_wake_mode { WAKE_ALL, WAKE_NEXT,}; static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas, void *entry, struct exceptional_entry_key *key){ unsigned long hash; unsigned long index = xas->xa_index; /* * If 'entry' is a PMD, align the 'index' that we use for the wait * queue to the start of that PMD. This ensures that all offsets in * the range covered by the PMD map to the same bit lock. */ if (dax_is_pmd_entry(entry)) index &= ~PG_PMD_COLOUR; key->xa = xas->xa; key->entry_start = index; hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS); return wait_table + hash;} static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode, int sync, void *keyp){ struct exceptional_entry_key *key = keyp; struct wait_exceptional_entry_queue *ewait = container_of(wait, struct wait_exceptional_entry_queue, wait); if (key->xa != ewait->key.xa || key->entry_start != ewait->key.entry_start) return 0; return autoremove_wake_function(wait, mode, sync, NULL);} /* * @entry may no longer be the entry at the index in the mapping. * The important information it's conveying is whether the entry at * this index used to be a PMD entry. */static void dax_wake_entry(struct xa_state *xas, void *entry, enum dax_wake_mode mode){ struct exceptional_entry_key key; wait_queue_head_t *wq; wq = dax_entry_waitqueue(xas, entry, &key); /* * Checking for locked entry and prepare_to_wait_exclusive() happens * under the i_pages lock, ditto for entry handling in our callers. * So at this point all tasks that could have seen our entry locked * must be in the waitqueue and the following check will see them. */ if (waitqueue_active(wq)) __wake_up(wq, TASK_NORMAL, mode == WAKE_ALL ? 0 : 1, &key);} /* * Look up entry in page cache, wait for it to become unlocked if it * is a DAX entry and return it. The caller must subsequently call * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry() * if it did. The entry returned may have a larger order than @order. * If @order is larger than the order of the entry found in i_pages, this * function returns a dax_is_conflict entry. * * Must be called with the i_pages lock held. */static void *get_next_unlocked_entry(struct xa_state *xas, unsigned int order){ void *entry; struct wait_exceptional_entry_queue ewait; wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; for (;;) { entry = xas_find_conflict(xas); if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) return entry; if (dax_entry_order(entry) < order) return XA_RETRY_ENTRY; if (!dax_is_locked(entry)) return entry; wq = dax_entry_waitqueue(xas, entry, &ewait.key); prepare_to_wait_exclusive(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); xas_unlock_irq(xas); xas_reset(xas); schedule(); finish_wait(wq, &ewait.wait); xas_lock_irq(xas); }} /* * Wait for the given entry to become unlocked. Caller must hold the i_pages * lock and call either put_unlocked_entry() if it did not lock the entry or * dax_unlock_entry() if it did. Returns an unlocked entry if still present. */static void *wait_entry_unlocked_exclusive(struct xa_state *xas, void *entry){ struct wait_exceptional_entry_queue ewait; wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; while (unlikely(dax_is_locked(entry))) { wq = dax_entry_waitqueue(xas, entry, &ewait.key); prepare_to_wait_exclusive(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); xas_reset(xas); xas_unlock_irq(xas); schedule(); finish_wait(wq, &ewait.wait); xas_lock_irq(xas); entry = xas_load(xas); } if (xa_is_internal(entry)) return NULL; return entry;} /* * The only thing keeping the address space around is the i_pages lock * (it's cycled in clear_inode() after removing the entries from i_pages) * After we call xas_unlock_irq(), we cannot touch xas->xa. */static void wait_entry_unlocked(struct xa_state *xas, void *entry){ struct wait_exceptional_entry_queue ewait; wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; wq = dax_entry_waitqueue(xas, entry, &ewait.key); /* * Unlike get_next_unlocked_entry() there is no guarantee that this * path ever successfully retrieves an unlocked entry before an * inode dies. Perform a non-exclusive wait in case this path * never successfully performs its own wake up. */ prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); xas_unlock_irq(xas); schedule(); finish_wait(wq, &ewait.wait);} static void put_unlocked_entry(struct xa_state *xas, void *entry, enum dax_wake_mode mode){ if (entry && !dax_is_conflict(entry)) dax_wake_entry(xas, entry, mode);} /* * We used the xa_state to get the entry, but then we locked the entry and * dropped the xa_lock, so we know the xa_state is stale and must be reset * before use. */static void dax_unlock_entry(struct xa_state *xas, void *entry){ void *old; BUG_ON(dax_is_locked(entry)); xas_reset(xas); xas_lock_irq(xas); old = xas_store(xas, entry); xas_unlock_irq(xas); BUG_ON(!dax_is_locked(old)); dax_wake_entry(xas, entry, WAKE_NEXT);} /* * Return: The entry stored at this location before it was locked. */static void *dax_lock_entry(struct xa_state *xas, void *entry){ unsigned long v = xa_to_value(entry); return xas_store(xas, xa_mk_value(v | DAX_LOCKED));} static unsigned long dax_entry_size(void *entry){ if (dax_is_zero_entry(entry)) return 0; else if (dax_is_empty_entry(entry)) return 0; else if (dax_is_pmd_entry(entry)) return PMD_SIZE; else return PAGE_SIZE;} /* * A DAX folio is considered shared if it has no mapping set and ->share (which * shares the ->index field) is non-zero. Note this may return false even if the * page is shared between multiple files but has not yet actually been mapped * into multiple address spaces. */static inline bool dax_folio_is_shared(struct folio *folio){ return !folio->mapping && folio->share;} /* * When it is called by dax_insert_entry(), the shared flag will indicate * whether this entry is shared by multiple files. If the page has not * previously been associated with any mappings the ->mapping and ->index * fields will be set. If it has already been associated with a mapping * the mapping will be cleared and the share count set. It's then up to * reverse map users like memory_failure() to call back into the filesystem to * recover ->mapping and ->index information. For example by implementing * dax_holder_operations. */static void dax_folio_make_shared(struct folio *folio){ /* * folio is not currently shared so mark it as shared by clearing * folio->mapping. */ folio->mapping = NULL; /* * folio has previously been mapped into one address space so set the * share count. */ folio->share = 1;} /** * dax_folio_reset_order - Reset a compound DAX folio to order-0 pages * @folio: The folio to reset * * Splits a compound folio back into individual order-0 pages, * clearing compound state and restoring pgmap pointers. * * Returns: the original folio order (0 if already order-0) */int dax_folio_reset_order(struct folio *folio){ struct dev_pagemap *pgmap = page_pgmap(&folio->page); int order = folio_order(folio); /* * Clear the mapping and the index/share union word. folio->share * and folio->index occupy the same union in struct folio. For * non-shared folios (mapping != NULL), the union holds folio->index * (file page offset); for shared folios (mapping == NULL), it holds * folio->share (reference count). Either way, we are releasing the * folio and both fields should be zeroed. */ folio->mapping = NULL; folio->share = 0; if (!order) { /* * Restore pgmap explicitly even for order-0 folios. For the * dax_folio_put() caller this is a no-op (same value), but * fsdev_clear_folio_state() may call this on folios that * were previously compound and need pgmap re-established. */ folio->pgmap = pgmap; return 0; } folio_reset_order(folio); for (int i = 0; i < (1UL << order); i++) { struct page *page = folio_page(folio, i); struct folio *f = (struct folio *)page; ClearPageHead(page); clear_compound_head(page); f->mapping = NULL; f->share = 0; f->pgmap = pgmap; } return order;}EXPORT_SYMBOL_GPL(dax_folio_reset_order); static inline unsigned long dax_folio_put(struct folio *folio){ unsigned long ref; int order, i; if (!dax_folio_is_shared(folio)) ref = 0; else ref = --folio->share; if (ref) return ref; order = dax_folio_reset_order(folio); /* Debug check: verify refcounts are zero for all sub-folios */ for (i = 0; i < (1UL << order); i++) { struct page *page = folio_page(folio, i); WARN_ON_ONCE(folio_ref_count((struct folio *)page)); } return ref;} static void dax_folio_init(void *entry){ struct folio *folio = dax_to_folio(entry); int order = dax_entry_order(entry); /* * Folio should have been split back to order-0 pages in * dax_folio_put() when they were removed from their * final mapping. */ WARN_ON_ONCE(folio_order(folio)); if (order > 0) { prep_compound_page(&folio->page, order); if (order > 1) INIT_LIST_HEAD(&folio->_deferred_list); WARN_ON_ONCE(folio_ref_count(folio)); }} static void dax_associate_entry(void *entry, struct address_space *mapping, struct vm_area_struct *vma, unsigned long address, bool shared){ unsigned long size = dax_entry_size(entry), index; struct folio *folio = dax_to_folio(entry); if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return; index = linear_page_index(vma, address & ~(size - 1)); if (shared && (folio->mapping || dax_folio_is_shared(folio))) { if (folio->mapping) dax_folio_make_shared(folio); WARN_ON_ONCE(!folio->share); WARN_ON_ONCE(dax_entry_order(entry) != folio_order(folio)); folio->share++; } else { WARN_ON_ONCE(folio->mapping); dax_folio_init(entry); folio = dax_to_folio(entry); folio->mapping = mapping;Lines 1–500 of 2278. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * fs/dax.c - Direct Access filesystem code * Copyright (c) 2013-2014 Intel Corporation * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> * Author: Ross Zwisler <ross.zwisler@linux.intel.com> */ #include <linux/atomic.h> #include <linux/blkdev.h> #include <linux/buffer_head.h> #include <linux/dax.h> #include <linux/fs.h> #include <linux/highmem.h> #include <linux/memcontrol.h> #include <linux/mm.h> #include <linux/mutex.h> #include <linux/sched.h> #include <linux/sched/signal.h> #include <linux/uio.h> #include <linux/vmstat.h> #include <linux/sizes.h> #include <linux/mmu_notifier.h> #include <linux/iomap.h> #include <linux/rmap.h> #include <linux/pgalloc.h> #define CREATE_TRACE_POINTS #include <trace/events/fs_dax.h> /* We choose 4096 entries - same as per-zone page wait tables */ #define DAX_WAIT_TABLE_BITS 12 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) /* The 'colour' (ie low bits) within a PMD of a page offset. */ #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) #define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT) static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; static int __init init_dax_wait_table(void) { int i; for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++) init_waitqueue_head(wait_table + i); return 0; } fs_initcall(init_dax_wait_table); /* * DAX pagecache entries use XArray value entries so they can't be mistaken * for pages. We use one bit for locking, one bit for the entry size (PMD) * and two more to tell us if the entry is a zero page or an empty entry that * is just used for locking. In total four special bits. * * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem * block allocation. */ #define DAX_SHIFT (4) #define DAX_LOCKED (1UL << 0) #define DAX_PMD (1UL << 1) #define DAX_ZERO_PAGE (1UL << 2) #define DAX_EMPTY (1UL << 3) static unsigned long dax_to_pfn(void *entry) { return xa_to_value(entry) >> DAX_SHIFT; } static struct folio *dax_to_folio(void *entry) { return page_folio(pfn_to_page(dax_to_pfn(entry))); } static void *dax_make_entry(unsigned long pfn, unsigned long flags) { return xa_mk_value(flags | (pfn << DAX_SHIFT)); } static bool dax_is_locked(void *entry) { return xa_to_value(entry) & DAX_LOCKED; } static unsigned int dax_entry_order(void *entry) { if (xa_to_value(entry) & DAX_PMD) return PMD_ORDER; return 0; } static unsigned long dax_is_pmd_entry(void *entry) { return xa_to_value(entry) & DAX_PMD; } static bool dax_is_pte_entry(void *entry) { return !(xa_to_value(entry) & DAX_PMD); } static int dax_is_zero_entry(void *entry) { return xa_to_value(entry) & DAX_ZERO_PAGE; } static int dax_is_empty_entry(void *entry) { return xa_to_value(entry) & DAX_EMPTY; } /* * true if the entry that was found is of a smaller order than the entry * we were looking for */ static bool dax_is_conflict(void *entry) { return entry == XA_RETRY_ENTRY; } /* * DAX page cache entry locking */ struct exceptional_entry_key { struct xarray *xa; pgoff_t entry_start; }; struct wait_exceptional_entry_queue { wait_queue_entry_t wait; struct exceptional_entry_key key; }; /** * enum dax_wake_mode: waitqueue wakeup behaviour * @WAKE_ALL: wake all waiters in the waitqueue * @WAKE_NEXT: wake only the first waiter in the waitqueue */ enum dax_wake_mode { WAKE_ALL, WAKE_NEXT, }; static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas, void *entry, struct exceptional_entry_key *key) { unsigned long hash; unsigned long index = xas->xa_index; /* * If 'entry' is a PMD, align the 'index' that we use for the wait * queue to the start of that PMD. This ensures that all offsets in * the range covered by the PMD map to the same bit lock. */ if (dax_is_pmd_entry(entry)) index &= ~PG_PMD_COLOUR; key->xa = xas->xa; key->entry_start = index; hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS); return wait_table + hash; } static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode, int sync, void *keyp) { struct exceptional_entry_key *key = keyp; struct wait_exceptional_entry_queue *ewait = container_of(wait, struct wait_exceptional_entry_queue, wait); if (key->xa != ewait->key.xa || key->entry_start != ewait->key.entry_start) return 0; return autoremove_wake_function(wait, mode, sync, NULL); } /* * @entry may no longer be the entry at the index in the mapping. * The important information it's conveying is whether the entry at * this index used to be a PMD entry. */ static void dax_wake_entry(struct xa_state *xas, void *entry, enum dax_wake_mode mode) { struct exceptional_entry_key key; wait_queue_head_t *wq; wq = dax_entry_waitqueue(xas, entry, &key); /* * Checking for locked entry and prepare_to_wait_exclusive() happens * under the i_pages lock, ditto for entry handling in our callers. * So at this point all tasks that could have seen our entry locked * must be in the waitqueue and the following check will see them. */ if (waitqueue_active(wq)) __wake_up(wq, TASK_NORMAL, mode == WAKE_ALL ? 0 : 1, &key); } /* * Look up entry in page cache, wait for it to become unlocked if it * is a DAX entry and return it. The caller must subsequently call * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry() * if it did. The entry returned may have a larger order than @order. * If @order is larger than the order of the entry found in i_pages, this * function returns a dax_is_conflict entry. * * Must be called with the i_pages lock held. */ static void *get_next_unlocked_entry(struct xa_state *xas, unsigned int order) { void *entry; struct wait_exceptional_entry_queue ewait; wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; for (;;) { entry = xas_find_conflict(xas); if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) return entry; if (dax_entry_order(entry) < order) return XA_RETRY_ENTRY; if (!dax_is_locked(entry)) return entry; wq = dax_entry_waitqueue(xas, entry, &ewait.key); prepare_to_wait_exclusive(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); xas_unlock_irq(xas); xas_reset(xas); schedule(); finish_wait(wq, &ewait.wait); xas_lock_irq(xas); } } /* * Wait for the given entry to become unlocked. Caller must hold the i_pages * lock and call either put_unlocked_entry() if it did not lock the entry or * dax_unlock_entry() if it did. Returns an unlocked entry if still present. */ static void *wait_entry_unlocked_exclusive(struct xa_state *xas, void *entry) { struct wait_exceptional_entry_queue ewait; wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; while (unlikely(dax_is_locked(entry))) { wq = dax_entry_waitqueue(xas, entry, &ewait.key); prepare_to_wait_exclusive(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); xas_reset(xas); xas_unlock_irq(xas); schedule(); finish_wait(wq, &ewait.wait); xas_lock_irq(xas); entry = xas_load(xas); } if (xa_is_internal(entry)) return NULL; return entry; } /* * The only thing keeping the address space around is the i_pages lock * (it's cycled in clear_inode() after removing the entries from i_pages) * After we call xas_unlock_irq(), we cannot touch xas->xa. */ static void wait_entry_unlocked(struct xa_state *xas, void *entry) { struct wait_exceptional_entry_queue ewait; wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; wq = dax_entry_waitqueue(xas, entry, &ewait.key); /* * Unlike get_next_unlocked_entry() there is no guarantee that this * path ever successfully retrieves an unlocked entry before an * inode dies. Perform a non-exclusive wait in case this path * never successfully performs its own wake up. */ prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); xas_unlock_irq(xas); schedule(); finish_wait(wq, &ewait.wait); } static void put_unlocked_entry(struct xa_state *xas, void *entry, enum dax_wake_mode mode) { if (entry && !dax_is_conflict(entry)) dax_wake_entry(xas, entry, mode); } /* * We used the xa_state to get the entry, but then we locked the entry and * dropped the xa_lock, so we know the xa_state is stale and must be reset * before use. */ static void dax_unlock_entry(struct xa_state *xas, void *entry) { void *old; BUG_ON(dax_is_locked(entry)); xas_reset(xas); xas_lock_irq(xas); old = xas_store(xas, entry); xas_unlock_irq(xas); BUG_ON(!dax_is_locked(old)); dax_wake_entry(xas, entry, WAKE_NEXT); } /* * Return: The entry stored at this location before it was locked. */ static void *dax_lock_entry(struct xa_state *xas, void *entry) { unsigned long v = xa_to_value(entry); return xas_store(xas, xa_mk_value(v | DAX_LOCKED)); } static unsigned long dax_entry_size(void *entry) { if (dax_is_zero_entry(entry)) return 0; else if (dax_is_empty_entry(entry)) return 0; else if (dax_is_pmd_entry(entry)) return PMD_SIZE; else return PAGE_SIZE; } /* * A DAX folio is considered shared if it has no mapping set and ->share (which * shares the ->index field) is non-zero. Note this may return false even if the * page is shared between multiple files but has not yet actually been mapped * into multiple address spaces. */ static inline bool dax_folio_is_shared(struct folio *folio) { return !folio->mapping && folio->share; } /* * When it is called by dax_insert_entry(), the shared flag will indicate * whether this entry is shared by multiple files. If the page has not * previously been associated with any mappings the ->mapping and ->index * fields will be set. If it has already been associated with a mapping * the mapping will be cleared and the share count set. It's then up to * reverse map users like memory_failure() to call back into the filesystem to * recover ->mapping and ->index information. For example by implementing * dax_holder_operations. */ static void dax_folio_make_shared(struct folio *folio) { /* * folio is not currently shared so mark it as shared by clearing * folio->mapping. */ folio->mapping = NULL; /* * folio has previously been mapped into one address space so set the * share count. */ folio->share = 1; } /** * dax_folio_reset_order - Reset a compound DAX folio to order-0 pages * @folio: The folio to reset * * Splits a compound folio back into individual order-0 pages, * clearing compound state and restoring pgmap pointers. * * Returns: the original folio order (0 if already order-0) */ int dax_folio_reset_order(struct folio *folio) { struct dev_pagemap *pgmap = page_pgmap(&folio->page); int order = folio_order(folio); /* * Clear the mapping and the index/share union word. folio->share * and folio->index occupy the same union in struct folio. For * non-shared folios (mapping != NULL), the union holds folio->index * (file page offset); for shared folios (mapping == NULL), it holds * folio->share (reference count). Either way, we are releasing the * folio and both fields should be zeroed. */ folio->mapping = NULL; folio->share = 0; if (!order) { /* * Restore pgmap explicitly even for order-0 folios. For the * dax_folio_put() caller this is a no-op (same value), but * fsdev_clear_folio_state() may call this on folios that * were previously compound and need pgmap re-established. */ folio->pgmap = pgmap; return 0; } folio_reset_order(folio); for (int i = 0; i < (1UL << order); i++) { struct page *page = folio_page(folio, i); struct folio *f = (struct folio *)page; ClearPageHead(page); clear_compound_head(page); f->mapping = NULL; f->share = 0; f->pgmap = pgmap; } return order; } EXPORT_SYMBOL_GPL(dax_folio_reset_order); static inline unsigned long dax_folio_put(struct folio *folio) { unsigned long ref; int order, i; if (!dax_folio_is_shared(folio)) ref = 0; else ref = --folio->share; if (ref) return ref; order = dax_folio_reset_order(folio); /* Debug check: verify refcounts are zero for all sub-folios */ for (i = 0; i < (1UL << order); i++) { struct page *page = folio_page(folio, i); WARN_ON_ONCE(folio_ref_count((struct folio *)page)); } return ref; } static void dax_folio_init(void *entry) { struct folio *folio = dax_to_folio(entry); int order = dax_entry_order(entry); /* * Folio should have been split back to order-0 pages in * dax_folio_put() when they were removed from their * final mapping. */ WARN_ON_ONCE(folio_order(folio)); if (order > 0) { prep_compound_page(&folio->page, order); if (order > 1) INIT_LIST_HEAD(&folio->_deferred_list); WARN_ON_ONCE(folio_ref_count(folio)); } } static void dax_associate_entry(void *entry, struct address_space *mapping, struct vm_area_struct *vma, unsigned long address, bool shared) { unsigned long size = dax_entry_size(entry), index; struct folio *folio = dax_to_folio(entry); if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return; index = linear_page_index(vma, address & ~(size - 1)); if (shared && (folio->mapping || dax_folio_is_shared(folio))) { if (folio->mapping) dax_folio_make_shared(folio); WARN_ON_ONCE(!folio->share); WARN_ON_ONCE(dax_entry_order(entry) != folio_order(folio)); folio->share++; } else { WARN_ON_ONCE(folio->mapping); dax_folio_init(entry); folio = dax_to_folio(entry); folio->mapping = mapping;