History

Merge branch 'akpm' (patches from Andrew)

Mergr misc fixes from Andrew Morton: "11 fixes" Mostly VM fixes, one psi polling fix, and one parisc build fix. * emailed patches from Andrew Morton <akpm@linux-foundation.org>: mm/kasan: fix false positive invalid-free reports with CONFIG_KASAN_SW_TAGS=y mm/zsmalloc.c: fix race condition in zs_destroy_pool mm/zsmalloc.c: migration can leave pages in ZS_EMPTY indefinitely mm, page_owner: handle THP splits correctly userfaultfd_release: always remove uffd flags and clear vm_userfaultfd_ctx psi: get poll_work to run when calling poll syscall next time mm: memcontrol: flush percpu vmevents before releasing memcg mm: memcontrol: flush percpu vmstats before releasing memcg parisc: fix compilation errrors mm, page_alloc: move_freepages should not examine struct page of reserved memory mm/z3fold.c: fix race between migration and destruction

Linus Torvalds committed 7 years ago

Commit f47edb52 parents e67095f 00fb24a9 files · +260 −36Browse files

Unified · Split

9 files
  1. arch/parisc/include/asm/pgtable.h +1 −2
  2. fs/userfaultfd.c +13 −12
  3. kernel/sched/psi.c +8 −0
  4. mm/huge_memory.c +4 −0
  5. mm/kasan/common.c +8 −2
  6. mm/memcontrol.c +60 −0
  7. mm/page_alloc.c +4 −15
  8. mm/z3fold.c +89 −0
  9. mm/zsmalloc.c +73 −5
arch/parisc/include/asm/pgtable.h+1−2
@@ -2,6 +2,7 @@ · 1 unmodified line
22 #ifndef _PARISC_PGTABLE_H
33 #define _PARISC_PGTABLE_H
44
5+#include <asm/page.h>
56 #include <asm-generic/4level-fixup.h>
67
78 #include <asm/fixmap.h>
@@ -98,8 +99,6 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) · 90 unmodified lines
9899
99100 #endif /* !__ASSEMBLY__ */
100101
101−#include <asm/page.h>
102−
103102 #define pte_ERROR(e) \
104103 printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
105104 #define pmd_ERROR(e) \
fs/userfaultfd.c+13−12
@@ -880,6 +880,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 879 unmodified lines
880880 /* len == 0 means wake all */
881881 struct userfaultfd_wake_range range = { .len = 0, };
882882 unsigned long new_flags;
883+ bool still_valid;
883884
884885 WRITE_ONCE(ctx->released, true);
885886
@@ -895,8 +896,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 9 unmodified lines
895896 * taking the mmap_sem for writing.
896897 */
897898 down_write(&mm->mmap_sem);
898− if (!mmget_still_valid(mm))
899− goto skip_mm;
899+ still_valid = mmget_still_valid(mm);
900900 prev = NULL;
901901 for (vma = mm->mmap; vma; vma = vma->vm_next) {
902902 cond_resched();
@@ -907,19 +907,20 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 4 unmodified lines
907907 continue;
908908 }
909909 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
910− prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
911− new_flags, vma->anon_vma,
912− vma->vm_file, vma->vm_pgoff,
913− vma_policy(vma),
914− NULL_VM_UFFD_CTX);
915− if (prev)
916− vma = prev;
917− else
918− prev = vma;
910+ if (still_valid) {
911+ prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
912+ new_flags, vma->anon_vma,
913+ vma->vm_file, vma->vm_pgoff,
914+ vma_policy(vma),
915+ NULL_VM_UFFD_CTX);
916+ if (prev)
917+ vma = prev;
918+ else
919+ prev = vma;
920+ }
919921 vma->vm_flags = new_flags;
920922 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
921923 }
922−skip_mm:
923924 up_write(&mm->mmap_sem);
924925 mmput(mm);
925926 wakeup:
kernel/sched/psi.c+8−0
@@ -1131,7 +1131,15 @@ static void psi_trigger_destroy(struct kref *ref) · 1130 unmodified lines
11311131 * deadlock while waiting for psi_poll_work to acquire trigger_lock
11321132 */
11331133 if (kworker_to_destroy) {
1134+ /*
1135+ * After the RCU grace period has expired, the worker
1136+ * can no longer be found through group->poll_kworker.
1137+ * But it might have been already scheduled before
1138+ * that - deschedule it cleanly before destroying it.
1139+ */
11341140 kthread_cancel_delayed_work_sync(&group->poll_work);
1141+ atomic_set(&group->poll_scheduled, 0);
1142+
11351143 kthread_destroy_worker(kworker_to_destroy);
11361144 }
11371145 kfree(t);
mm/huge_memory.c+4−0
@@ -32,6 +32,7 @@ · 31 unmodified lines
3232 #include <linux/shmem_fs.h>
3333 #include <linux/oom.h>
3434 #include <linux/numa.h>
35+#include <linux/page_owner.h>
3536
3637 #include <asm/tlb.h>
3738 #include <asm/pgalloc.h>
@@ -2516,6 +2517,9 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 2478 unmodified lines
25162517 }
25172518
25182519 ClearPageCompound(head);
2520+
2521+ split_page_owner(head, HPAGE_PMD_ORDER);
2522+
25192523 /* See comment in __split_huge_page_tail() */
25202524 if (PageAnon(head)) {
25212525 /* Additional pin to swap cache */
mm/kasan/common.c+8−2
@@ -407,8 +407,14 @@ static inline bool shadow_invalid(u8 tag, s8 shadow_byte) · 406 unmodified lines
407407 if (IS_ENABLED(CONFIG_KASAN_GENERIC))
408408 return shadow_byte < 0 ||
409409 shadow_byte >= KASAN_SHADOW_SCALE_SIZE;
410− else
411− return tag != (u8)shadow_byte;
410+
411+ /* else CONFIG_KASAN_SW_TAGS: */
412+ if ((u8)shadow_byte == KASAN_TAG_INVALID)
413+ return true;
414+ if ((tag != KASAN_TAG_KERNEL) && (tag != (u8)shadow_byte))
415+ return true;
416+
417+ return false;
412418 }
413419
414420 static bool __kasan_slab_free(struct kmem_cache *cache, void *object,
mm/memcontrol.c+60−0
@@ -3260,6 +3260,60 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, · 3259 unmodified lines
32603260 }
32613261 }
32623262
3263+static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
3264+{
3265+ unsigned long stat[MEMCG_NR_STAT];
3266+ struct mem_cgroup *mi;
3267+ int node, cpu, i;
3268+
3269+ for (i = 0; i < MEMCG_NR_STAT; i++)
3270+ stat[i] = 0;
3271+
3272+ for_each_online_cpu(cpu)
3273+ for (i = 0; i < MEMCG_NR_STAT; i++)
3274+ stat[i] += raw_cpu_read(memcg->vmstats_percpu->stat[i]);
3275+
3276+ for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3277+ for (i = 0; i < MEMCG_NR_STAT; i++)
3278+ atomic_long_add(stat[i], &mi->vmstats[i]);
3279+
3280+ for_each_node(node) {
3281+ struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
3282+ struct mem_cgroup_per_node *pi;
3283+
3284+ for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3285+ stat[i] = 0;
3286+
3287+ for_each_online_cpu(cpu)
3288+ for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3289+ stat[i] += raw_cpu_read(
3290+ pn->lruvec_stat_cpu->count[i]);
3291+
3292+ for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
3293+ for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3294+ atomic_long_add(stat[i], &pi->lruvec_stat[i]);
3295+ }
3296+}
3297+
3298+static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3299+{
3300+ unsigned long events[NR_VM_EVENT_ITEMS];
3301+ struct mem_cgroup *mi;
3302+ int cpu, i;
3303+
3304+ for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3305+ events[i] = 0;
3306+
3307+ for_each_online_cpu(cpu)
3308+ for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3309+ events[i] += raw_cpu_read(
3310+ memcg->vmstats_percpu->events[i]);
3311+
3312+ for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3313+ for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3314+ atomic_long_add(events[i], &mi->vmevents[i]);
3315+}
3316+
32633317 #ifdef CONFIG_MEMCG_KMEM
32643318 static int memcg_online_kmem(struct mem_cgroup *memcg)
32653319 {
@@ -4682,6 +4736,12 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) · 1416 unmodified lines
46824736 {
46834737 int node;
46844738
4739+ /*
4740+ * Flush percpu vmstats and vmevents to guarantee the value correctness
4741+ * on parent's and all ancestor levels.
4742+ */
4743+ memcg_flush_percpu_vmstats(memcg);
4744+ memcg_flush_percpu_vmevents(memcg);
46854745 for_each_node(node)
46864746 free_mem_cgroup_per_node_info(memcg, node);
46874747 free_percpu(memcg->vmstats_percpu);
mm/page_alloc.c+4−15
@@ -2238,27 +2238,12 @@ static int move_freepages(struct zone *zone, · 2237 unmodified lines
22382238 unsigned int order;
22392239 int pages_moved = 0;
22402240
2241−#ifndef CONFIG_HOLES_IN_ZONE
2242− /*
2243− * page_zone is not safe to call in this context when
2244− * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
2245− * anyway as we check zone boundaries in move_freepages_block().
2246− * Remove at a later date when no bug reports exist related to
2247− * grouping pages by mobility
2248− */
2249− VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
2250− pfn_valid(page_to_pfn(end_page)) &&
2251− page_zone(start_page) != page_zone(end_page));
2252−#endif
22532241 for (page = start_page; page <= end_page;) {
22542242 if (!pfn_valid_within(page_to_pfn(page))) {
22552243 page++;
22562244 continue;
22572245 }
22582246
2259− /* Make sure we are not inadvertently changing nodes */
2260− VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2261−
22622247 if (!PageBuddy(page)) {
22632248 /*
22642249 * We assume that pages that could be isolated for
@@ -2273,6 +2258,10 @@ static int move_freepages(struct zone *zone, · 8 unmodified lines
22732258 continue;
22742259 }
22752260
2261+ /* Make sure we are not inadvertently changing nodes */
2262+ VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2263+ VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2264+
22762265 order = page_order(page);
22772266 move_to_free_area(page, &zone->free_area[order], migratetype);
22782267 page += 1 << order;
mm/z3fold.c+89−0
@@ -41,6 +41,7 @@ · 40 unmodified lines
4141 #include <linux/workqueue.h>
4242 #include <linux/slab.h>
4343 #include <linux/spinlock.h>
44+#include <linux/wait.h>
4445 #include <linux/zpool.h>
4546 #include <linux/magic.h>
4647
@@ -145,6 +146,8 @@ struct z3fold_header { · 98 unmodified lines
145146 * @release_wq: workqueue for safe page release
146147 * @work: work_struct for safe page release
147148 * @inode: inode for z3fold pseudo filesystem
149+ * @destroying: bool to stop migration once we start destruction
150+ * @isolated: int to count the number of pages currently in isolation
148151 *
149152 * This structure is allocated at pool creation time and maintains metadata
150153 * pertaining to a particular z3fold pool.
@@ -163,8 +166,11 @@ struct z3fold_pool { · 12 unmodified lines
163166 const struct zpool_ops *zpool_ops;
164167 struct workqueue_struct *compact_wq;
165168 struct workqueue_struct *release_wq;
169+ struct wait_queue_head isolate_wait;
166170 struct work_struct work;
167171 struct inode *inode;
172+ bool destroying;
173+ int isolated;
168174 };
169175
170176 /*
@@ -769,6 +775,7 @@ static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp, · 598 unmodified lines
769775 goto out_c;
770776 spin_lock_init(&pool->lock);
771777 spin_lock_init(&pool->stale_lock);
778+ init_waitqueue_head(&pool->isolate_wait);
772779 pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2);
773780 if (!pool->unbuddied)
774781 goto out_pool;
@@ -808,6 +815,15 @@ out: · 33 unmodified lines
808815 return NULL;
809816 }
810817
818+static bool pool_isolated_are_drained(struct z3fold_pool *pool)
819+{
820+ bool ret;
821+
822+ spin_lock(&pool->lock);
823+ ret = pool->isolated == 0;
824+ spin_unlock(&pool->lock);
825+ return ret;
826+}
811827 /**
812828 * z3fold_destroy_pool() - destroys an existing z3fold pool
813829 * @pool: the z3fold pool to be destroyed
@@ -817,6 +833,22 @@ out: · 3 unmodified lines
817833 static void z3fold_destroy_pool(struct z3fold_pool *pool)
818834 {
819835 kmem_cache_destroy(pool->c_handle);
836+ /*
837+ * We set pool-> destroying under lock to ensure that
838+ * z3fold_page_isolate() sees any changes to destroying. This way we
839+ * avoid the need for any memory barriers.
840+ */
841+
842+ spin_lock(&pool->lock);
843+ pool->destroying = true;
844+ spin_unlock(&pool->lock);
845+
846+ /*
847+ * We need to ensure that no pages are being migrated while we destroy
848+ * these workqueues, as migration can queue work on either of the
849+ * workqueues.
850+ */
851+ wait_event(pool->isolate_wait, !pool_isolated_are_drained(pool));
820852
821853 /*
822854 * We need to destroy pool->compact_wq before pool->release_wq,
@@ -1307,6 +1339,28 @@ static u64 z3fold_get_pool_size(struct z3fold_pool *pool) · 484 unmodified lines
13071339 return atomic64_read(&pool->pages_nr);
13081340 }
13091341
1342+/*
1343+ * z3fold_dec_isolated() expects to be called while pool->lock is held.
1344+ */
1345+static void z3fold_dec_isolated(struct z3fold_pool *pool)
1346+{
1347+ assert_spin_locked(&pool->lock);
1348+ VM_BUG_ON(pool->isolated <= 0);
1349+ pool->isolated--;
1350+
1351+ /*
1352+ * If we have no more isolated pages, we have to see if
1353+ * z3fold_destroy_pool() is waiting for a signal.
1354+ */
1355+ if (pool->isolated == 0 && waitqueue_active(&pool->isolate_wait))
1356+ wake_up_all(&pool->isolate_wait);
1357+}
1358+
1359+static void z3fold_inc_isolated(struct z3fold_pool *pool)
1360+{
1361+ pool->isolated++;
1362+}
1363+
13101364 static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)
13111365 {
13121366 struct z3fold_header *zhdr;
@@ -1333,6 +1387,33 @@ static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode) · 20 unmodified lines
13331387 spin_lock(&pool->lock);
13341388 if (!list_empty(&page->lru))
13351389 list_del(&page->lru);
1390+ /*
1391+ * We need to check for destruction while holding pool->lock, as
1392+ * otherwise destruction could see 0 isolated pages, and
1393+ * proceed.
1394+ */
1395+ if (unlikely(pool->destroying)) {
1396+ spin_unlock(&pool->lock);
1397+ /*
1398+ * If this page isn't stale, somebody else holds a
1399+ * reference to it. Let't drop our refcount so that they
1400+ * can call the release logic.
1401+ */
1402+ if (unlikely(kref_put(&zhdr->refcount,
1403+ release_z3fold_page_locked))) {
1404+ /*
1405+ * If we get here we have kref problems, so we
1406+ * should freak out.
1407+ */
1408+ WARN(1, "Z3fold is experiencing kref problems\n");
1409+ return false;
1410+ }
1411+ z3fold_page_unlock(zhdr);
1412+ return false;
1413+ }
1414+
1415+
1416+ z3fold_inc_isolated(pool);
13361417 spin_unlock(&pool->lock);
13371418 z3fold_page_unlock(zhdr);
13381419 return true;
@@ -1401,6 +1482,10 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa · 62 unmodified lines
14011482
14021483 queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work);
14031484
1485+ spin_lock(&pool->lock);
1486+ z3fold_dec_isolated(pool);
1487+ spin_unlock(&pool->lock);
1488+
14041489 page_mapcount_reset(page);
14051490 put_page(page);
14061491 return 0;
@@ -1420,10 +1505,14 @@ static void z3fold_page_putback(struct page *page) · 13 unmodified lines
14201505 INIT_LIST_HEAD(&page->lru);
14211506 if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {
14221507 atomic64_dec(&pool->pages_nr);
1508+ spin_lock(&pool->lock);
1509+ z3fold_dec_isolated(pool);
1510+ spin_unlock(&pool->lock);
14231511 return;
14241512 }
14251513 spin_lock(&pool->lock);
14261514 list_add(&page->lru, &pool->lru);
1515+ z3fold_dec_isolated(pool);
14271516 spin_unlock(&pool->lock);
14281517 z3fold_page_unlock(zhdr);
14291518 }
mm/zsmalloc.c+73−5
@@ -54,6 +54,7 @@ · 53 unmodified lines
5454 #include <linux/mount.h>
5555 #include <linux/pseudo_fs.h>
5656 #include <linux/migrate.h>
57+#include <linux/wait.h>
5758 #include <linux/pagemap.h>
5859 #include <linux/fs.h>
5960
@@ -268,6 +269,10 @@ struct zs_pool { · 208 unmodified lines
268269 #ifdef CONFIG_COMPACTION
269270 struct inode *inode;
270271 struct work_struct free_work;
272+ /* A wait queue for when migration races with async_free_zspage() */
273+ struct wait_queue_head migration_wait;
274+ atomic_long_t isolated_pages;
275+ bool destroying;
271276 #endif
272277 };
273278
@@ -1862,6 +1867,31 @@ static void dec_zspage_isolation(struct zspage *zspage) · 1588 unmodified lines
18621867 zspage->isolated--;
18631868 }
18641869
1870+static void putback_zspage_deferred(struct zs_pool *pool,
1871+ struct size_class *class,
1872+ struct zspage *zspage)
1873+{
1874+ enum fullness_group fg;
1875+
1876+ fg = putback_zspage(class, zspage);
1877+ if (fg == ZS_EMPTY)
1878+ schedule_work(&pool->free_work);
1879+
1880+}
1881+
1882+static inline void zs_pool_dec_isolated(struct zs_pool *pool)
1883+{
1884+ VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
1885+ atomic_long_dec(&pool->isolated_pages);
1886+ /*
1887+ * There's no possibility of racing, since wait_for_isolated_drain()
1888+ * checks the isolated count under &class->lock after enqueuing
1889+ * on migration_wait.
1890+ */
1891+ if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
1892+ wake_up_all(&pool->migration_wait);
1893+}
1894+
18651895 static void replace_sub_page(struct size_class *class, struct zspage *zspage,
18661896 struct page *newpage, struct page *oldpage)
18671897 {
@@ -1931,6 +1961,7 @@ static bool zs_page_isolate(struct page *page, isolate_mode_t mode) · 63 unmodified lines
19311961 */
19321962 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
19331963 get_zspage_mapping(zspage, &class_idx, &fullness);
1964+ atomic_long_inc(&pool->isolated_pages);
19341965 remove_zspage(class, zspage, fullness);
19351966 }
19361967
@@ -2030,8 +2061,16 @@ static int zs_page_migrate(struct address_space *mapping, struct page *newpage, · 93 unmodified lines
20302061 * Page migration is done so let's putback isolated zspage to
20312062 * the list if @page is final isolated subpage in the zspage.
20322063 */
2033− if (!is_zspage_isolated(zspage))
2034− putback_zspage(class, zspage);
2064+ if (!is_zspage_isolated(zspage)) {
2065+ /*
2066+ * We cannot race with zs_destroy_pool() here because we wait
2067+ * for isolation to hit zero before we start destroying.
2068+ * Also, we ensure that everyone can see pool->destroying before
2069+ * we start waiting.
2070+ */
2071+ putback_zspage_deferred(pool, class, zspage);
2072+ zs_pool_dec_isolated(pool);
2073+ }
20352074
20362075 reset_page(page);
20372076 put_page(page);
@@ -2077,13 +2116,12 @@ static void zs_page_putback(struct page *page) · 39 unmodified lines
20772116 spin_lock(&class->lock);
20782117 dec_zspage_isolation(zspage);
20792118 if (!is_zspage_isolated(zspage)) {
2080− fg = putback_zspage(class, zspage);
20812119 /*
20822120 * Due to page_lock, we cannot free zspage immediately
20832121 * so let's defer.
20842122 */
2085− if (fg == ZS_EMPTY)
2086− schedule_work(&pool->free_work);
2123+ putback_zspage_deferred(pool, class, zspage);
2124+ zs_pool_dec_isolated(pool);
20872125 }
20882126 spin_unlock(&class->lock);
20892127 }
@@ -2107,8 +2145,36 @@ static int zs_register_migration(struct zs_pool *pool) · 17 unmodified lines
21072145 return 0;
21082146 }
21092147
2148+static bool pool_isolated_are_drained(struct zs_pool *pool)
2149+{
2150+ return atomic_long_read(&pool->isolated_pages) == 0;
2151+}
2152+
2153+/* Function for resolving migration */
2154+static void wait_for_isolated_drain(struct zs_pool *pool)
2155+{
2156+
2157+ /*
2158+ * We're in the process of destroying the pool, so there are no
2159+ * active allocations. zs_page_isolate() fails for completely free
2160+ * zspages, so we need only wait for the zs_pool's isolated
2161+ * count to hit zero.
2162+ */
2163+ wait_event(pool->migration_wait,
2164+ pool_isolated_are_drained(pool));
2165+}
2166+
21102167 static void zs_unregister_migration(struct zs_pool *pool)
21112168 {
2169+ pool->destroying = true;
2170+ /*
2171+ * We need a memory barrier here to ensure global visibility of
2172+ * pool->destroying. Thus pool->isolated pages will either be 0 in which
2173+ * case we don't care, or it will be > 0 and pool->destroying will
2174+ * ensure that we wake up once isolation hits 0.
2175+ */
2176+ smp_mb();
2177+ wait_for_isolated_drain(pool); /* This can block */
21122178 flush_work(&pool->free_work);
21132179 iput(pool->inode);
21142180 }
@@ -2346,6 +2412,8 @@ struct zs_pool *zs_create_pool(const char *name) · 231 unmodified lines
23462412 if (!pool->name)
23472413 goto err;
23482414
2415+ init_waitqueue_head(&pool->migration_wait);
2416+
23492417 if (create_cache(pool))
23502418 goto err;
23512419