// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2001 Sistina Software (UK) Limited. * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. * * This file is released under the GPL. */ #include "dm-core.h"#include "dm-rq.h" #include <linux/module.h>#include <linux/vmalloc.h>#include <linux/blkdev.h>#include <linux/blk-integrity.h>#include <linux/namei.h>#include <linux/ctype.h>#include <linux/string.h>#include <linux/slab.h>#include <linux/interrupt.h>#include <linux/mutex.h>#include <linux/delay.h>#include <linux/atomic.h>#include <linux/blk-mq.h>#include <linux/mount.h>#include <linux/dax.h> #define DM_MSG_PREFIX "table" #define NODE_SIZE L1_CACHE_BYTES#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1) /* * Similar to ceiling(log_size(n)) */static unsigned int int_log(unsigned int n, unsigned int base){ int result = 0; while (n > 1) { n = dm_div_up(n, base); result++; } return result;} /* * Calculate the index of the child node of the n'th node k'th key. */static inline unsigned int get_child(unsigned int n, unsigned int k){ return (n * CHILDREN_PER_NODE) + k;} /* * Return the n'th node of level l from table t. */static inline sector_t *get_node(struct dm_table *t, unsigned int l, unsigned int n){ return t->index[l] + (n * KEYS_PER_NODE);} /* * Return the highest key that you could lookup from the n'th * node on level l of the btree. */static sector_t high(struct dm_table *t, unsigned int l, unsigned int n){ for (; l < t->depth - 1; l++) n = get_child(n, CHILDREN_PER_NODE - 1); if (n >= t->counts[l]) return (sector_t) -1; return get_node(t, l, n)[KEYS_PER_NODE - 1];} /* * Fills in a level of the btree based on the highs of the level * below it. */static int setup_btree_index(unsigned int l, struct dm_table *t){ unsigned int n, k; sector_t *node; for (n = 0U; n < t->counts[l]; n++) { node = get_node(t, l, n); for (k = 0U; k < KEYS_PER_NODE; k++) node[k] = high(t, l + 1, get_child(n, k)); } return 0;} /* * highs, and targets are managed as dynamic arrays during a * table load. */static int alloc_targets(struct dm_table *t, unsigned int num){ sector_t *n_highs; struct dm_target *n_targets; /* * Allocate both the target array and offset array at once. */ n_highs = kvcalloc(num, sizeof(struct dm_target) + sizeof(sector_t), GFP_KERNEL); if (!n_highs) return -ENOMEM; n_targets = (struct dm_target *) (n_highs + num); memset(n_highs, -1, sizeof(*n_highs) * num); t->num_allocated = num; t->highs = n_highs; t->targets = n_targets; return 0;} int dm_table_create(struct dm_table **result, blk_mode_t mode, unsigned int num_targets, struct mapped_device *md){ struct dm_table *t; if (num_targets > DM_MAX_TARGETS) return -EOVERFLOW; t = kzalloc_obj(*t); if (!t) return -ENOMEM; INIT_LIST_HEAD(&t->devices); if (!num_targets) num_targets = KEYS_PER_NODE; num_targets = dm_round_up(num_targets, KEYS_PER_NODE); if (!num_targets) { kfree(t); return -EOVERFLOW; } if (alloc_targets(t, num_targets)) { kfree(t); return -ENOMEM; } t->type = DM_TYPE_NONE; t->mode = mode; t->md = md; t->flush_bypasses_map = true; *result = t; return 0;} static void free_devices(struct list_head *devices, struct mapped_device *md){ struct list_head *tmp, *next; list_for_each_safe(tmp, next, devices) { struct dm_dev_internal *dd = list_entry(tmp, struct dm_dev_internal, list); DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s", dm_device_name(md), dd->dm_dev->name); dm_put_table_device(md, dd->dm_dev); kfree(dd); }} static void dm_table_destroy_crypto_profile(struct dm_table *t); void dm_table_destroy(struct dm_table *t){ if (!t) return; /* free the indexes */ if (t->depth >= 2) kvfree(t->index[t->depth - 2]); /* free the targets */ for (unsigned int i = 0; i < t->num_targets; i++) { struct dm_target *ti = dm_table_get_target(t, i); if (ti->type->dtr) ti->type->dtr(ti); dm_put_target_type(ti->type); } kvfree(t->highs); /* free the device list */ free_devices(&t->devices, t->md); dm_free_md_mempools(t->mempools); dm_table_destroy_crypto_profile(t); kfree(t);} /* * See if we've already got a device in the list. */static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev){ struct dm_dev_internal *dd; list_for_each_entry(dd, l, list) if (dd->dm_dev->bdev->bd_dev == dev) return dd; return NULL;} /* * If possible, this checks an area of a destination device is invalid. */static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data){ struct queue_limits *limits = data; struct block_device *bdev = dev->bdev; sector_t dev_size = bdev_nr_sectors(bdev); unsigned short logical_block_size_sectors = limits->logical_block_size >> SECTOR_SHIFT; if (!dev_size) return 0; if ((start >= dev_size) || (start + len > dev_size)) { DMERR("%s: %pg too small for target: start=%llu, len=%llu, dev_size=%llu", dm_device_name(ti->table->md), bdev, (unsigned long long)start, (unsigned long long)len, (unsigned long long)dev_size); return 1; } /* * If the target is mapped to zoned block device(s), check * that the zones are not partially mapped. */ if (bdev_is_zoned(bdev)) { unsigned int zone_sectors = bdev_zone_sectors(bdev); if (!bdev_is_zone_aligned(bdev, start)) { DMERR("%s: start=%llu not aligned to h/w zone size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)start, zone_sectors, bdev); return 1; } /* * Note: The last zone of a zoned block device may be smaller * than other zones. So for a target mapping the end of a * zoned block device with such a zone, len would not be zone * aligned. We do not allow such last smaller zone to be part * of the mapping here to ensure that mappings with multiple * devices do not end up with a smaller zone in the middle of * the sector range. */ if (!bdev_is_zone_aligned(bdev, len)) { DMERR("%s: len=%llu not aligned to h/w zone size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)len, zone_sectors, bdev); return 1; } } if (logical_block_size_sectors <= 1) return 0; if (start & (logical_block_size_sectors - 1)) { DMERR("%s: start=%llu not aligned to h/w logical block size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)start, limits->logical_block_size, bdev); return 1; } if (len & (logical_block_size_sectors - 1)) { DMERR("%s: len=%llu not aligned to h/w logical block size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)len, limits->logical_block_size, bdev); return 1; } return 0;} /* * This upgrades the mode on an already open dm_dev, being * careful to leave things as they were if we fail to reopen the * device and not to touch the existing bdev field in case * it is accessed concurrently. */static int upgrade_mode(struct dm_dev_internal *dd, blk_mode_t new_mode, struct mapped_device *md){ int r; struct dm_dev *old_dev, *new_dev; old_dev = dd->dm_dev; r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev, dd->dm_dev->mode | new_mode, &new_dev); if (r) return r; dd->dm_dev = new_dev; dm_put_table_device(md, old_dev); return 0;} /* * Note: the __ref annotation is because this function can call the __init * marked early_lookup_bdev when called during early boot code from dm-init.c. */int __ref dm_devt_from_path(const char *path, dev_t *dev_p){ int r; dev_t dev; unsigned int major, minor; char dummy; if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) { /* Extract the major/minor numbers */ dev = MKDEV(major, minor); if (MAJOR(dev) != major || MINOR(dev) != minor) return -EOVERFLOW; } else { r = lookup_bdev(path, &dev);#ifndef MODULE if (r && system_state < SYSTEM_RUNNING) r = early_lookup_bdev(path, &dev);#endif if (r) return r; } *dev_p = dev; return 0;}EXPORT_SYMBOL(dm_devt_from_path); /* * Add a device to the list, or just increment the usage count if * it's already present. */int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode, struct dm_dev **result){ int r; dev_t dev; struct dm_dev_internal *dd; struct dm_table *t = ti->table; BUG_ON(!t); r = dm_devt_from_path(path, &dev); if (r) return r; if (dev == disk_devt(t->md->disk)) return -EINVAL; dd = find_device(&t->devices, dev); if (!dd) { dd = kmalloc_obj(*dd); if (!dd) return -ENOMEM; r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); if (r) { kfree(dd); return r; } refcount_set(&dd->count, 1); list_add(&dd->list, &t->devices); goto out; } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) { r = upgrade_mode(dd, mode, t->md); if (r) return r; } refcount_inc(&dd->count);out: *result = dd->dm_dev; return 0;}EXPORT_SYMBOL(dm_get_device); static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data){ struct queue_limits *limits = data; struct block_device *bdev = dev->bdev; struct request_queue *q = bdev_get_queue(bdev); if (unlikely(!q)) { DMWARN("%s: Cannot set limits for nonexistent device %pg", dm_device_name(ti->table->md), bdev); return 0; } mutex_lock(&q->limits_lock); /* * BLK_FEAT_ATOMIC_WRITES is not inherited from the bottom device in * blk_stack_limits(), so do it manually. */ limits->features |= (q->limits.features & BLK_FEAT_ATOMIC_WRITES); if (blk_stack_limits(limits, &q->limits, get_start_sect(bdev) + start) < 0) DMWARN("%s: adding target device %pg caused an alignment inconsistency: " "physical_block_size=%u, logical_block_size=%u, " "alignment_offset=%u, start=%llu", dm_device_name(ti->table->md), bdev, q->limits.physical_block_size, q->limits.logical_block_size, q->limits.alignment_offset, (unsigned long long) start << SECTOR_SHIFT); /* * Only stack the integrity profile if the target doesn't have native * integrity support. */ if (!dm_target_has_integrity(ti->type)) queue_limits_stack_integrity_bdev(limits, bdev); mutex_unlock(&q->limits_lock); return 0;} /* * Decrement a device's use count and remove it if necessary. */void dm_put_device(struct dm_target *ti, struct dm_dev *d){ int found = 0; struct list_head *devices = &ti->table->devices; struct dm_dev_internal *dd; list_for_each_entry(dd, devices, list) { if (dd->dm_dev == d) { found = 1; break; } } if (!found) { DMERR("%s: device %s not in table devices list", dm_device_name(ti->table->md), d->name); return; } if (refcount_dec_and_test(&dd->count)) { dm_put_table_device(ti->table->md, d); list_del(&dd->list); kfree(dd); }}EXPORT_SYMBOL(dm_put_device); /* * Checks to see if the target joins onto the end of the table. */static int adjoin(struct dm_table *t, struct dm_target *ti){ struct dm_target *prev; if (!t->num_targets) return !ti->begin; prev = &t->targets[t->num_targets - 1]; return (ti->begin == (prev->begin + prev->len));} /* * Used to dynamically allocate the arg array. * * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must * process messages even if some device is suspended. These messages have a * small fixed number of arguments. * * On the other hand, dm-switch needs to process bulk data using messages andLines 1–500 of 2231. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2001 Sistina Software (UK) Limited. * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. * * This file is released under the GPL. */ #include "dm-core.h" #include "dm-rq.h" #include <linux/module.h> #include <linux/vmalloc.h> #include <linux/blkdev.h> #include <linux/blk-integrity.h> #include <linux/namei.h> #include <linux/ctype.h> #include <linux/string.h> #include <linux/slab.h> #include <linux/interrupt.h> #include <linux/mutex.h> #include <linux/delay.h> #include <linux/atomic.h> #include <linux/blk-mq.h> #include <linux/mount.h> #include <linux/dax.h> #define DM_MSG_PREFIX "table" #define NODE_SIZE L1_CACHE_BYTES #define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t)) #define CHILDREN_PER_NODE (KEYS_PER_NODE + 1) /* * Similar to ceiling(log_size(n)) */ static unsigned int int_log(unsigned int n, unsigned int base) { int result = 0; while (n > 1) { n = dm_div_up(n, base); result++; } return result; } /* * Calculate the index of the child node of the n'th node k'th key. */ static inline unsigned int get_child(unsigned int n, unsigned int k) { return (n * CHILDREN_PER_NODE) + k; } /* * Return the n'th node of level l from table t. */ static inline sector_t *get_node(struct dm_table *t, unsigned int l, unsigned int n) { return t->index[l] + (n * KEYS_PER_NODE); } /* * Return the highest key that you could lookup from the n'th * node on level l of the btree. */ static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) { for (; l < t->depth - 1; l++) n = get_child(n, CHILDREN_PER_NODE - 1); if (n >= t->counts[l]) return (sector_t) -1; return get_node(t, l, n)[KEYS_PER_NODE - 1]; } /* * Fills in a level of the btree based on the highs of the level * below it. */ static int setup_btree_index(unsigned int l, struct dm_table *t) { unsigned int n, k; sector_t *node; for (n = 0U; n < t->counts[l]; n++) { node = get_node(t, l, n); for (k = 0U; k < KEYS_PER_NODE; k++) node[k] = high(t, l + 1, get_child(n, k)); } return 0; } /* * highs, and targets are managed as dynamic arrays during a * table load. */ static int alloc_targets(struct dm_table *t, unsigned int num) { sector_t *n_highs; struct dm_target *n_targets; /* * Allocate both the target array and offset array at once. */ n_highs = kvcalloc(num, sizeof(struct dm_target) + sizeof(sector_t), GFP_KERNEL); if (!n_highs) return -ENOMEM; n_targets = (struct dm_target *) (n_highs + num); memset(n_highs, -1, sizeof(*n_highs) * num); t->num_allocated = num; t->highs = n_highs; t->targets = n_targets; return 0; } int dm_table_create(struct dm_table **result, blk_mode_t mode, unsigned int num_targets, struct mapped_device *md) { struct dm_table *t; if (num_targets > DM_MAX_TARGETS) return -EOVERFLOW; t = kzalloc_obj(*t); if (!t) return -ENOMEM; INIT_LIST_HEAD(&t->devices); if (!num_targets) num_targets = KEYS_PER_NODE; num_targets = dm_round_up(num_targets, KEYS_PER_NODE); if (!num_targets) { kfree(t); return -EOVERFLOW; } if (alloc_targets(t, num_targets)) { kfree(t); return -ENOMEM; } t->type = DM_TYPE_NONE; t->mode = mode; t->md = md; t->flush_bypasses_map = true; *result = t; return 0; } static void free_devices(struct list_head *devices, struct mapped_device *md) { struct list_head *tmp, *next; list_for_each_safe(tmp, next, devices) { struct dm_dev_internal *dd = list_entry(tmp, struct dm_dev_internal, list); DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s", dm_device_name(md), dd->dm_dev->name); dm_put_table_device(md, dd->dm_dev); kfree(dd); } } static void dm_table_destroy_crypto_profile(struct dm_table *t); void dm_table_destroy(struct dm_table *t) { if (!t) return; /* free the indexes */ if (t->depth >= 2) kvfree(t->index[t->depth - 2]); /* free the targets */ for (unsigned int i = 0; i < t->num_targets; i++) { struct dm_target *ti = dm_table_get_target(t, i); if (ti->type->dtr) ti->type->dtr(ti); dm_put_target_type(ti->type); } kvfree(t->highs); /* free the device list */ free_devices(&t->devices, t->md); dm_free_md_mempools(t->mempools); dm_table_destroy_crypto_profile(t); kfree(t); } /* * See if we've already got a device in the list. */ static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev) { struct dm_dev_internal *dd; list_for_each_entry(dd, l, list) if (dd->dm_dev->bdev->bd_dev == dev) return dd; return NULL; } /* * If possible, this checks an area of a destination device is invalid. */ static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct queue_limits *limits = data; struct block_device *bdev = dev->bdev; sector_t dev_size = bdev_nr_sectors(bdev); unsigned short logical_block_size_sectors = limits->logical_block_size >> SECTOR_SHIFT; if (!dev_size) return 0; if ((start >= dev_size) || (start + len > dev_size)) { DMERR("%s: %pg too small for target: start=%llu, len=%llu, dev_size=%llu", dm_device_name(ti->table->md), bdev, (unsigned long long)start, (unsigned long long)len, (unsigned long long)dev_size); return 1; } /* * If the target is mapped to zoned block device(s), check * that the zones are not partially mapped. */ if (bdev_is_zoned(bdev)) { unsigned int zone_sectors = bdev_zone_sectors(bdev); if (!bdev_is_zone_aligned(bdev, start)) { DMERR("%s: start=%llu not aligned to h/w zone size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)start, zone_sectors, bdev); return 1; } /* * Note: The last zone of a zoned block device may be smaller * than other zones. So for a target mapping the end of a * zoned block device with such a zone, len would not be zone * aligned. We do not allow such last smaller zone to be part * of the mapping here to ensure that mappings with multiple * devices do not end up with a smaller zone in the middle of * the sector range. */ if (!bdev_is_zone_aligned(bdev, len)) { DMERR("%s: len=%llu not aligned to h/w zone size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)len, zone_sectors, bdev); return 1; } } if (logical_block_size_sectors <= 1) return 0; if (start & (logical_block_size_sectors - 1)) { DMERR("%s: start=%llu not aligned to h/w logical block size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)start, limits->logical_block_size, bdev); return 1; } if (len & (logical_block_size_sectors - 1)) { DMERR("%s: len=%llu not aligned to h/w logical block size %u of %pg", dm_device_name(ti->table->md), (unsigned long long)len, limits->logical_block_size, bdev); return 1; } return 0; } /* * This upgrades the mode on an already open dm_dev, being * careful to leave things as they were if we fail to reopen the * device and not to touch the existing bdev field in case * it is accessed concurrently. */ static int upgrade_mode(struct dm_dev_internal *dd, blk_mode_t new_mode, struct mapped_device *md) { int r; struct dm_dev *old_dev, *new_dev; old_dev = dd->dm_dev; r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev, dd->dm_dev->mode | new_mode, &new_dev); if (r) return r; dd->dm_dev = new_dev; dm_put_table_device(md, old_dev); return 0; } /* * Note: the __ref annotation is because this function can call the __init * marked early_lookup_bdev when called during early boot code from dm-init.c. */ int __ref dm_devt_from_path(const char *path, dev_t *dev_p) { int r; dev_t dev; unsigned int major, minor; char dummy; if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) { /* Extract the major/minor numbers */ dev = MKDEV(major, minor); if (MAJOR(dev) != major || MINOR(dev) != minor) return -EOVERFLOW; } else { r = lookup_bdev(path, &dev); #ifndef MODULE if (r && system_state < SYSTEM_RUNNING) r = early_lookup_bdev(path, &dev); #endif if (r) return r; } *dev_p = dev; return 0; } EXPORT_SYMBOL(dm_devt_from_path); /* * Add a device to the list, or just increment the usage count if * it's already present. */ int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode, struct dm_dev **result) { int r; dev_t dev; struct dm_dev_internal *dd; struct dm_table *t = ti->table; BUG_ON(!t); r = dm_devt_from_path(path, &dev); if (r) return r; if (dev == disk_devt(t->md->disk)) return -EINVAL; dd = find_device(&t->devices, dev); if (!dd) { dd = kmalloc_obj(*dd); if (!dd) return -ENOMEM; r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); if (r) { kfree(dd); return r; } refcount_set(&dd->count, 1); list_add(&dd->list, &t->devices); goto out; } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) { r = upgrade_mode(dd, mode, t->md); if (r) return r; } refcount_inc(&dd->count); out: *result = dd->dm_dev; return 0; } EXPORT_SYMBOL(dm_get_device); static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct queue_limits *limits = data; struct block_device *bdev = dev->bdev; struct request_queue *q = bdev_get_queue(bdev); if (unlikely(!q)) { DMWARN("%s: Cannot set limits for nonexistent device %pg", dm_device_name(ti->table->md), bdev); return 0; } mutex_lock(&q->limits_lock); /* * BLK_FEAT_ATOMIC_WRITES is not inherited from the bottom device in * blk_stack_limits(), so do it manually. */ limits->features |= (q->limits.features & BLK_FEAT_ATOMIC_WRITES); if (blk_stack_limits(limits, &q->limits, get_start_sect(bdev) + start) < 0) DMWARN("%s: adding target device %pg caused an alignment inconsistency: " "physical_block_size=%u, logical_block_size=%u, " "alignment_offset=%u, start=%llu", dm_device_name(ti->table->md), bdev, q->limits.physical_block_size, q->limits.logical_block_size, q->limits.alignment_offset, (unsigned long long) start << SECTOR_SHIFT); /* * Only stack the integrity profile if the target doesn't have native * integrity support. */ if (!dm_target_has_integrity(ti->type)) queue_limits_stack_integrity_bdev(limits, bdev); mutex_unlock(&q->limits_lock); return 0; } /* * Decrement a device's use count and remove it if necessary. */ void dm_put_device(struct dm_target *ti, struct dm_dev *d) { int found = 0; struct list_head *devices = &ti->table->devices; struct dm_dev_internal *dd; list_for_each_entry(dd, devices, list) { if (dd->dm_dev == d) { found = 1; break; } } if (!found) { DMERR("%s: device %s not in table devices list", dm_device_name(ti->table->md), d->name); return; } if (refcount_dec_and_test(&dd->count)) { dm_put_table_device(ti->table->md, d); list_del(&dd->list); kfree(dd); } } EXPORT_SYMBOL(dm_put_device); /* * Checks to see if the target joins onto the end of the table. */ static int adjoin(struct dm_table *t, struct dm_target *ti) { struct dm_target *prev; if (!t->num_targets) return !ti->begin; prev = &t->targets[t->num_targets - 1]; return (ti->begin == (prev->begin + prev->len)); } /* * Used to dynamically allocate the arg array. * * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must * process messages even if some device is suspended. These messages have a * small fixed number of arguments. * * On the other hand, dm-switch needs to process bulk data using messages and