/* * Copyright (C) 2015 IT University of Copenhagen * Initial release: Matias Bjorling <m@bjorling.me> * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License version * 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU * General Public License for more details. * * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs. */ #include "rrpc.h" static struct kmem_cache *rrpc_gcb_cache, *rrpc_rq_cache;static DECLARE_RWSEM(rrpc_lock); static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,				struct nvm_rq *rqd, unsigned long flags); #define rrpc_for_each_lun(rrpc, rlun, i) \		for ((i) = 0, rlun = &(rrpc)->luns[0]; \			(i) < (rrpc)->nr_luns; (i)++, rlun = &(rrpc)->luns[(i)]) static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a){	struct nvm_tgt_dev *dev = rrpc->dev;	struct rrpc_block *rblk = a->rblk;	unsigned int pg_offset; 	lockdep_assert_held(&rrpc->rev_lock); 	if (a->addr == ADDR_EMPTY || !rblk)		return; 	spin_lock(&rblk->lock); 	div_u64_rem(a->addr, dev->geo.sec_per_blk, &pg_offset);	WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages));	rblk->nr_invalid_pages++; 	spin_unlock(&rblk->lock); 	rrpc->rev_trans_map[a->addr].addr = ADDR_EMPTY;} static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba,							unsigned int len){	sector_t i; 	spin_lock(&rrpc->rev_lock);	for (i = slba; i < slba + len; i++) {		struct rrpc_addr *gp = &rrpc->trans_map[i]; 		rrpc_page_invalidate(rrpc, gp);		gp->rblk = NULL;	}	spin_unlock(&rrpc->rev_lock);} static struct nvm_rq *rrpc_inflight_laddr_acquire(struct rrpc *rrpc,					sector_t laddr, unsigned int pages){	struct nvm_rq *rqd;	struct rrpc_inflight_rq *inf; 	rqd = mempool_alloc(rrpc->rq_pool, GFP_ATOMIC);	if (!rqd)		return ERR_PTR(-ENOMEM); 	inf = rrpc_get_inflight_rq(rqd);	if (rrpc_lock_laddr(rrpc, laddr, pages, inf)) {		mempool_free(rqd, rrpc->rq_pool);		return NULL;	} 	return rqd;} static void rrpc_inflight_laddr_release(struct rrpc *rrpc, struct nvm_rq *rqd){	struct rrpc_inflight_rq *inf = rrpc_get_inflight_rq(rqd); 	rrpc_unlock_laddr(rrpc, inf); 	mempool_free(rqd, rrpc->rq_pool);} static void rrpc_discard(struct rrpc *rrpc, struct bio *bio){	sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG;	sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;	struct nvm_rq *rqd; 	while (1) {		rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len);		if (rqd)			break; 		schedule();	} 	if (IS_ERR(rqd)) {		pr_err("rrpc: unable to acquire inflight IO\n");		bio_io_error(bio);		return;	} 	rrpc_invalidate_range(rrpc, slba, len);	rrpc_inflight_laddr_release(rrpc, rqd);} static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk){	struct nvm_tgt_dev *dev = rrpc->dev; 	return (rblk->next_page == dev->geo.sec_per_blk);} /* Calculate relative addr for the given block, considering instantiated LUNs */static u64 block_to_rel_addr(struct rrpc *rrpc, struct rrpc_block *rblk){	struct nvm_tgt_dev *dev = rrpc->dev;	struct rrpc_lun *rlun = rblk->rlun; 	return rlun->id * dev->geo.sec_per_blk;} static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_tgt_dev *dev,					 struct rrpc_addr *gp){	struct rrpc_block *rblk = gp->rblk;	struct rrpc_lun *rlun = rblk->rlun;	u64 addr = gp->addr;	struct ppa_addr paddr; 	paddr.ppa = addr;	paddr = rrpc_linear_to_generic_addr(&dev->geo, paddr);	paddr.g.ch = rlun->bppa.g.ch;	paddr.g.lun = rlun->bppa.g.lun;	paddr.g.blk = rblk->id; 	return paddr;} /* requires lun->lock taken */static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk,						struct rrpc_block **cur_rblk){	struct rrpc *rrpc = rlun->rrpc; 	if (*cur_rblk) {		spin_lock(&(*cur_rblk)->lock);		WARN_ON(!block_is_full(rrpc, *cur_rblk));		spin_unlock(&(*cur_rblk)->lock);	}	*cur_rblk = new_rblk;} static struct rrpc_block *__rrpc_get_blk(struct rrpc *rrpc,							struct rrpc_lun *rlun){	struct rrpc_block *rblk = NULL; 	if (list_empty(&rlun->free_list))		goto out; 	rblk = list_first_entry(&rlun->free_list, struct rrpc_block, list); 	list_move_tail(&rblk->list, &rlun->used_list);	rblk->state = NVM_BLK_ST_TGT;	rlun->nr_free_blocks--; out:	return rblk;} static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,							unsigned long flags){	struct nvm_tgt_dev *dev = rrpc->dev;	struct rrpc_block *rblk;	int is_gc = flags & NVM_IOTYPE_GC; 	spin_lock(&rlun->lock);	if (!is_gc && rlun->nr_free_blocks < rlun->reserved_blocks) {		pr_err("nvm: rrpc: cannot give block to non GC request\n");		spin_unlock(&rlun->lock);		return NULL;	} 	rblk = __rrpc_get_blk(rrpc, rlun);	if (!rblk) {		pr_err("nvm: rrpc: cannot get new block\n");		spin_unlock(&rlun->lock);		return NULL;	}	spin_unlock(&rlun->lock); 	bitmap_zero(rblk->invalid_pages, dev->geo.sec_per_blk);	rblk->next_page = 0;	rblk->nr_invalid_pages = 0;	atomic_set(&rblk->data_cmnt_size, 0); 	return rblk;} static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk){	struct rrpc_lun *rlun = rblk->rlun; 	spin_lock(&rlun->lock);	if (rblk->state & NVM_BLK_ST_TGT) {		list_move_tail(&rblk->list, &rlun->free_list);		rlun->nr_free_blocks++;		rblk->state = NVM_BLK_ST_FREE;	} else if (rblk->state & NVM_BLK_ST_BAD) {		list_move_tail(&rblk->list, &rlun->bb_list);		rblk->state = NVM_BLK_ST_BAD;	} else {		WARN_ON_ONCE(1);		pr_err("rrpc: erroneous type (ch:%d,lun:%d,blk%d-> %u)\n",					rlun->bppa.g.ch, rlun->bppa.g.lun,					rblk->id, rblk->state);		list_move_tail(&rblk->list, &rlun->bb_list);	}	spin_unlock(&rlun->lock);} static void rrpc_put_blks(struct rrpc *rrpc){	struct rrpc_lun *rlun;	int i; 	for (i = 0; i < rrpc->nr_luns; i++) {		rlun = &rrpc->luns[i];		if (rlun->cur)			rrpc_put_blk(rrpc, rlun->cur);		if (rlun->gc_cur)			rrpc_put_blk(rrpc, rlun->gc_cur);	}} static struct rrpc_lun *get_next_lun(struct rrpc *rrpc){	int next = atomic_inc_return(&rrpc->next_lun); 	return &rrpc->luns[next % rrpc->nr_luns];} static void rrpc_gc_kick(struct rrpc *rrpc){	struct rrpc_lun *rlun;	unsigned int i; 	for (i = 0; i < rrpc->nr_luns; i++) {		rlun = &rrpc->luns[i];		queue_work(rrpc->krqd_wq, &rlun->ws_gc);	}} /* * timed GC every interval. */static void rrpc_gc_timer(unsigned long data){	struct rrpc *rrpc = (struct rrpc *)data; 	rrpc_gc_kick(rrpc);	mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10));} static void rrpc_end_sync_bio(struct bio *bio){	struct completion *waiting = bio->bi_private; 	if (bio->bi_status)		pr_err("nvm: gc request failed (%u).\n", bio->bi_status); 	complete(waiting);} /* * rrpc_move_valid_pages -- migrate live data off the block * @rrpc: the 'rrpc' structure * @block: the block from which to migrate live pages * * Description: *   GC algorithms may call this function to migrate remaining live *   pages off the block prior to erasing it. This function blocks *   further execution until the operation is complete. */static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk){	struct nvm_tgt_dev *dev = rrpc->dev;	struct request_queue *q = dev->q;	struct rrpc_rev_addr *rev;	struct nvm_rq *rqd;	struct bio *bio;	struct page *page;	int slot;	int nr_sec_per_blk = dev->geo.sec_per_blk;	u64 phys_addr;	DECLARE_COMPLETION_ONSTACK(wait); 	if (bitmap_full(rblk->invalid_pages, nr_sec_per_blk))		return 0; 	bio = bio_alloc(GFP_NOIO, 1);	if (!bio) {		pr_err("nvm: could not alloc bio to gc\n");		return -ENOMEM;	} 	page = mempool_alloc(rrpc->page_pool, GFP_NOIO); 	while ((slot = find_first_zero_bit(rblk->invalid_pages,					    nr_sec_per_blk)) < nr_sec_per_blk) { 		/* Lock laddr */		phys_addr = rrpc_blk_to_ppa(rrpc, rblk) + slot; try:		spin_lock(&rrpc->rev_lock);		/* Get logical address from physical to logical table */		rev = &rrpc->rev_trans_map[phys_addr];		/* already updated by previous regular write */		if (rev->addr == ADDR_EMPTY) {			spin_unlock(&rrpc->rev_lock);			continue;		} 		rqd = rrpc_inflight_laddr_acquire(rrpc, rev->addr, 1);		if (IS_ERR_OR_NULL(rqd)) {			spin_unlock(&rrpc->rev_lock);			schedule();			goto try;		} 		spin_unlock(&rrpc->rev_lock); 		/* Perform read to do GC */		bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);		bio_set_op_attrs(bio,  REQ_OP_READ, 0);		bio->bi_private = &wait;		bio->bi_end_io = rrpc_end_sync_bio; 		/* TODO: may fail when EXP_PG_SIZE > PAGE_SIZE */		bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0); 		if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {			pr_err("rrpc: gc read failed.\n");			rrpc_inflight_laddr_release(rrpc, rqd);			goto finished;		}		wait_for_completion_io(&wait);		if (bio->bi_status) {			rrpc_inflight_laddr_release(rrpc, rqd);			goto finished;		} 		bio_reset(bio);		reinit_completion(&wait); 		bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);		bio->bi_private = &wait;		bio->bi_end_io = rrpc_end_sync_bio; 		bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0); 		/* turn the command around and write the data back to a new		 * address		 */		if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {			pr_err("rrpc: gc write failed.\n");			rrpc_inflight_laddr_release(rrpc, rqd);			goto finished;		}		wait_for_completion_io(&wait); 		rrpc_inflight_laddr_release(rrpc, rqd);		if (bio->bi_status)			goto finished; 		bio_reset(bio);	} finished:	mempool_free(page, rrpc->page_pool);	bio_put(bio); 	if (!bitmap_full(rblk->invalid_pages, nr_sec_per_blk)) {		pr_err("nvm: failed to garbage collect block\n");		return -EIO;	} 	return 0;} static void rrpc_block_gc(struct work_struct *work){	struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,									ws_gc);	struct rrpc *rrpc = gcb->rrpc;	struct rrpc_block *rblk = gcb->rblk;	struct rrpc_lun *rlun = rblk->rlun;	struct ppa_addr ppa; 	mempool_free(gcb, rrpc->gcb_pool);	pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' being reclaimed\n",			rlun->bppa.g.ch, rlun->bppa.g.lun,			rblk->id); 	if (rrpc_move_valid_pages(rrpc, rblk))		goto put_back; 	ppa.ppa = 0;	ppa.g.ch = rlun->bppa.g.ch;	ppa.g.lun = rlun->bppa.g.lun;	ppa.g.blk = rblk->id; 	if (nvm_erase_sync(rrpc->dev, &ppa, 1))		goto put_back; 	rrpc_put_blk(rrpc, rblk); 	return; put_back:	spin_lock(&rlun->lock);	list_add_tail(&rblk->prio, &rlun->prio_list);	spin_unlock(&rlun->lock);} /* the block with highest number of invalid pages, will be in the beginning * of the list */static struct rrpc_block *rblk_max_invalid(struct rrpc_block *ra,							struct rrpc_block *rb){	if (ra->nr_invalid_pages == rb->nr_invalid_pages)		return ra; 	return (ra->nr_invalid_pages < rb->nr_invalid_pages) ? rb : ra;} /* linearly find the block with highest number of invalid pages * requires lun->lock */static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun){	struct list_head *prio_list = &rlun->prio_list;	struct rrpc_block *rblk, *max; 	BUG_ON(list_empty(prio_list)); 	max = list_first_entry(prio_list, struct rrpc_block, prio);	list_for_each_entry(rblk, prio_list, prio)		max = rblk_max_invalid(max, rblk); 	return max;} static void rrpc_lun_gc(struct work_struct *work){	struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc);	struct rrpc *rrpc = rlun->rrpc;	struct nvm_tgt_dev *dev = rrpc->dev;	struct rrpc_block_gc *gcb;	unsigned int nr_blocks_need; 	nr_blocks_need = dev->geo.blks_per_lun / GC_LIMIT_INVERSE; 	if (nr_blocks_need < rrpc->nr_luns)		nr_blocks_need = rrpc->nr_luns; 	spin_lock(&rlun->lock);	while (nr_blocks_need > rlun->nr_free_blocks &&					!list_empty(&rlun->prio_list)) {		struct rrpc_block *rblk = block_prio_find_max(rlun); 		if (!rblk->nr_invalid_pages)			break; 		gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);		if (!gcb)			break; 		list_del_init(&rblk->prio); 		WARN_ON(!block_is_full(rrpc, rblk)); 		pr_debug("rrpc: selected block 'ch:%d,lun:%d,blk:%d' for GC\n",					rlun->bppa.g.ch, rlun->bppa.g.lun,

Lines 1–500 of 1625. Load more