History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma fixes from Jason Gunthorpe: - Several hfi1 patches fixing some long standing driver bugs - Overflow when working with sg lists with elements greater than 4G - An rxe regression with object numbering after the mrs reach their limit - A theoretical problem with the scatterlist merging code * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: lib/scatterlist: Fix to calculate the last_pg properly IB/hfi1: Remove user expected buffer invalidate race IB/hfi1: Immediately remove invalid memory from hardware IB/hfi1: Fix expected receive setup error exit issues IB/hfi1: Reserve user expected TIDs IB/hfi1: Reject a zero-length user expected buffer RDMA/core: Fix ib block iterator counter overflow RDMA/rxe: Prevent faulty rkey generation RDMA/rxe: Fix inaccurate constants in rxe_type_info

Linus Torvalds committed 3 years ago

Commit 8974efa2 parents edc0035 0f097f06 files · +179 −88Browse files

Unified · Split

6 files
  1. drivers/infiniband/core/verbs.c +5 −2
  2. drivers/infiniband/hw/hfi1/user_exp_rcv.c +140 −60
  3. drivers/infiniband/hw/hfi1/user_exp_rcv.h +3 −0
  4. drivers/infiniband/sw/rxe/rxe_param.h +5 −5
  5. drivers/infiniband/sw/rxe/rxe_pool.c +11 −11
  6. lib/scatterlist.c +15 −10
drivers/infiniband/core/verbs.c+5−2
@@ -2957,15 +2957,18 @@ EXPORT_SYMBOL(__rdma_block_iter_start); · 2956 unmodified lines
2957bool __rdma_block_iter_next(struct ib_block_iter *biter)2957bool __rdma_block_iter_next(struct ib_block_iter *biter)
2958{2958{
2959 unsigned int block_offset;2959 unsigned int block_offset;
2960 unsigned int sg_delta;
2960 2961
2961 if (!biter->__sg_nents || !biter->__sg)2962 if (!biter->__sg_nents || !biter->__sg)
2962 return false;2963 return false;
2963 2964
2964 biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;2965 biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
2965 block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);2966 block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
2966 biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;2967 sg_delta = BIT_ULL(biter->__pg_bit) - block_offset;
2967 2968
2968 if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {2969 if (sg_dma_len(biter->__sg) - biter->__sg_advance > sg_delta) {
2970 biter->__sg_advance += sg_delta;
2971 } else {
2969 biter->__sg_advance = 0;2972 biter->__sg_advance = 0;
2970 biter->__sg = sg_next(biter->__sg);2973 biter->__sg = sg_next(biter->__sg);
2971 biter->__sg_nents--;2974 biter->__sg_nents--;
drivers/infiniband/hw/hfi1/user_exp_rcv.c+140−60
@@ -23,18 +23,25 @@ static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata, · 22 unmodified lines
23static bool tid_rb_invalidate(struct mmu_interval_notifier *mni,23static bool tid_rb_invalidate(struct mmu_interval_notifier *mni,
24 const struct mmu_notifier_range *range,24 const struct mmu_notifier_range *range,
25 unsigned long cur_seq);25 unsigned long cur_seq);
26static bool tid_cover_invalidate(struct mmu_interval_notifier *mni,
27 const struct mmu_notifier_range *range,
28 unsigned long cur_seq);
26static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,29static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
27 struct tid_group *grp,30 struct tid_group *grp,
28 unsigned int start, u16 count,31 unsigned int start, u16 count,
29 u32 *tidlist, unsigned int *tididx,32 u32 *tidlist, unsigned int *tididx,
30 unsigned int *pmapped);33 unsigned int *pmapped);
31static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,34static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo);
32 struct tid_group **grp);35static void __clear_tid_node(struct hfi1_filedata *fd,
36 struct tid_rb_node *node);
33static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);37static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);
34 38
35static const struct mmu_interval_notifier_ops tid_mn_ops = {39static const struct mmu_interval_notifier_ops tid_mn_ops = {
36 .invalidate = tid_rb_invalidate,40 .invalidate = tid_rb_invalidate,
37};41};
42static const struct mmu_interval_notifier_ops tid_cover_ops = {
43 .invalidate = tid_cover_invalidate,
44};
38 45
39/*46/*
40 * Initialize context and file private data needed for Expected47 * Initialize context and file private data needed for Expected
@@ -253,53 +260,65 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 212 unmodified lines
253 tididx = 0, mapped, mapped_pages = 0;260 tididx = 0, mapped, mapped_pages = 0;
254 u32 *tidlist = NULL;261 u32 *tidlist = NULL;
255 struct tid_user_buf *tidbuf;262 struct tid_user_buf *tidbuf;
263 unsigned long mmu_seq = 0;
256 264
257 if (!PAGE_ALIGNED(tinfo->vaddr))265 if (!PAGE_ALIGNED(tinfo->vaddr))
258 return -EINVAL;266 return -EINVAL;
267 if (tinfo->length == 0)
268 return -EINVAL;
259 269
260 tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);270 tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);
261 if (!tidbuf)271 if (!tidbuf)
262 return -ENOMEM;272 return -ENOMEM;
263 273
274 mutex_init(&tidbuf->cover_mutex);
264 tidbuf->vaddr = tinfo->vaddr;275 tidbuf->vaddr = tinfo->vaddr;
265 tidbuf->length = tinfo->length;276 tidbuf->length = tinfo->length;
266 tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),277 tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),
267 GFP_KERNEL);278 GFP_KERNEL);
268 if (!tidbuf->psets) {279 if (!tidbuf->psets) {
269 kfree(tidbuf);280 ret = -ENOMEM;
270 return -ENOMEM;281 goto fail_release_mem;
282 }
283
284 if (fd->use_mn) {
285 ret = mmu_interval_notifier_insert(
286 &tidbuf->notifier, current->mm,
287 tidbuf->vaddr, tidbuf->npages * PAGE_SIZE,
288 &tid_cover_ops);
289 if (ret)
290 goto fail_release_mem;
291 mmu_seq = mmu_interval_read_begin(&tidbuf->notifier);
271 }292 }
272 293
273 pinned = pin_rcv_pages(fd, tidbuf);294 pinned = pin_rcv_pages(fd, tidbuf);
274 if (pinned <= 0) {295 if (pinned <= 0) {
275 kfree(tidbuf->psets);296 ret = (pinned < 0) ? pinned : -ENOSPC;
276 kfree(tidbuf);297 goto fail_unpin;
277 return pinned;
278 }298 }
279 299
280 /* Find sets of physically contiguous pages */300 /* Find sets of physically contiguous pages */
281 tidbuf->n_psets = find_phys_blocks(tidbuf, pinned);301 tidbuf->n_psets = find_phys_blocks(tidbuf, pinned);
282 302
283 /*303 /* Reserve the number of expected tids to be used. */
284 * We don't need to access this under a lock since tid_used is per
285 * process and the same process cannot be in hfi1_user_exp_rcv_clear()
286 * and hfi1_user_exp_rcv_setup() at the same time.
287 */
288 spin_lock(&fd->tid_lock);304 spin_lock(&fd->tid_lock);
289 if (fd->tid_used + tidbuf->n_psets > fd->tid_limit)305 if (fd->tid_used + tidbuf->n_psets > fd->tid_limit)
290 pageset_count = fd->tid_limit - fd->tid_used;306 pageset_count = fd->tid_limit - fd->tid_used;
291 else307 else
292 pageset_count = tidbuf->n_psets;308 pageset_count = tidbuf->n_psets;
309 fd->tid_used += pageset_count;
293 spin_unlock(&fd->tid_lock);310 spin_unlock(&fd->tid_lock);
294 311
295 if (!pageset_count)312 if (!pageset_count) {
296 goto bail;313 ret = -ENOSPC;
314 goto fail_unreserve;
315 }
297 316
298 ngroups = pageset_count / dd->rcv_entries.group_size;317 ngroups = pageset_count / dd->rcv_entries.group_size;
299 tidlist = kcalloc(pageset_count, sizeof(*tidlist), GFP_KERNEL);318 tidlist = kcalloc(pageset_count, sizeof(*tidlist), GFP_KERNEL);
300 if (!tidlist) {319 if (!tidlist) {
301 ret = -ENOMEM;320 ret = -ENOMEM;
302 goto nomem;321 goto fail_unreserve;
303 }322 }
304 323
305 tididx = 0;324 tididx = 0;
@@ -395,43 +414,78 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 89 unmodified lines
395 }414 }
396unlock:415unlock:
397 mutex_unlock(&uctxt->exp_mutex);416 mutex_unlock(&uctxt->exp_mutex);
398nomem:
399 hfi1_cdbg(TID, "total mapped: tidpairs:%u pages:%u (%d)", tididx,417 hfi1_cdbg(TID, "total mapped: tidpairs:%u pages:%u (%d)", tididx,
400 mapped_pages, ret);418 mapped_pages, ret);
401 if (tididx) {419
402 spin_lock(&fd->tid_lock);420 /* fail if nothing was programmed, set error if none provided */
403 fd->tid_used += tididx;421 if (tididx == 0) {
404 spin_unlock(&fd->tid_lock);422 if (ret >= 0)
405 tinfo->tidcnt = tididx;423 ret = -ENOSPC;
406 tinfo->length = mapped_pages * PAGE_SIZE;424 goto fail_unreserve;
407 425 }
408 if (copy_to_user(u64_to_user_ptr(tinfo->tidlist),426
409 tidlist, sizeof(tidlist[0]) * tididx)) {427 /* adjust reserved tid_used to actual count */
410 /*428 spin_lock(&fd->tid_lock);
411 * On failure to copy to the user level, we need to undo429 fd->tid_used -= pageset_count - tididx;
412 * everything done so far so we don't leak resources.430 spin_unlock(&fd->tid_lock);
413 */431
414 tinfo->tidlist = (unsigned long)&tidlist;432 /* unpin all pages not covered by a TID */
415 hfi1_user_exp_rcv_clear(fd, tinfo);433 unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages, pinned - mapped_pages,
416 tinfo->tidlist = 0;434 false);
417 ret = -EFAULT;435
418 goto bail;436 if (fd->use_mn) {
437 /* check for an invalidate during setup */
438 bool fail = false;
439
440 mutex_lock(&tidbuf->cover_mutex);
441 fail = mmu_interval_read_retry(&tidbuf->notifier, mmu_seq);
442 mutex_unlock(&tidbuf->cover_mutex);
443
444 if (fail) {
445 ret = -EBUSY;
446 goto fail_unprogram;
419 }447 }
420 }448 }
421 449
422 /*450 tinfo->tidcnt = tididx;
423 * If not everything was mapped (due to insufficient RcvArray entries,451 tinfo->length = mapped_pages * PAGE_SIZE;
424 * for example), unpin all unmapped pages so we can pin them nex time.452
425 */453 if (copy_to_user(u64_to_user_ptr(tinfo->tidlist),
426 if (mapped_pages != pinned)454 tidlist, sizeof(tidlist[0]) * tididx)) {
427 unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages,455 ret = -EFAULT;
428 (pinned - mapped_pages), false);456 goto fail_unprogram;
429bail:457 }
458
459 if (fd->use_mn)
460 mmu_interval_notifier_remove(&tidbuf->notifier);
461 kfree(tidbuf->pages);
430 kfree(tidbuf->psets);462 kfree(tidbuf->psets);
463 kfree(tidbuf);
431 kfree(tidlist);464 kfree(tidlist);
465 return 0;
466
467fail_unprogram:
468 /* unprogram, unmap, and unpin all allocated TIDs */
469 tinfo->tidlist = (unsigned long)tidlist;
470 hfi1_user_exp_rcv_clear(fd, tinfo);
471 tinfo->tidlist = 0;
472 pinned = 0; /* nothing left to unpin */
473 pageset_count = 0; /* nothing left reserved */
474fail_unreserve:
475 spin_lock(&fd->tid_lock);
476 fd->tid_used -= pageset_count;
477 spin_unlock(&fd->tid_lock);
478fail_unpin:
479 if (fd->use_mn)
480 mmu_interval_notifier_remove(&tidbuf->notifier);
481 if (pinned > 0)
482 unpin_rcv_pages(fd, tidbuf, NULL, 0, pinned, false);
483fail_release_mem:
432 kfree(tidbuf->pages);484 kfree(tidbuf->pages);
485 kfree(tidbuf->psets);
433 kfree(tidbuf);486 kfree(tidbuf);
434 return ret > 0 ? 0 : ret;487 kfree(tidlist);
488 return ret;
435}489}
436 490
437int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,491int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,
@@ -452,7 +506,7 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd, · 14 unmodified lines
452 506
453 mutex_lock(&uctxt->exp_mutex);507 mutex_lock(&uctxt->exp_mutex);
454 for (tididx = 0; tididx < tinfo->tidcnt; tididx++) {508 for (tididx = 0; tididx < tinfo->tidcnt; tididx++) {
455 ret = unprogram_rcvarray(fd, tidinfo[tididx], NULL);509 ret = unprogram_rcvarray(fd, tidinfo[tididx]);
456 if (ret) {510 if (ret) {
457 hfi1_cdbg(TID, "Failed to unprogram rcv array %d",511 hfi1_cdbg(TID, "Failed to unprogram rcv array %d",
458 ret);512 ret);
@@ -706,6 +760,7 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd, · 247 unmodified lines
706 }760 }
707 761
708 node->fdata = fd;762 node->fdata = fd;
763 mutex_init(&node->invalidate_mutex);
709 node->phys = page_to_phys(pages[0]);764 node->phys = page_to_phys(pages[0]);
710 node->npages = npages;765 node->npages = npages;
711 node->rcventry = rcventry;766 node->rcventry = rcventry;
@@ -721,11 +776,6 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd, · 9 unmodified lines
721 &tid_mn_ops);776 &tid_mn_ops);
722 if (ret)777 if (ret)
723 goto out_unmap;778 goto out_unmap;
724 /*
725 * FIXME: This is in the wrong order, the notifier should be
726 * established before the pages are pinned by pin_rcv_pages.
727 */
728 mmu_interval_read_begin(&node->notifier);
729 }779 }
730 fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node;780 fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node;
731 781
@@ -745,8 +795,7 @@ out_unmap: · 13 unmodified lines
745 return -EFAULT;795 return -EFAULT;
746}796}
747 797
748static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,798static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo)
749 struct tid_group **grp)
750{799{
751 struct hfi1_ctxtdata *uctxt = fd->uctxt;800 struct hfi1_ctxtdata *uctxt = fd->uctxt;
752 struct hfi1_devdata *dd = uctxt->dd;801 struct hfi1_devdata *dd = uctxt->dd;
@@ -769,9 +818,6 @@ static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo, · 16 unmodified lines
769 if (!node || node->rcventry != (uctxt->expected_base + rcventry))818 if (!node || node->rcventry != (uctxt->expected_base + rcventry))
770 return -EBADF;819 return -EBADF;
771 820
772 if (grp)
773 *grp = node->grp;
774
775 if (fd->use_mn)821 if (fd->use_mn)
776 mmu_interval_notifier_remove(&node->notifier);822 mmu_interval_notifier_remove(&node->notifier);
777 cacheless_tid_rb_remove(fd, node);823 cacheless_tid_rb_remove(fd, node);
@@ -779,23 +825,34 @@ static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo, · 1 unmodified line
779 return 0;825 return 0;
780}826}
781 827
782static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)828static void __clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
783{829{
784 struct hfi1_ctxtdata *uctxt = fd->uctxt;830 struct hfi1_ctxtdata *uctxt = fd->uctxt;
785 struct hfi1_devdata *dd = uctxt->dd;831 struct hfi1_devdata *dd = uctxt->dd;
786 832
833 mutex_lock(&node->invalidate_mutex);
834 if (node->freed)
835 goto done;
836 node->freed = true;
837
787 trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,838 trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,
788 node->npages,839 node->npages,
789 node->notifier.interval_tree.start, node->phys,840 node->notifier.interval_tree.start, node->phys,
790 node->dma_addr);841 node->dma_addr);
791 842
792 /*843 /* Make sure device has seen the write before pages are unpinned */
793 * Make sure device has seen the write before we unpin the
794 * pages.
795 */
796 hfi1_put_tid(dd, node->rcventry, PT_INVALID_FLUSH, 0, 0);844 hfi1_put_tid(dd, node->rcventry, PT_INVALID_FLUSH, 0, 0);
797 845
798 unpin_rcv_pages(fd, NULL, node, 0, node->npages, true);846 unpin_rcv_pages(fd, NULL, node, 0, node->npages, true);
847done:
848 mutex_unlock(&node->invalidate_mutex);
849}
850
851static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
852{
853 struct hfi1_ctxtdata *uctxt = fd->uctxt;
854
855 __clear_tid_node(fd, node);
799 856
800 node->grp->used--;857 node->grp->used--;
801 node->grp->map &= ~(1 << (node->rcventry - node->grp->base));858 node->grp->map &= ~(1 << (node->rcventry - node->grp->base));
@@ -854,10 +911,16 @@ static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, · 52 unmodified lines
854 if (node->freed)911 if (node->freed)
855 return true;912 return true;
856 913
914 /* take action only if unmapping */
915 if (range->event != MMU_NOTIFY_UNMAP)
916 return true;
917
857 trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt,918 trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt,
858 node->notifier.interval_tree.start,919 node->notifier.interval_tree.start,
859 node->rcventry, node->npages, node->dma_addr);920 node->rcventry, node->npages, node->dma_addr);
860 node->freed = true;921
922 /* clear the hardware rcvarray entry */
923 __clear_tid_node(fdata, node);
861 924
862 spin_lock(&fdata->invalid_lock);925 spin_lock(&fdata->invalid_lock);
863 if (fdata->invalid_tid_idx < uctxt->expected_count) {926 if (fdata->invalid_tid_idx < uctxt->expected_count) {
@@ -887,6 +950,23 @@ static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, · 23 unmodified lines
887 return true;950 return true;
888}951}
889 952
953static bool tid_cover_invalidate(struct mmu_interval_notifier *mni,
954 const struct mmu_notifier_range *range,
955 unsigned long cur_seq)
956{
957 struct tid_user_buf *tidbuf =
958 container_of(mni, struct tid_user_buf, notifier);
959
960 /* take action only if unmapping */
961 if (range->event == MMU_NOTIFY_UNMAP) {
962 mutex_lock(&tidbuf->cover_mutex);
963 mmu_interval_set_seq(mni, cur_seq);
964 mutex_unlock(&tidbuf->cover_mutex);
965 }
966
967 return true;
968}
969
890static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,970static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
891 struct tid_rb_node *tnode)971 struct tid_rb_node *tnode)
892{972{
drivers/infiniband/hw/hfi1/user_exp_rcv.h+3−0
@@ -16,6 +16,8 @@ struct tid_pageset { · 15 unmodified lines
16};16};
17 17
18struct tid_user_buf {18struct tid_user_buf {
19 struct mmu_interval_notifier notifier;
20 struct mutex cover_mutex;
19 unsigned long vaddr;21 unsigned long vaddr;
20 unsigned long length;22 unsigned long length;
21 unsigned int npages;23 unsigned int npages;
@@ -27,6 +29,7 @@ struct tid_user_buf { · 5 unmodified lines
27struct tid_rb_node {29struct tid_rb_node {
28 struct mmu_interval_notifier notifier;30 struct mmu_interval_notifier notifier;
29 struct hfi1_filedata *fdata;31 struct hfi1_filedata *fdata;
32 struct mutex invalidate_mutex; /* covers hw removal */
30 unsigned long phys;33 unsigned long phys;
31 struct tid_group *grp;34 struct tid_group *grp;
32 u32 rcventry;35 u32 rcventry;
drivers/infiniband/sw/rxe/rxe_param.h+5−5
@@ -98,11 +98,11 @@ enum rxe_device_param { · 97 unmodified lines
98 RXE_MAX_SRQ = DEFAULT_MAX_VALUE - RXE_MIN_SRQ_INDEX,98 RXE_MAX_SRQ = DEFAULT_MAX_VALUE - RXE_MIN_SRQ_INDEX,
99 99
100 RXE_MIN_MR_INDEX = 0x00000001,100 RXE_MIN_MR_INDEX = 0x00000001,
101 RXE_MAX_MR_INDEX = DEFAULT_MAX_VALUE,101 RXE_MAX_MR_INDEX = DEFAULT_MAX_VALUE >> 1,
102 RXE_MAX_MR = DEFAULT_MAX_VALUE - RXE_MIN_MR_INDEX,102 RXE_MAX_MR = RXE_MAX_MR_INDEX - RXE_MIN_MR_INDEX,
103 RXE_MIN_MW_INDEX = 0x00010001,103 RXE_MIN_MW_INDEX = RXE_MAX_MR_INDEX + 1,
104 RXE_MAX_MW_INDEX = 0x00020000,104 RXE_MAX_MW_INDEX = DEFAULT_MAX_VALUE,
105 RXE_MAX_MW = 0x00001000,105 RXE_MAX_MW = RXE_MAX_MW_INDEX - RXE_MIN_MW_INDEX,
106 106
107 RXE_MAX_PKT_PER_ACK = 64,107 RXE_MAX_PKT_PER_ACK = 64,
108 108
drivers/infiniband/sw/rxe/rxe_pool.c+11−11
@@ -23,16 +23,16 @@ static const struct rxe_type_info { · 22 unmodified lines
23 .size = sizeof(struct rxe_ucontext),23 .size = sizeof(struct rxe_ucontext),
24 .elem_offset = offsetof(struct rxe_ucontext, elem),24 .elem_offset = offsetof(struct rxe_ucontext, elem),
25 .min_index = 1,25 .min_index = 1,
26 .max_index = UINT_MAX,26 .max_index = RXE_MAX_UCONTEXT,
27 .max_elem = UINT_MAX,27 .max_elem = RXE_MAX_UCONTEXT,
28 },28 },
29 [RXE_TYPE_PD] = {29 [RXE_TYPE_PD] = {
30 .name = "pd",30 .name = "pd",
31 .size = sizeof(struct rxe_pd),31 .size = sizeof(struct rxe_pd),
32 .elem_offset = offsetof(struct rxe_pd, elem),32 .elem_offset = offsetof(struct rxe_pd, elem),
33 .min_index = 1,33 .min_index = 1,
34 .max_index = UINT_MAX,34 .max_index = RXE_MAX_PD,
35 .max_elem = UINT_MAX,35 .max_elem = RXE_MAX_PD,
36 },36 },
37 [RXE_TYPE_AH] = {37 [RXE_TYPE_AH] = {
38 .name = "ah",38 .name = "ah",
@@ -40,7 +40,7 @@ static const struct rxe_type_info { · 1 unmodified line
40 .elem_offset = offsetof(struct rxe_ah, elem),40 .elem_offset = offsetof(struct rxe_ah, elem),
41 .min_index = RXE_MIN_AH_INDEX,41 .min_index = RXE_MIN_AH_INDEX,
42 .max_index = RXE_MAX_AH_INDEX,42 .max_index = RXE_MAX_AH_INDEX,
43 .max_elem = RXE_MAX_AH_INDEX - RXE_MIN_AH_INDEX + 1,43 .max_elem = RXE_MAX_AH,
44 },44 },
45 [RXE_TYPE_SRQ] = {45 [RXE_TYPE_SRQ] = {
46 .name = "srq",46 .name = "srq",
@@ -49,7 +49,7 @@ static const struct rxe_type_info { · 2 unmodified lines
49 .cleanup = rxe_srq_cleanup,49 .cleanup = rxe_srq_cleanup,
50 .min_index = RXE_MIN_SRQ_INDEX,50 .min_index = RXE_MIN_SRQ_INDEX,
51 .max_index = RXE_MAX_SRQ_INDEX,51 .max_index = RXE_MAX_SRQ_INDEX,
52 .max_elem = RXE_MAX_SRQ_INDEX - RXE_MIN_SRQ_INDEX + 1,52 .max_elem = RXE_MAX_SRQ,
53 },53 },
54 [RXE_TYPE_QP] = {54 [RXE_TYPE_QP] = {
55 .name = "qp",55 .name = "qp",
@@ -58,7 +58,7 @@ static const struct rxe_type_info { · 2 unmodified lines
58 .cleanup = rxe_qp_cleanup,58 .cleanup = rxe_qp_cleanup,
59 .min_index = RXE_MIN_QP_INDEX,59 .min_index = RXE_MIN_QP_INDEX,
60 .max_index = RXE_MAX_QP_INDEX,60 .max_index = RXE_MAX_QP_INDEX,
61 .max_elem = RXE_MAX_QP_INDEX - RXE_MIN_QP_INDEX + 1,61 .max_elem = RXE_MAX_QP,
62 },62 },
63 [RXE_TYPE_CQ] = {63 [RXE_TYPE_CQ] = {
64 .name = "cq",64 .name = "cq",
@@ -66,8 +66,8 @@ static const struct rxe_type_info { · 1 unmodified line
66 .elem_offset = offsetof(struct rxe_cq, elem),66 .elem_offset = offsetof(struct rxe_cq, elem),
67 .cleanup = rxe_cq_cleanup,67 .cleanup = rxe_cq_cleanup,
68 .min_index = 1,68 .min_index = 1,
69 .max_index = UINT_MAX,69 .max_index = RXE_MAX_CQ,
70 .max_elem = UINT_MAX,70 .max_elem = RXE_MAX_CQ,
71 },71 },
72 [RXE_TYPE_MR] = {72 [RXE_TYPE_MR] = {
73 .name = "mr",73 .name = "mr",
@@ -76,7 +76,7 @@ static const struct rxe_type_info { · 2 unmodified lines
76 .cleanup = rxe_mr_cleanup,76 .cleanup = rxe_mr_cleanup,
77 .min_index = RXE_MIN_MR_INDEX,77 .min_index = RXE_MIN_MR_INDEX,
78 .max_index = RXE_MAX_MR_INDEX,78 .max_index = RXE_MAX_MR_INDEX,
79 .max_elem = RXE_MAX_MR_INDEX - RXE_MIN_MR_INDEX + 1,79 .max_elem = RXE_MAX_MR,
80 },80 },
81 [RXE_TYPE_MW] = {81 [RXE_TYPE_MW] = {
82 .name = "mw",82 .name = "mw",
@@ -85,7 +85,7 @@ static const struct rxe_type_info { · 2 unmodified lines
85 .cleanup = rxe_mw_cleanup,85 .cleanup = rxe_mw_cleanup,
86 .min_index = RXE_MIN_MW_INDEX,86 .min_index = RXE_MIN_MW_INDEX,
87 .max_index = RXE_MAX_MW_INDEX,87 .max_index = RXE_MAX_MW_INDEX,
88 .max_elem = RXE_MAX_MW_INDEX - RXE_MIN_MW_INDEX + 1,88 .max_elem = RXE_MAX_MW,
89 },89 },
90};90};
91 91
lib/scatterlist.c+15−10
@@ -470,22 +470,27 @@ int sg_alloc_append_table_from_pages(struct sg_append_table *sgt_append, · 469 unmodified lines
470 return -EOPNOTSUPP;470 return -EOPNOTSUPP;
471 471
472 if (sgt_append->prv) {472 if (sgt_append->prv) {
473 unsigned long next_pfn = (page_to_phys(sg_page(sgt_append->prv)) +
474 sgt_append->prv->offset + sgt_append->prv->length) / PAGE_SIZE;
475
473 if (WARN_ON(offset))476 if (WARN_ON(offset))
474 return -EINVAL;477 return -EINVAL;
475 478
476 /* Merge contiguous pages into the last SG */479 /* Merge contiguous pages into the last SG */
477 prv_len = sgt_append->prv->length;480 prv_len = sgt_append->prv->length;
478 last_pg = sg_page(sgt_append->prv);481 if (page_to_pfn(pages[0]) == next_pfn) {
479 while (n_pages && pages_are_mergeable(pages[0], last_pg)) {482 last_pg = pfn_to_page(next_pfn - 1);
480 if (sgt_append->prv->length + PAGE_SIZE > max_segment)483 while (n_pages && pages_are_mergeable(pages[0], last_pg)) {
481 break;484 if (sgt_append->prv->length + PAGE_SIZE > max_segment)
482 sgt_append->prv->length += PAGE_SIZE;485 break;
483 last_pg = pages[0];486 sgt_append->prv->length += PAGE_SIZE;
484 pages++;487 last_pg = pages[0];
485 n_pages--;488 pages++;
489 n_pages--;
490 }
491 if (!n_pages)
492 goto out;
486 }493 }
487 if (!n_pages)
488 goto out;
489 }494 }
490 495
491 /* compute number of contiguous chunks */496 /* compute number of contiguous chunks */