A new user flag is required to make memory map hint mandatory, in contrast to the current situation where it is best effort. This is due to the requirement to map certain data to specific pre-determined device virtual address ranges.
Unified · Split
| @@ -564,6 +564,7 @@ static inline bool is_hint_crossing_range(enum hl_va_range_type range_type, · 563 unmodified lines | ||
| 564 | 564 | * @hint_addr: hint for requested address by the user. |
| 565 | 565 | * @va_block_align: required alignment of the virtual block start address. |
| 566 | 566 | * @range_type: va range type (host, dram) |
| 567 | + * @flags: additional memory flags, currently only uses HL_MEM_FORCE_HINT | |
| 567 | 568 | * |
| 568 | 569 | * This function does the following: |
| 569 | 570 | * - Iterate on the virtual block list to find a suitable virtual block for the |
| @@ -574,7 +575,8 @@ static inline bool is_hint_crossing_range(enum hl_va_range_type range_type, · 4 unmodified lines | ||
| 574 | 575 | static u64 get_va_block(struct hl_device *hdev, |
| 575 | 576 | struct hl_va_range *va_range, |
| 576 | 577 | u64 size, u64 hint_addr, u32 va_block_align, |
| 577 | − enum hl_va_range_type range_type) | |
| 578 | + enum hl_va_range_type range_type, | |
| 579 | + u32 flags) | |
| 578 | 580 | { |
| 579 | 581 | struct hl_vm_va_block *va_block, *new_va_block = NULL; |
| 580 | 582 | struct asic_fixed_properties *prop = &hdev->asic_prop; |
| @@ -584,6 +586,7 @@ static u64 get_va_block(struct hl_device *hdev, · 3 unmodified lines | ||
| 584 | 586 | bool add_prev = false; |
| 585 | 587 | bool is_align_pow_2 = is_power_of_2(va_range->page_size); |
| 586 | 588 | bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr); |
| 589 | + bool force_hint = flags & HL_MEM_FORCE_HINT; | |
| 587 | 590 | |
| 588 | 591 | if (is_align_pow_2) |
| 589 | 592 | align_mask = ~((u64)va_block_align - 1); |
| @@ -603,6 +606,15 @@ static u64 get_va_block(struct hl_device *hdev, · 13 unmodified lines | ||
| 603 | 606 | (!is_align_pow_2 && is_hint_dram_addr && |
| 604 | 607 | do_div(tmp_hint_addr, va_range->page_size))) { |
| 605 | 608 | |
| 609 | + if (force_hint) { | |
| 610 | + /* Hint must be repected, so here we just fail. | |
| 611 | + */ | |
| 612 | + dev_err(hdev->dev, | |
| 613 | + "Hint address 0x%llx is not page aligned - cannot be respected\n", | |
| 614 | + hint_addr); | |
| 615 | + return 0; | |
| 616 | + } | |
| 617 | + | |
| 606 | 618 | dev_dbg(hdev->dev, |
| 607 | 619 | "Hint address 0x%llx will be ignored because it is not aligned\n", |
| 608 | 620 | hint_addr); |
| @@ -660,6 +672,17 @@ static u64 get_va_block(struct hl_device *hdev, · 51 unmodified lines | ||
| 660 | 672 | goto out; |
| 661 | 673 | } |
| 662 | 674 | |
| 675 | + if (force_hint && reserved_valid_start != hint_addr) { | |
| 676 | + /* Hint address must be respected. If we are here - this means | |
| 677 | + * we could not respect it. | |
| 678 | + */ | |
| 679 | + dev_err(hdev->dev, | |
| 680 | + "Hint address 0x%llx could not be respected\n", | |
| 681 | + hint_addr); | |
| 682 | + reserved_valid_start = 0; | |
| 683 | + goto out; | |
| 684 | + } | |
| 685 | + | |
| 663 | 686 | /* |
| 664 | 687 | * Check if there is some leftover range due to reserving the new |
| 665 | 688 | * va block, then return it to the main virtual addresses list. |
| @@ -712,7 +735,8 @@ u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, · 46 unmodified lines | ||
| 712 | 735 | enum hl_va_range_type type, u32 size, u32 alignment) |
| 713 | 736 | { |
| 714 | 737 | return get_va_block(hdev, ctx->va_range[type], size, 0, |
| 715 | − max(alignment, ctx->va_range[type]->page_size), type); | |
| 738 | + max(alignment, ctx->va_range[type]->page_size), | |
| 739 | + type, 0); | |
| 716 | 740 | } |
| 717 | 741 | |
| 718 | 742 | /** |
| @@ -1145,9 +1169,24 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 426 unmodified lines | ||
| 1145 | 1169 | goto hnode_err; |
| 1146 | 1170 | } |
| 1147 | 1171 | |
| 1172 | + if (hint_addr && phys_pg_pack->offset) { | |
| 1173 | + if (args->flags & HL_MEM_FORCE_HINT) { | |
| 1174 | + /* If hint must be repected, since we can't - just fail. | |
| 1175 | + */ | |
| 1176 | + dev_err(hdev->dev, | |
| 1177 | + "Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n", | |
| 1178 | + hint_addr, phys_pg_pack->offset); | |
| 1179 | + rc = -EINVAL; | |
| 1180 | + goto va_block_err; | |
| 1181 | + } | |
| 1182 | + dev_dbg(hdev->dev, | |
| 1183 | + "Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n", | |
| 1184 | + hint_addr, phys_pg_pack->offset); | |
| 1185 | + } | |
| 1186 | + | |
| 1148 | 1187 | ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size, |
| 1149 | 1188 | hint_addr, va_block_align, |
| 1150 | − va_range_type); | |
| 1189 | + va_range_type, args->flags); | |
| 1151 | 1190 | if (!ret_vaddr) { |
| 1152 | 1191 | dev_err(hdev->dev, "no available va block for handle %u\n", |
| 1153 | 1192 | handle); |
| @@ -813,6 +813,7 @@ union hl_wait_cs_args { · 812 unmodified lines | ||
| 813 | 813 | #define HL_MEM_CONTIGUOUS 0x1 |
| 814 | 814 | #define HL_MEM_SHARED 0x2 |
| 815 | 815 | #define HL_MEM_USERPTR 0x4 |
| 816 | +#define HL_MEM_FORCE_HINT 0x8 | |
| 816 | 817 | |
| 817 | 818 | struct hl_mem_in { |
| 818 | 819 | union { |