DMA-BUF requires that each device that accesses a DMA-BUF attaches to it separately. To do so the host1x_bo_pin() and host1x_bo_unpin() functions need to be reimplemented so that they can return a mapping, which either represents an attachment or a map of the driver's own GEM object.
Unified · Split
| @@ -23,86 +23,96 @@ · 22 unmodified lines | ||
| 23 | 23 | |
| 24 | 24 | MODULE_IMPORT_NS(DMA_BUF); |
| 25 | 25 | |
| 26 | −static void tegra_bo_put(struct host1x_bo *bo) | |
| 26 | +static unsigned int sg_dma_count_chunks(struct scatterlist *sgl, unsigned int nents) | |
| 27 | 27 | { |
| 28 | − struct tegra_bo *obj = host1x_to_tegra_bo(bo); | |
| 28 | + dma_addr_t next = ~(dma_addr_t)0; | |
| 29 | + unsigned int count = 0, i; | |
| 30 | + struct scatterlist *s; | |
| 29 | 31 | |
| 30 | − drm_gem_object_put(&obj->gem); | |
| 31 | −} | |
| 32 | + for_each_sg(sgl, s, nents, i) { | |
| 33 | + /* sg_dma_address(s) is only valid for entries that have sg_dma_len(s) != 0. */ | |
| 34 | + if (!sg_dma_len(s)) | |
| 35 | + continue; | |
| 32 | 36 | |
| 33 | −/* XXX move this into lib/scatterlist.c? */ | |
| 34 | −static int sg_alloc_table_from_sg(struct sg_table *sgt, struct scatterlist *sg, | |
| 35 | − unsigned int nents, gfp_t gfp_mask) | |
| 36 | −{ | |
| 37 | − struct scatterlist *dst; | |
| 38 | − unsigned int i; | |
| 39 | − int err; | |
| 37 | + if (sg_dma_address(s) != next) { | |
| 38 | + next = sg_dma_address(s) + sg_dma_len(s); | |
| 39 | + count++; | |
| 40 | + } | |
| 41 | + } | |
| 40 | 42 | |
| 41 | − err = sg_alloc_table(sgt, nents, gfp_mask); | |
| 42 | − if (err < 0) | |
| 43 | − return err; | |
| 43 | + return count; | |
| 44 | +} | |
| 44 | 45 | |
| 45 | − dst = sgt->sgl; | |
| 46 | +static inline unsigned int sgt_dma_count_chunks(struct sg_table *sgt) | |
| 47 | +{ | |
| 48 | + return sg_dma_count_chunks(sgt->sgl, sgt->nents); | |
| 49 | +} | |
| 46 | 50 | |
| 47 | − for (i = 0; i < nents; i++) { | |
| 48 | − sg_set_page(dst, sg_page(sg), sg->length, 0); | |
| 49 | − dst = sg_next(dst); | |
| 50 | − sg = sg_next(sg); | |
| 51 | − } | |
| 51 | +static void tegra_bo_put(struct host1x_bo *bo) | |
| 52 | +{ | |
| 53 | + struct tegra_bo *obj = host1x_to_tegra_bo(bo); | |
| 52 | 54 | |
| 53 | − return 0; | |
| 55 | + drm_gem_object_put(&obj->gem); | |
| 54 | 56 | } |
| 55 | 57 | |
| 56 | −static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, | |
| 57 | − dma_addr_t *phys) | |
| 58 | +static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, | |
| 59 | + enum dma_data_direction direction) | |
| 58 | 60 | { |
| 59 | 61 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
| 60 | − struct sg_table *sgt; | |
| 62 | + struct drm_gem_object *gem = &obj->gem; | |
| 63 | + struct host1x_bo_mapping *map; | |
| 61 | 64 | int err; |
| 62 | 65 | |
| 66 | + map = kzalloc(sizeof(*map), GFP_KERNEL); | |
| 67 | + if (!map) | |
| 68 | + return ERR_PTR(-ENOMEM); | |
| 69 | + | |
| 70 | + map->bo = host1x_bo_get(bo); | |
| 71 | + map->direction = direction; | |
| 72 | + map->dev = dev; | |
| 73 | + | |
| 63 | 74 | /* |
| 64 | − * If we've manually mapped the buffer object through the IOMMU, make | |
| 65 | − * sure to return the IOVA address of our mapping. | |
| 66 | − * | |
| 67 | − * Similarly, for buffers that have been allocated by the DMA API the | |
| 68 | − * physical address can be used for devices that are not attached to | |
| 69 | − * an IOMMU. For these devices, callers must pass a valid pointer via | |
| 70 | − * the @phys argument. | |
| 71 | − * | |
| 72 | − * Imported buffers were also already mapped at import time, so the | |
| 73 | − * existing mapping can be reused. | |
| 75 | + * Imported buffers need special treatment to satisfy the semantics of DMA-BUF. | |
| 74 | 76 | */ |
| 75 | − if (phys) { | |
| 76 | − *phys = obj->iova; | |
| 77 | − return NULL; | |
| 77 | + if (gem->import_attach) { | |
| 78 | + struct dma_buf *buf = gem->import_attach->dmabuf; | |
| 79 | + | |
| 80 | + map->attach = dma_buf_attach(buf, dev); | |
| 81 | + if (IS_ERR(map->attach)) { | |
| 82 | + err = PTR_ERR(map->attach); | |
| 83 | + goto free; | |
| 84 | + } | |
| 85 | + | |
| 86 | + map->sgt = dma_buf_map_attachment(map->attach, direction); | |
| 87 | + if (IS_ERR(map->sgt)) { | |
| 88 | + dma_buf_detach(buf, map->attach); | |
| 89 | + err = PTR_ERR(map->sgt); | |
| 90 | + goto free; | |
| 91 | + } | |
| 92 | + | |
| 93 | + err = sgt_dma_count_chunks(map->sgt); | |
| 94 | + map->size = gem->size; | |
| 95 | + | |
| 96 | + goto out; | |
| 78 | 97 | } |
| 79 | 98 | |
| 80 | 99 | /* |
| 81 | 100 | * If we don't have a mapping for this buffer yet, return an SG table |
| 82 | 101 | * so that host1x can do the mapping for us via the DMA API. |
| 83 | 102 | */ |
| 84 | − sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); | |
| 85 | − if (!sgt) | |
| 86 | − return ERR_PTR(-ENOMEM); | |
| 103 | + map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); | |
| 104 | + if (!map->sgt) { | |
| 105 | + err = -ENOMEM; | |
| 106 | + goto free; | |
| 107 | + } | |
| 87 | 108 | |
| 88 | 109 | if (obj->pages) { |
| 89 | 110 | /* |
| 90 | 111 | * If the buffer object was allocated from the explicit IOMMU |
| 91 | 112 | * API code paths, construct an SG table from the pages. |
| 92 | 113 | */ |
| 93 | − err = sg_alloc_table_from_pages(sgt, obj->pages, obj->num_pages, | |
| 94 | − 0, obj->gem.size, GFP_KERNEL); | |
| 95 | − if (err < 0) | |
| 96 | − goto free; | |
| 97 | − } else if (obj->sgt) { | |
| 98 | − /* | |
| 99 | − * If the buffer object already has an SG table but no pages | |
| 100 | − * were allocated for it, it means the buffer was imported and | |
| 101 | − * the SG table needs to be copied to avoid overwriting any | |
| 102 | − * other potential users of the original SG table. | |
| 103 | − */ | |
| 104 | − err = sg_alloc_table_from_sg(sgt, obj->sgt->sgl, | |
| 105 | − obj->sgt->orig_nents, GFP_KERNEL); | |
| 114 | + err = sg_alloc_table_from_pages(map->sgt, obj->pages, obj->num_pages, 0, gem->size, | |
| 115 | + GFP_KERNEL); | |
| 106 | 116 | if (err < 0) |
| 107 | 117 | goto free; |
| 108 | 118 | } else { |
| @@ -111,25 +121,56 @@ static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, · 2 unmodified lines | ||
| 111 | 121 | * not imported, it had to be allocated with the DMA API, so |
| 112 | 122 | * the DMA API helper can be used. |
| 113 | 123 | */ |
| 114 | − err = dma_get_sgtable(dev, sgt, obj->vaddr, obj->iova, | |
| 115 | − obj->gem.size); | |
| 124 | + err = dma_get_sgtable(dev, map->sgt, obj->vaddr, obj->iova, gem->size); | |
| 116 | 125 | if (err < 0) |
| 117 | 126 | goto free; |
| 118 | 127 | } |
| 119 | 128 | |
| 120 | − return sgt; | |
| 129 | + err = dma_map_sgtable(dev, map->sgt, direction, 0); | |
| 130 | + if (err) | |
| 131 | + goto free_sgt; | |
| 132 | + | |
| 133 | +out: | |
| 134 | + /* | |
| 135 | + * If we've manually mapped the buffer object through the IOMMU, make sure to return the | |
| 136 | + * existing IOVA address of our mapping. | |
| 137 | + */ | |
| 138 | + if (!obj->mm) { | |
| 139 | + map->phys = sg_dma_address(map->sgt->sgl); | |
| 140 | + map->chunks = err; | |
| 141 | + } else { | |
| 142 | + map->phys = obj->iova; | |
| 143 | + map->chunks = 1; | |
| 144 | + } | |
| 145 | + | |
| 146 | + map->size = gem->size; | |
| 121 | 147 | |
| 148 | + return map; | |
| 149 | + | |
| 150 | +free_sgt: | |
| 151 | + sg_free_table(map->sgt); | |
| 122 | 152 | free: |
| 123 | − kfree(sgt); | |
| 153 | + kfree(map->sgt); | |
| 154 | + kfree(map); | |
| 124 | 155 | return ERR_PTR(err); |
| 125 | 156 | } |
| 126 | 157 | |
| 127 | −static void tegra_bo_unpin(struct device *dev, struct sg_table *sgt) | |
| 158 | +static void tegra_bo_unpin(struct host1x_bo_mapping *map) | |
| 128 | 159 | { |
| 129 | − if (sgt) { | |
| 130 | − sg_free_table(sgt); | |
| 131 | − kfree(sgt); | |
| 160 | + if (!map) | |
| 161 | + return; | |
| 162 | + | |
| 163 | + if (map->attach) { | |
| 164 | + dma_buf_unmap_attachment(map->attach, map->sgt, map->direction); | |
| 165 | + dma_buf_detach(map->attach->dmabuf, map->attach); | |
| 166 | + } else { | |
| 167 | + dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); | |
| 168 | + sg_free_table(map->sgt); | |
| 169 | + kfree(map->sgt); | |
| 132 | 170 | } |
| 171 | + | |
| 172 | + host1x_bo_put(map->bo); | |
| 173 | + kfree(map); | |
| 133 | 174 | } |
| 134 | 175 | |
| 135 | 176 | static void *tegra_bo_mmap(struct host1x_bo *bo) |
| @@ -74,7 +74,7 @@ tegra_plane_atomic_duplicate_state(struct drm_plane *plane) · 73 unmodified lines | ||
| 74 | 74 | |
| 75 | 75 | for (i = 0; i < 3; i++) { |
| 76 | 76 | copy->iova[i] = DMA_MAPPING_ERROR; |
| 77 | − copy->sgt[i] = NULL; | |
| 77 | + copy->map[i] = NULL; | |
| 78 | 78 | } |
| 79 | 79 | |
| 80 | 80 | return ©->base; |
| @@ -138,55 +138,37 @@ const struct drm_plane_funcs tegra_plane_funcs = { · 57 unmodified lines | ||
| 138 | 138 | |
| 139 | 139 | static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) |
| 140 | 140 | { |
| 141 | − struct iommu_domain *domain = iommu_get_domain_for_dev(dc->dev); | |
| 142 | 141 | unsigned int i; |
| 143 | 142 | int err; |
| 144 | 143 | |
| 145 | 144 | for (i = 0; i < state->base.fb->format->num_planes; i++) { |
| 146 | 145 | struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); |
| 147 | − dma_addr_t phys_addr, *phys; | |
| 148 | − struct sg_table *sgt; | |
| 146 | + struct host1x_bo_mapping *map; | |
| 149 | 147 | |
| 150 | − /* | |
| 151 | − * If we're not attached to a domain, we already stored the | |
| 152 | − * physical address when the buffer was allocated. If we're | |
| 153 | − * part of a group that's shared between all display | |
| 154 | − * controllers, we've also already mapped the framebuffer | |
| 155 | − * through the SMMU. In both cases we can short-circuit the | |
| 156 | − * code below and retrieve the stored IOV address. | |
| 157 | − */ | |
| 158 | − if (!domain || dc->client.group) | |
| 159 | − phys = &phys_addr; | |
| 160 | − else | |
| 161 | − phys = NULL; | |
| 162 | − | |
| 163 | − sgt = host1x_bo_pin(dc->dev, &bo->base, phys); | |
| 164 | − if (IS_ERR(sgt)) { | |
| 165 | − err = PTR_ERR(sgt); | |
| 148 | + map = host1x_bo_pin(dc->dev, &bo->base, DMA_TO_DEVICE); | |
| 149 | + if (IS_ERR(map)) { | |
| 150 | + err = PTR_ERR(map); | |
| 166 | 151 | goto unpin; |
| 167 | 152 | } |
| 168 | 153 | |
| 169 | − if (sgt) { | |
| 170 | − err = dma_map_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); | |
| 171 | − if (err) | |
| 172 | − goto unpin; | |
| 173 | − | |
| 154 | + if (!dc->client.group) { | |
| 174 | 155 | /* |
| 175 | 156 | * The display controller needs contiguous memory, so |
| 176 | 157 | * fail if the buffer is discontiguous and we fail to |
| 177 | 158 | * map its SG table to a single contiguous chunk of |
| 178 | 159 | * I/O virtual memory. |
| 179 | 160 | */ |
| 180 | − if (sgt->nents > 1) { | |
| 161 | + if (map->chunks > 1) { | |
| 181 | 162 | err = -EINVAL; |
| 182 | 163 | goto unpin; |
| 183 | 164 | } |
| 184 | 165 | |
| 185 | − state->iova[i] = sg_dma_address(sgt->sgl); | |
| 186 | − state->sgt[i] = sgt; | |
| 166 | + state->iova[i] = map->phys; | |
| 187 | 167 | } else { |
| 188 | − state->iova[i] = phys_addr; | |
| 168 | + state->iova[i] = bo->iova; | |
| 189 | 169 | } |
| 170 | + | |
| 171 | + state->map[i] = map; | |
| 190 | 172 | } |
| 191 | 173 | |
| 192 | 174 | return 0; |
| @@ -195,15 +177,9 @@ unpin: · 2 unmodified lines | ||
| 195 | 177 | dev_err(dc->dev, "failed to map plane %u: %d\n", i, err); |
| 196 | 178 | |
| 197 | 179 | while (i--) { |
| 198 | − struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); | |
| 199 | − struct sg_table *sgt = state->sgt[i]; | |
| 200 | − | |
| 201 | − if (sgt) | |
| 202 | − dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); | |
| 203 | − | |
| 204 | − host1x_bo_unpin(dc->dev, &bo->base, sgt); | |
| 180 | + host1x_bo_unpin(state->map[i]); | |
| 205 | 181 | state->iova[i] = DMA_MAPPING_ERROR; |
| 206 | − state->sgt[i] = NULL; | |
| 182 | + state->map[i] = NULL; | |
| 207 | 183 | } |
| 208 | 184 | |
| 209 | 185 | return err; |
| @@ -214,15 +190,9 @@ static void tegra_dc_unpin(struct tegra_dc *dc, struct tegra_plane_state *state) · 4 unmodified lines | ||
| 214 | 190 | unsigned int i; |
| 215 | 191 | |
| 216 | 192 | for (i = 0; i < state->base.fb->format->num_planes; i++) { |
| 217 | − struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); | |
| 218 | − struct sg_table *sgt = state->sgt[i]; | |
| 219 | − | |
| 220 | − if (sgt) | |
| 221 | − dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); | |
| 222 | − | |
| 223 | − host1x_bo_unpin(dc->dev, &bo->base, sgt); | |
| 193 | + host1x_bo_unpin(state->map[i]); | |
| 224 | 194 | state->iova[i] = DMA_MAPPING_ERROR; |
| 225 | − state->sgt[i] = NULL; | |
| 195 | + state->map[i] = NULL; | |
| 226 | 196 | } |
| 227 | 197 | } |
| 228 | 198 | |
| @@ -43,7 +43,7 @@ struct tegra_plane_legacy_blending_state { · 42 unmodified lines | ||
| 43 | 43 | struct tegra_plane_state { |
| 44 | 44 | struct drm_plane_state base; |
| 45 | 45 | |
| 46 | − struct sg_table *sgt[3]; | |
| 46 | + struct host1x_bo_mapping *map[3]; | |
| 47 | 47 | dma_addr_t iova[3]; |
| 48 | 48 | |
| 49 | 49 | struct tegra_bo_tiling tiling; |
| @@ -64,33 +64,61 @@ static void gather_bo_put(struct host1x_bo *host_bo) · 63 unmodified lines | ||
| 64 | 64 | kref_put(&bo->ref, gather_bo_release); |
| 65 | 65 | } |
| 66 | 66 | |
| 67 | −static struct sg_table * | |
| 68 | −gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys) | |
| 67 | +static struct host1x_bo_mapping * | |
| 68 | +gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) | |
| 69 | 69 | { |
| 70 | − struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); | |
| 71 | − struct sg_table *sgt; | |
| 70 | + struct gather_bo *gather = container_of(bo, struct gather_bo, base); | |
| 71 | + struct host1x_bo_mapping *map; | |
| 72 | 72 | int err; |
| 73 | 73 | |
| 74 | − sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); | |
| 75 | − if (!sgt) | |
| 74 | + map = kzalloc(sizeof(*map), GFP_KERNEL); | |
| 75 | + if (!map) | |
| 76 | 76 | return ERR_PTR(-ENOMEM); |
| 77 | 77 | |
| 78 | − err = dma_get_sgtable(bo->dev, sgt, bo->gather_data, bo->gather_data_dma, | |
| 79 | − bo->gather_data_words * 4); | |
| 80 | − if (err) { | |
| 81 | − kfree(sgt); | |
| 82 | − return ERR_PTR(err); | |
| 78 | + map->bo = host1x_bo_get(bo); | |
| 79 | + map->direction = direction; | |
| 80 | + map->dev = dev; | |
| 81 | + | |
| 82 | + map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); | |
| 83 | + if (!map->sgt) { | |
| 84 | + err = -ENOMEM; | |
| 85 | + goto free; | |
| 83 | 86 | } |
| 84 | 87 | |
| 85 | − return sgt; | |
| 88 | + err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, | |
| 89 | + gather->gather_data_words * 4); | |
| 90 | + if (err) | |
| 91 | + goto free_sgt; | |
| 92 | + | |
| 93 | + err = dma_map_sgtable(dev, map->sgt, direction, 0); | |
| 94 | + if (err) | |
| 95 | + goto free_sgt; | |
| 96 | + | |
| 97 | + map->phys = sg_dma_address(map->sgt->sgl); | |
| 98 | + map->size = gather->gather_data_words * 4; | |
| 99 | + map->chunks = err; | |
| 100 | + | |
| 101 | + return map; | |
| 102 | + | |
| 103 | +free_sgt: | |
| 104 | + sg_free_table(map->sgt); | |
| 105 | + kfree(map->sgt); | |
| 106 | +free: | |
| 107 | + kfree(map); | |
| 108 | + return ERR_PTR(err); | |
| 86 | 109 | } |
| 87 | 110 | |
| 88 | −static void gather_bo_unpin(struct device *dev, struct sg_table *sgt) | |
| 111 | +static void gather_bo_unpin(struct host1x_bo_mapping *map) | |
| 89 | 112 | { |
| 90 | − if (sgt) { | |
| 91 | − sg_free_table(sgt); | |
| 92 | − kfree(sgt); | |
| 93 | − } | |
| 113 | + if (!map) | |
| 114 | + return; | |
| 115 | + | |
| 116 | + dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); | |
| 117 | + sg_free_table(map->sgt); | |
| 118 | + kfree(map->sgt); | |
| 119 | + host1x_bo_put(map->bo); | |
| 120 | + | |
| 121 | + kfree(map); | |
| 94 | 122 | } |
| 95 | 123 | |
| 96 | 124 | static void *gather_bo_mmap(struct host1x_bo *host_bo) |
| @@ -17,11 +17,7 @@ static void tegra_drm_mapping_release(struct kref *ref) · 16 unmodified lines | ||
| 17 | 17 | struct tegra_drm_mapping *mapping = |
| 18 | 18 | container_of(ref, struct tegra_drm_mapping, ref); |
| 19 | 19 | |
| 20 | − if (mapping->sgt) | |
| 21 | − dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, | |
| 22 | − DMA_ATTR_SKIP_CPU_SYNC); | |
| 23 | − | |
| 24 | − host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); | |
| 20 | + host1x_bo_unpin(mapping->map); | |
| 25 | 21 | host1x_bo_put(mapping->bo); |
| 26 | 22 | |
| 27 | 23 | kfree(mapping); |
| @@ -159,6 +155,7 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f · 131 unmodified lines | ||
| 159 | 155 | struct drm_tegra_channel_map *args = data; |
| 160 | 156 | struct tegra_drm_mapping *mapping; |
| 161 | 157 | struct tegra_drm_context *context; |
| 158 | + enum dma_data_direction direction; | |
| 162 | 159 | int err = 0; |
| 163 | 160 | |
| 164 | 161 | if (args->flags & ~DRM_TEGRA_CHANNEL_MAP_READ_WRITE) |
| @@ -180,68 +177,53 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f · 15 unmodified lines | ||
| 180 | 177 | |
| 181 | 178 | kref_init(&mapping->ref); |
| 182 | 179 | |
| 183 | − mapping->dev = context->client->base.dev; | |
| 184 | 180 | mapping->bo = tegra_gem_lookup(file, args->handle); |
| 185 | 181 | if (!mapping->bo) { |
| 186 | 182 | err = -EINVAL; |
| 187 | − goto unlock; | |
| 183 | + goto free; | |
| 188 | 184 | } |
| 189 | 185 | |
| 190 | − if (context->client->base.group) { | |
| 191 | − /* IOMMU domain managed directly using IOMMU API */ | |
| 192 | − host1x_bo_pin(mapping->dev, mapping->bo, &mapping->iova); | |
| 193 | − } else { | |
| 194 | − switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { | |
| 195 | − case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: | |
| 196 | − mapping->direction = DMA_BIDIRECTIONAL; | |
| 197 | − break; | |
| 198 | − | |
| 199 | − case DRM_TEGRA_CHANNEL_MAP_WRITE: | |
| 200 | − mapping->direction = DMA_FROM_DEVICE; | |
| 201 | − break; | |
| 202 | − | |
| 203 | − case DRM_TEGRA_CHANNEL_MAP_READ: | |
| 204 | − mapping->direction = DMA_TO_DEVICE; | |
| 205 | − break; | |
| 186 | + switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { | |
| 187 | + case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: | |
| 188 | + direction = DMA_BIDIRECTIONAL; | |
| 189 | + break; | |
| 206 | 190 | |
| 207 | − default: | |
| 208 | − return -EINVAL; | |
| 209 | − } | |
| 191 | + case DRM_TEGRA_CHANNEL_MAP_WRITE: | |
| 192 | + direction = DMA_FROM_DEVICE; | |
| 193 | + break; | |
| 210 | 194 | |
| 211 | − mapping->sgt = host1x_bo_pin(mapping->dev, mapping->bo, NULL); | |
| 212 | − if (IS_ERR(mapping->sgt)) { | |
| 213 | − err = PTR_ERR(mapping->sgt); | |
| 214 | − goto put_gem; | |
| 215 | − } | |
| 195 | + case DRM_TEGRA_CHANNEL_MAP_READ: | |
| 196 | + direction = DMA_TO_DEVICE; | |
| 197 | + break; | |
| 216 | 198 | |
| 217 | − err = dma_map_sgtable(mapping->dev, mapping->sgt, mapping->direction, | |
| 218 | − DMA_ATTR_SKIP_CPU_SYNC); | |
| 219 | − if (err) | |
| 220 | − goto unpin; | |
| 199 | + default: | |
| 200 | + err = -EINVAL; | |
| 201 | + goto put_gem; | |
| 202 | + } | |
| 221 | 203 | |
| 222 | − mapping->iova = sg_dma_address(mapping->sgt->sgl); | |
| 204 | + mapping->map = host1x_bo_pin(context->client->base.dev, mapping->bo, direction); | |
| 205 | + if (IS_ERR(mapping->map)) { | |
| 206 | + err = PTR_ERR(mapping->map); | |
| 207 | + goto put_gem; | |
| 223 | 208 | } |
| 224 | 209 | |
| 210 | + mapping->iova = mapping->map->phys; | |
| 225 | 211 | mapping->iova_end = mapping->iova + host1x_to_tegra_bo(mapping->bo)->gem.size; |
| 226 | 212 | |
| 227 | 213 | err = xa_alloc(&context->mappings, &args->mapping, mapping, XA_LIMIT(1, U32_MAX), |
| 228 | 214 | GFP_KERNEL); |
| 229 | 215 | if (err < 0) |
| 230 | − goto unmap; | |
| 216 | + goto unpin; | |
| 231 | 217 | |
| 232 | 218 | mutex_unlock(&fpriv->lock); |
| 233 | 219 | |
| 234 | 220 | return 0; |
| 235 | 221 | |
| 236 | −unmap: | |
| 237 | − if (mapping->sgt) { | |
| 238 | − dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, | |
| 239 | − DMA_ATTR_SKIP_CPU_SYNC); | |
| 240 | − } | |
| 241 | 222 | unpin: |
| 242 | − host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); | |
| 223 | + host1x_bo_unpin(mapping->map); | |
| 243 | 224 | put_gem: |
| 244 | 225 | host1x_bo_put(mapping->bo); |
| 226 | +free: | |
| 245 | 227 | kfree(mapping); |
| 246 | 228 | unlock: |
| 247 | 229 | mutex_unlock(&fpriv->lock); |
| @@ -27,10 +27,9 @@ struct tegra_drm_file { · 26 unmodified lines | ||
| 27 | 27 | struct tegra_drm_mapping { |
| 28 | 28 | struct kref ref; |
| 29 | 29 | |
| 30 | − struct device *dev; | |
| 30 | + struct host1x_bo_mapping *map; | |
| 31 | 31 | struct host1x_bo *bo; |
| 32 | − struct sg_table *sgt; | |
| 33 | − enum dma_data_direction direction; | |
| 32 | + | |
| 34 | 33 | dma_addr_t iova; |
| 35 | 34 | dma_addr_t iova_end; |
| 36 | 35 | }; |
| @@ -134,20 +134,20 @@ EXPORT_SYMBOL(host1x_job_add_wait); · 133 unmodified lines | ||
| 134 | 134 | |
| 135 | 135 | static unsigned int pin_job(struct host1x *host, struct host1x_job *job) |
| 136 | 136 | { |
| 137 | + unsigned long mask = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE; | |
| 137 | 138 | struct host1x_client *client = job->client; |
| 138 | 139 | struct device *dev = client->dev; |
| 139 | 140 | struct host1x_job_gather *g; |
| 140 | − struct iommu_domain *domain; | |
| 141 | − struct sg_table *sgt; | |
| 142 | 141 | unsigned int i; |
| 143 | 142 | int err; |
| 144 | 143 | |
| 145 | − domain = iommu_get_domain_for_dev(dev); | |
| 146 | 144 | job->num_unpins = 0; |
| 147 | 145 | |
| 148 | 146 | for (i = 0; i < job->num_relocs; i++) { |
| 149 | 147 | struct host1x_reloc *reloc = &job->relocs[i]; |
| 150 | − dma_addr_t phys_addr, *phys; | |
| 148 | + enum dma_data_direction direction; | |
| 149 | + struct host1x_bo_mapping *map; | |
| 150 | + struct host1x_bo *bo; | |
| 151 | 151 | |
| 152 | 152 | reloc->target.bo = host1x_bo_get(reloc->target.bo); |
| 153 | 153 | if (!reloc->target.bo) { |
| @@ -155,64 +155,44 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 1 unmodified line | ||
| 155 | 155 | goto unpin; |
| 156 | 156 | } |
| 157 | 157 | |
| 158 | − /* | |
| 159 | − * If the client device is not attached to an IOMMU, the | |
| 160 | − * physical address of the buffer object can be used. | |
| 161 | − * | |
| 162 | − * Similarly, when an IOMMU domain is shared between all | |
| 163 | − * host1x clients, the IOVA is already available, so no | |
| 164 | − * need to map the buffer object again. | |
| 165 | − * | |
| 166 | − * XXX Note that this isn't always safe to do because it | |
| 167 | − * relies on an assumption that no cache maintenance is | |
| 168 | − * needed on the buffer objects. | |
| 169 | − */ | |
| 170 | − if (!domain || client->group) | |
| 171 | − phys = &phys_addr; | |
| 172 | − else | |
| 173 | − phys = NULL; | |
| 174 | − | |
| 175 | − sgt = host1x_bo_pin(dev, reloc->target.bo, phys); | |
| 176 | − if (IS_ERR(sgt)) { | |
| 177 | − err = PTR_ERR(sgt); | |
| 178 | − goto unpin; | |
| 179 | − } | |
| 158 | + bo = reloc->target.bo; | |
| 180 | 159 | |
| 181 | − if (sgt) { | |
| 182 | − unsigned long mask = HOST1X_RELOC_READ | | |
| 183 | − HOST1X_RELOC_WRITE; | |
| 184 | − enum dma_data_direction dir; | |
| 185 | − | |
| 186 | − switch (reloc->flags & mask) { | |
| 187 | − case HOST1X_RELOC_READ: | |
| 188 | − dir = DMA_TO_DEVICE; | |
| 189 | − break; | |
| 160 | + switch (reloc->flags & mask) { | |
| 161 | + case HOST1X_RELOC_READ: | |
| 162 | + direction = DMA_TO_DEVICE; | |
| 163 | + break; | |
| 190 | 164 | |
| 191 | − case HOST1X_RELOC_WRITE: | |
| 192 | − dir = DMA_FROM_DEVICE; | |
| 193 | − break; | |
| 165 | + case HOST1X_RELOC_WRITE: | |
| 166 | + direction = DMA_FROM_DEVICE; | |
| 167 | + break; | |
| 194 | 168 | |
| 195 | − case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: | |
| 196 | − dir = DMA_BIDIRECTIONAL; | |
| 197 | − break; | |
| 169 | + case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: | |
| 170 | + direction = DMA_BIDIRECTIONAL; | |
| 171 | + break; | |
| 198 | 172 | |
| 199 | − default: | |
| 200 | − err = -EINVAL; | |
| 201 | − goto unpin; | |
| 202 | − } | |
| 173 | + default: | |
| 174 | + err = -EINVAL; | |
| 175 | + goto unpin; | |
| 176 | + } | |
| 203 | 177 | |
| 204 | − err = dma_map_sgtable(dev, sgt, dir, 0); | |
| 205 | − if (err) | |
| 206 | − goto unpin; | |
| 178 | + map = host1x_bo_pin(dev, bo, direction); | |
| 179 | + if (IS_ERR(map)) { | |
| 180 | + err = PTR_ERR(map); | |
| 181 | + goto unpin; | |
| 182 | + } | |
| 207 | 183 | |
| 208 | − job->unpins[job->num_unpins].dev = dev; | |
| 209 | − job->unpins[job->num_unpins].dir = dir; | |
| 210 | − phys_addr = sg_dma_address(sgt->sgl); | |
| 184 | + /* | |
| 185 | + * host1x clients are generally not able to do scatter-gather themselves, so fail | |
| 186 | + * if the buffer is discontiguous and we fail to map its SG table to a single | |
| 187 | + * contiguous chunk of I/O virtual memory. | |
| 188 | + */ | |
| 189 | + if (map->chunks > 1) { | |
| 190 | + err = -EINVAL; | |
| 191 | + goto unpin; | |
| 211 | 192 | } |
| 212 | 193 | |
| 213 | − job->addr_phys[job->num_unpins] = phys_addr; | |
| 214 | − job->unpins[job->num_unpins].bo = reloc->target.bo; | |
| 215 | − job->unpins[job->num_unpins].sgt = sgt; | |
| 194 | + job->addr_phys[job->num_unpins] = map->phys; | |
| 195 | + job->unpins[job->num_unpins].map = map; | |
| 216 | 196 | job->num_unpins++; |
| 217 | 197 | } |
| 218 | 198 | |
| @@ -224,12 +204,11 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 5 unmodified lines | ||
| 224 | 204 | return 0; |
| 225 | 205 | |
| 226 | 206 | for (i = 0; i < job->num_cmds; i++) { |
| 207 | + struct host1x_bo_mapping *map; | |
| 227 | 208 | size_t gather_size = 0; |
| 228 | 209 | struct scatterlist *sg; |
| 229 | − dma_addr_t phys_addr; | |
| 230 | 210 | unsigned long shift; |
| 231 | 211 | struct iova *alloc; |
| 232 | − dma_addr_t *phys; | |
| 233 | 212 | unsigned int j; |
| 234 | 213 | |
| 235 | 214 | if (job->cmds[i].is_wait) |
| @@ -243,25 +222,16 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 7 unmodified lines | ||
| 243 | 222 | goto unpin; |
| 244 | 223 | } |
| 245 | 224 | |
| 246 | − /** | |
| 247 | − * If the host1x is not attached to an IOMMU, there is no need | |
| 248 | − * to map the buffer object for the host1x, since the physical | |
| 249 | − * address can simply be used. | |
| 250 | − */ | |
| 251 | − if (!iommu_get_domain_for_dev(host->dev)) | |
| 252 | − phys = &phys_addr; | |
| 253 | − else | |
| 254 | − phys = NULL; | |
| 255 | − | |
| 256 | − sgt = host1x_bo_pin(host->dev, g->bo, phys); | |
| 257 | − if (IS_ERR(sgt)) { | |
| 258 | − err = PTR_ERR(sgt); | |
| 259 | − goto put; | |
| 225 | + map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE); | |
| 226 | + if (IS_ERR(map)) { | |
| 227 | + err = PTR_ERR(map); | |
| 228 | + goto unpin; | |
| 260 | 229 | } |
| 261 | 230 | |
| 262 | 231 | if (host->domain) { |
| 263 | − for_each_sgtable_sg(sgt, sg, j) | |
| 232 | + for_each_sgtable_sg(map->sgt, sg, j) | |
| 264 | 233 | gather_size += sg->length; |
| 234 | + | |
| 265 | 235 | gather_size = iova_align(&host->iova, gather_size); |
| 266 | 236 | |
| 267 | 237 | shift = iova_shift(&host->iova); |
| @@ -272,33 +242,23 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 4 unmodified lines | ||
| 272 | 242 | goto put; |
| 273 | 243 | } |
| 274 | 244 | |
| 275 | − err = iommu_map_sgtable(host->domain, | |
| 276 | − iova_dma_addr(&host->iova, alloc), | |
| 277 | − sgt, IOMMU_READ); | |
| 245 | + err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), | |
| 246 | + map->sgt, IOMMU_READ); | |
| 278 | 247 | if (err == 0) { |
| 279 | 248 | __free_iova(&host->iova, alloc); |
| 280 | 249 | err = -EINVAL; |
| 281 | 250 | goto put; |
| 282 | 251 | } |
| 283 | 252 | |
| 284 | − job->unpins[job->num_unpins].size = gather_size; | |
| 285 | − phys_addr = iova_dma_addr(&host->iova, alloc); | |
| 286 | − } else if (sgt) { | |
| 287 | − err = dma_map_sgtable(host->dev, sgt, DMA_TO_DEVICE, 0); | |
| 288 | − if (err) | |
| 289 | − goto put; | |
| 290 | − | |
| 291 | − job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; | |
| 292 | − job->unpins[job->num_unpins].dev = host->dev; | |
| 293 | − phys_addr = sg_dma_address(sgt->sgl); | |
| 253 | + map->phys = iova_dma_addr(&host->iova, alloc); | |
| 254 | + map->size = gather_size; | |
| 294 | 255 | } |
| 295 | 256 | |
| 296 | − job->addr_phys[job->num_unpins] = phys_addr; | |
| 297 | − job->gather_addr_phys[i] = phys_addr; | |
| 298 | − | |
| 299 | − job->unpins[job->num_unpins].bo = g->bo; | |
| 300 | − job->unpins[job->num_unpins].sgt = sgt; | |
| 257 | + job->addr_phys[job->num_unpins] = map->phys; | |
| 258 | + job->unpins[job->num_unpins].map = map; | |
| 301 | 259 | job->num_unpins++; |
| 260 | + | |
| 261 | + job->gather_addr_phys[i] = map->phys; | |
| 302 | 262 | } |
| 303 | 263 | |
| 304 | 264 | return 0; |
| @@ -690,22 +650,16 @@ void host1x_job_unpin(struct host1x_job *job) · 385 unmodified lines | ||
| 690 | 650 | unsigned int i; |
| 691 | 651 | |
| 692 | 652 | for (i = 0; i < job->num_unpins; i++) { |
| 693 | − struct host1x_job_unpin_data *unpin = &job->unpins[i]; | |
| 694 | − struct device *dev = unpin->dev ?: host->dev; | |
| 695 | − struct sg_table *sgt = unpin->sgt; | |
| 696 | − | |
| 697 | − if (!job->enable_firewall && unpin->size && host->domain) { | |
| 698 | − iommu_unmap(host->domain, job->addr_phys[i], | |
| 699 | − unpin->size); | |
| 700 | − free_iova(&host->iova, | |
| 701 | − iova_pfn(&host->iova, job->addr_phys[i])); | |
| 702 | − } | |
| 653 | + struct host1x_bo_mapping *map = job->unpins[i].map; | |
| 654 | + struct host1x_bo *bo = map->bo; | |
| 703 | 655 | |
| 704 | − if (unpin->dev && sgt) | |
| 705 | − dma_unmap_sgtable(unpin->dev, sgt, unpin->dir, 0); | |
| 656 | + if (!job->enable_firewall && map->size && host->domain) { | |
| 657 | + iommu_unmap(host->domain, job->addr_phys[i], map->size); | |
| 658 | + free_iova(&host->iova, iova_pfn(&host->iova, job->addr_phys[i])); | |
| 659 | + } | |
| 706 | 660 | |
| 707 | − host1x_bo_unpin(dev, unpin->bo, sgt); | |
| 708 | − host1x_bo_put(unpin->bo); | |
| 661 | + host1x_bo_unpin(map); | |
| 662 | + host1x_bo_put(bo); | |
| 709 | 663 | } |
| 710 | 664 | |
| 711 | 665 | job->num_unpins = 0; |
| @@ -35,11 +35,7 @@ struct host1x_job_cmd { · 34 unmodified lines | ||
| 35 | 35 | }; |
| 36 | 36 | |
| 37 | 37 | struct host1x_job_unpin_data { |
| 38 | − struct host1x_bo *bo; | |
| 39 | − struct sg_table *sgt; | |
| 40 | − struct device *dev; | |
| 41 | − size_t size; | |
| 42 | − enum dma_data_direction dir; | |
| 38 | + struct host1x_bo_mapping *map; | |
| 43 | 39 | }; |
| 44 | 40 | |
| 45 | 41 | /* |
| @@ -7,6 +7,7 @@ · 6 unmodified lines | ||
| 7 | 7 | #define __LINUX_HOST1X_H |
| 8 | 8 | |
| 9 | 9 | #include <linux/device.h> |
| 10 | +#include <linux/dma-direction.h> | |
| 10 | 11 | #include <linux/types.h> |
| 11 | 12 | |
| 12 | 13 | enum host1x_class { |
| @@ -82,12 +83,23 @@ struct host1x_client { · 69 unmodified lines | ||
| 82 | 83 | struct host1x_bo; |
| 83 | 84 | struct sg_table; |
| 84 | 85 | |
| 86 | +struct host1x_bo_mapping { | |
| 87 | + struct dma_buf_attachment *attach; | |
| 88 | + enum dma_data_direction direction; | |
| 89 | + struct host1x_bo *bo; | |
| 90 | + struct sg_table *sgt; | |
| 91 | + unsigned int chunks; | |
| 92 | + struct device *dev; | |
| 93 | + dma_addr_t phys; | |
| 94 | + size_t size; | |
| 95 | +}; | |
| 96 | + | |
| 85 | 97 | struct host1x_bo_ops { |
| 86 | 98 | struct host1x_bo *(*get)(struct host1x_bo *bo); |
| 87 | 99 | void (*put)(struct host1x_bo *bo); |
| 88 | − struct sg_table *(*pin)(struct device *dev, struct host1x_bo *bo, | |
| 89 | − dma_addr_t *phys); | |
| 90 | − void (*unpin)(struct device *dev, struct sg_table *sgt); | |
| 100 | + struct host1x_bo_mapping *(*pin)(struct device *dev, struct host1x_bo *bo, | |
| 101 | + enum dma_data_direction dir); | |
| 102 | + void (*unpin)(struct host1x_bo_mapping *map); | |
| 91 | 103 | void *(*mmap)(struct host1x_bo *bo); |
| 92 | 104 | void (*munmap)(struct host1x_bo *bo, void *addr); |
| 93 | 105 | }; |
| @@ -112,17 +124,15 @@ static inline void host1x_bo_put(struct host1x_bo *bo) · 18 unmodified lines | ||
| 112 | 124 | bo->ops->put(bo); |
| 113 | 125 | } |
| 114 | 126 | |
| 115 | −static inline struct sg_table *host1x_bo_pin(struct device *dev, | |
| 116 | − struct host1x_bo *bo, | |
| 117 | − dma_addr_t *phys) | |
| 127 | +static inline struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, | |
| 128 | + enum dma_data_direction dir) | |
| 118 | 129 | { |
| 119 | − return bo->ops->pin(dev, bo, phys); | |
| 130 | + return bo->ops->pin(dev, bo, dir); | |
| 120 | 131 | } |
| 121 | 132 | |
| 122 | −static inline void host1x_bo_unpin(struct device *dev, struct host1x_bo *bo, | |
| 123 | − struct sg_table *sgt) | |
| 133 | +static inline void host1x_bo_unpin(struct host1x_bo_mapping *map) | |
| 124 | 134 | { |
| 125 | − bo->ops->unpin(dev, sgt); | |
| 135 | + map->bo->ops->unpin(map); | |
| 126 | 136 | } |
| 127 | 137 | |
| 128 | 138 | static inline void *host1x_bo_mmap(struct host1x_bo *bo) |