DMA-BUF requires that each device that accesses a DMA-BUF attaches to it separately. To do so the host1x_bo_pin() and host1x_bo_unpin() functions need to be reimplemented so that they can return a mapping, which either represents an attachment or a map of the driver's own GEM object.
Unified · Split
| @@ -23,86 +23,96 @@ · 22 unmodified lines | |||
| 23 | 23 | ||
| 24 | MODULE_IMPORT_NS(DMA_BUF); | 24 | MODULE_IMPORT_NS(DMA_BUF); |
| 25 | 25 | ||
| 26 | static void tegra_bo_put(struct host1x_bo *bo) | 26 | static unsigned int sg_dma_count_chunks(struct scatterlist *sgl, unsigned int nents) |
| 27 | { | 27 | { |
| 28 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); | 28 | dma_addr_t next = ~(dma_addr_t)0; |
| 29 | unsigned int count = 0, i; | ||
| 30 | struct scatterlist *s; | ||
| 29 | 31 | ||
| 30 | drm_gem_object_put(&obj->gem); | 32 | for_each_sg(sgl, s, nents, i) { |
| 31 | } | 33 | /* sg_dma_address(s) is only valid for entries that have sg_dma_len(s) != 0. */ |
| 34 | if (!sg_dma_len(s)) | ||
| 35 | continue; | ||
| 32 | 36 | ||
| 33 | /* XXX move this into lib/scatterlist.c? */ | 37 | if (sg_dma_address(s) != next) { |
| 34 | static int sg_alloc_table_from_sg(struct sg_table *sgt, struct scatterlist *sg, | 38 | next = sg_dma_address(s) + sg_dma_len(s); |
| 35 | unsigned int nents, gfp_t gfp_mask) | 39 | count++; |
| 36 | { | 40 | } |
| 37 | struct scatterlist *dst; | 41 | } |
| 38 | unsigned int i; | ||
| 39 | int err; | ||
| 40 | 42 | ||
| 41 | err = sg_alloc_table(sgt, nents, gfp_mask); | 43 | return count; |
| 42 | if (err < 0) | 44 | } |
| 43 | return err; | ||
| 44 | 45 | ||
| 45 | dst = sgt->sgl; | 46 | static inline unsigned int sgt_dma_count_chunks(struct sg_table *sgt) |
| 47 | { | ||
| 48 | return sg_dma_count_chunks(sgt->sgl, sgt->nents); | ||
| 49 | } | ||
| 46 | 50 | ||
| 47 | for (i = 0; i < nents; i++) { | 51 | static void tegra_bo_put(struct host1x_bo *bo) |
| 48 | sg_set_page(dst, sg_page(sg), sg->length, 0); | 52 | { |
| 49 | dst = sg_next(dst); | 53 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
| 50 | sg = sg_next(sg); | ||
| 51 | } | ||
| 52 | 54 | ||
| 53 | return 0; | 55 | drm_gem_object_put(&obj->gem); |
| 54 | } | 56 | } |
| 55 | 57 | ||
| 56 | static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, | 58 | static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, |
| 57 | dma_addr_t *phys) | 59 | enum dma_data_direction direction) |
| 58 | { | 60 | { |
| 59 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); | 61 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
| 60 | struct sg_table *sgt; | 62 | struct drm_gem_object *gem = &obj->gem; |
| 63 | struct host1x_bo_mapping *map; | ||
| 61 | int err; | 64 | int err; |
| 62 | 65 | ||
| 66 | map = kzalloc(sizeof(*map), GFP_KERNEL); | ||
| 67 | if (!map) | ||
| 68 | return ERR_PTR(-ENOMEM); | ||
| 69 | |||
| 70 | map->bo = host1x_bo_get(bo); | ||
| 71 | map->direction = direction; | ||
| 72 | map->dev = dev; | ||
| 73 | |||
| 63 | /* | 74 | /* |
| 64 | * If we've manually mapped the buffer object through the IOMMU, make | 75 | * Imported buffers need special treatment to satisfy the semantics of DMA-BUF. |
| 65 | * sure to return the IOVA address of our mapping. | ||
| 66 | * | ||
| 67 | * Similarly, for buffers that have been allocated by the DMA API the | ||
| 68 | * physical address can be used for devices that are not attached to | ||
| 69 | * an IOMMU. For these devices, callers must pass a valid pointer via | ||
| 70 | * the @phys argument. | ||
| 71 | * | ||
| 72 | * Imported buffers were also already mapped at import time, so the | ||
| 73 | * existing mapping can be reused. | ||
| 74 | */ | 76 | */ |
| 75 | if (phys) { | 77 | if (gem->import_attach) { |
| 76 | *phys = obj->iova; | 78 | struct dma_buf *buf = gem->import_attach->dmabuf; |
| 77 | return NULL; | 79 | |
| 80 | map->attach = dma_buf_attach(buf, dev); | ||
| 81 | if (IS_ERR(map->attach)) { | ||
| 82 | err = PTR_ERR(map->attach); | ||
| 83 | goto free; | ||
| 84 | } | ||
| 85 | |||
| 86 | map->sgt = dma_buf_map_attachment(map->attach, direction); | ||
| 87 | if (IS_ERR(map->sgt)) { | ||
| 88 | dma_buf_detach(buf, map->attach); | ||
| 89 | err = PTR_ERR(map->sgt); | ||
| 90 | goto free; | ||
| 91 | } | ||
| 92 | |||
| 93 | err = sgt_dma_count_chunks(map->sgt); | ||
| 94 | map->size = gem->size; | ||
| 95 | |||
| 96 | goto out; | ||
| 78 | } | 97 | } |
| 79 | 98 | ||
| 80 | /* | 99 | /* |
| 81 | * If we don't have a mapping for this buffer yet, return an SG table | 100 | * If we don't have a mapping for this buffer yet, return an SG table |
| 82 | * so that host1x can do the mapping for us via the DMA API. | 101 | * so that host1x can do the mapping for us via the DMA API. |
| 83 | */ | 102 | */ |
| 84 | sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); | 103 | map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); |
| 85 | if (!sgt) | 104 | if (!map->sgt) { |
| 86 | return ERR_PTR(-ENOMEM); | 105 | err = -ENOMEM; |
| 106 | goto free; | ||
| 107 | } | ||
| 87 | 108 | ||
| 88 | if (obj->pages) { | 109 | if (obj->pages) { |
| 89 | /* | 110 | /* |
| 90 | * If the buffer object was allocated from the explicit IOMMU | 111 | * If the buffer object was allocated from the explicit IOMMU |
| 91 | * API code paths, construct an SG table from the pages. | 112 | * API code paths, construct an SG table from the pages. |
| 92 | */ | 113 | */ |
| 93 | err = sg_alloc_table_from_pages(sgt, obj->pages, obj->num_pages, | 114 | err = sg_alloc_table_from_pages(map->sgt, obj->pages, obj->num_pages, 0, gem->size, |
| 94 | 0, obj->gem.size, GFP_KERNEL); | 115 | GFP_KERNEL); |
| 95 | if (err < 0) | ||
| 96 | goto free; | ||
| 97 | } else if (obj->sgt) { | ||
| 98 | /* | ||
| 99 | * If the buffer object already has an SG table but no pages | ||
| 100 | * were allocated for it, it means the buffer was imported and | ||
| 101 | * the SG table needs to be copied to avoid overwriting any | ||
| 102 | * other potential users of the original SG table. | ||
| 103 | */ | ||
| 104 | err = sg_alloc_table_from_sg(sgt, obj->sgt->sgl, | ||
| 105 | obj->sgt->orig_nents, GFP_KERNEL); | ||
| 106 | if (err < 0) | 116 | if (err < 0) |
| 107 | goto free; | 117 | goto free; |
| 108 | } else { | 118 | } else { |
| @@ -111,25 +121,56 @@ static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, · 2 unmodified lines | |||
| 111 | * not imported, it had to be allocated with the DMA API, so | 121 | * not imported, it had to be allocated with the DMA API, so |
| 112 | * the DMA API helper can be used. | 122 | * the DMA API helper can be used. |
| 113 | */ | 123 | */ |
| 114 | err = dma_get_sgtable(dev, sgt, obj->vaddr, obj->iova, | 124 | err = dma_get_sgtable(dev, map->sgt, obj->vaddr, obj->iova, gem->size); |
| 115 | obj->gem.size); | ||
| 116 | if (err < 0) | 125 | if (err < 0) |
| 117 | goto free; | 126 | goto free; |
| 118 | } | 127 | } |
| 119 | 128 | ||
| 120 | return sgt; | 129 | err = dma_map_sgtable(dev, map->sgt, direction, 0); |
| 130 | if (err) | ||
| 131 | goto free_sgt; | ||
| 132 | |||
| 133 | out: | ||
| 134 | /* | ||
| 135 | * If we've manually mapped the buffer object through the IOMMU, make sure to return the | ||
| 136 | * existing IOVA address of our mapping. | ||
| 137 | */ | ||
| 138 | if (!obj->mm) { | ||
| 139 | map->phys = sg_dma_address(map->sgt->sgl); | ||
| 140 | map->chunks = err; | ||
| 141 | } else { | ||
| 142 | map->phys = obj->iova; | ||
| 143 | map->chunks = 1; | ||
| 144 | } | ||
| 145 | |||
| 146 | map->size = gem->size; | ||
| 121 | 147 | ||
| 148 | return map; | ||
| 149 | |||
| 150 | free_sgt: | ||
| 151 | sg_free_table(map->sgt); | ||
| 122 | free: | 152 | free: |
| 123 | kfree(sgt); | 153 | kfree(map->sgt); |
| 154 | kfree(map); | ||
| 124 | return ERR_PTR(err); | 155 | return ERR_PTR(err); |
| 125 | } | 156 | } |
| 126 | 157 | ||
| 127 | static void tegra_bo_unpin(struct device *dev, struct sg_table *sgt) | 158 | static void tegra_bo_unpin(struct host1x_bo_mapping *map) |
| 128 | { | 159 | { |
| 129 | if (sgt) { | 160 | if (!map) |
| 130 | sg_free_table(sgt); | 161 | return; |
| 131 | kfree(sgt); | 162 | |
| 163 | if (map->attach) { | ||
| 164 | dma_buf_unmap_attachment(map->attach, map->sgt, map->direction); | ||
| 165 | dma_buf_detach(map->attach->dmabuf, map->attach); | ||
| 166 | } else { | ||
| 167 | dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); | ||
| 168 | sg_free_table(map->sgt); | ||
| 169 | kfree(map->sgt); | ||
| 132 | } | 170 | } |
| 171 | |||
| 172 | host1x_bo_put(map->bo); | ||
| 173 | kfree(map); | ||
| 133 | } | 174 | } |
| 134 | 175 | ||
| 135 | static void *tegra_bo_mmap(struct host1x_bo *bo) | 176 | static void *tegra_bo_mmap(struct host1x_bo *bo) |
| @@ -74,7 +74,7 @@ tegra_plane_atomic_duplicate_state(struct drm_plane *plane) · 73 unmodified lines | |||
| 74 | 74 | ||
| 75 | for (i = 0; i < 3; i++) { | 75 | for (i = 0; i < 3; i++) { |
| 76 | copy->iova[i] = DMA_MAPPING_ERROR; | 76 | copy->iova[i] = DMA_MAPPING_ERROR; |
| 77 | copy->sgt[i] = NULL; | 77 | copy->map[i] = NULL; |
| 78 | } | 78 | } |
| 79 | 79 | ||
| 80 | return ©->base; | 80 | return ©->base; |
| @@ -138,55 +138,37 @@ const struct drm_plane_funcs tegra_plane_funcs = { · 57 unmodified lines | |||
| 138 | 138 | ||
| 139 | static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) | 139 | static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) |
| 140 | { | 140 | { |
| 141 | struct iommu_domain *domain = iommu_get_domain_for_dev(dc->dev); | ||
| 142 | unsigned int i; | 141 | unsigned int i; |
| 143 | int err; | 142 | int err; |
| 144 | 143 | ||
| 145 | for (i = 0; i < state->base.fb->format->num_planes; i++) { | 144 | for (i = 0; i < state->base.fb->format->num_planes; i++) { |
| 146 | struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); | 145 | struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); |
| 147 | dma_addr_t phys_addr, *phys; | 146 | struct host1x_bo_mapping *map; |
| 148 | struct sg_table *sgt; | ||
| 149 | 147 | ||
| 150 | /* | 148 | map = host1x_bo_pin(dc->dev, &bo->base, DMA_TO_DEVICE); |
| 151 | * If we're not attached to a domain, we already stored the | 149 | if (IS_ERR(map)) { |
| 152 | * physical address when the buffer was allocated. If we're | 150 | err = PTR_ERR(map); |
| 153 | * part of a group that's shared between all display | ||
| 154 | * controllers, we've also already mapped the framebuffer | ||
| 155 | * through the SMMU. In both cases we can short-circuit the | ||
| 156 | * code below and retrieve the stored IOV address. | ||
| 157 | */ | ||
| 158 | if (!domain || dc->client.group) | ||
| 159 | phys = &phys_addr; | ||
| 160 | else | ||
| 161 | phys = NULL; | ||
| 162 | |||
| 163 | sgt = host1x_bo_pin(dc->dev, &bo->base, phys); | ||
| 164 | if (IS_ERR(sgt)) { | ||
| 165 | err = PTR_ERR(sgt); | ||
| 166 | goto unpin; | 151 | goto unpin; |
| 167 | } | 152 | } |
| 168 | 153 | ||
| 169 | if (sgt) { | 154 | if (!dc->client.group) { |
| 170 | err = dma_map_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); | ||
| 171 | if (err) | ||
| 172 | goto unpin; | ||
| 173 | |||
| 174 | /* | 155 | /* |
| 175 | * The display controller needs contiguous memory, so | 156 | * The display controller needs contiguous memory, so |
| 176 | * fail if the buffer is discontiguous and we fail to | 157 | * fail if the buffer is discontiguous and we fail to |
| 177 | * map its SG table to a single contiguous chunk of | 158 | * map its SG table to a single contiguous chunk of |
| 178 | * I/O virtual memory. | 159 | * I/O virtual memory. |
| 179 | */ | 160 | */ |
| 180 | if (sgt->nents > 1) { | 161 | if (map->chunks > 1) { |
| 181 | err = -EINVAL; | 162 | err = -EINVAL; |
| 182 | goto unpin; | 163 | goto unpin; |
| 183 | } | 164 | } |
| 184 | 165 | ||
| 185 | state->iova[i] = sg_dma_address(sgt->sgl); | 166 | state->iova[i] = map->phys; |
| 186 | state->sgt[i] = sgt; | ||
| 187 | } else { | 167 | } else { |
| 188 | state->iova[i] = phys_addr; | 168 | state->iova[i] = bo->iova; |
| 189 | } | 169 | } |
| 170 | |||
| 171 | state->map[i] = map; | ||
| 190 | } | 172 | } |
| 191 | 173 | ||
| 192 | return 0; | 174 | return 0; |
| @@ -195,15 +177,9 @@ unpin: · 2 unmodified lines | |||
| 195 | dev_err(dc->dev, "failed to map plane %u: %d\n", i, err); | 177 | dev_err(dc->dev, "failed to map plane %u: %d\n", i, err); |
| 196 | 178 | ||
| 197 | while (i--) { | 179 | while (i--) { |
| 198 | struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); | 180 | host1x_bo_unpin(state->map[i]); |
| 199 | struct sg_table *sgt = state->sgt[i]; | ||
| 200 | |||
| 201 | if (sgt) | ||
| 202 | dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); | ||
| 203 | |||
| 204 | host1x_bo_unpin(dc->dev, &bo->base, sgt); | ||
| 205 | state->iova[i] = DMA_MAPPING_ERROR; | 181 | state->iova[i] = DMA_MAPPING_ERROR; |
| 206 | state->sgt[i] = NULL; | 182 | state->map[i] = NULL; |
| 207 | } | 183 | } |
| 208 | 184 | ||
| 209 | return err; | 185 | return err; |
| @@ -214,15 +190,9 @@ static void tegra_dc_unpin(struct tegra_dc *dc, struct tegra_plane_state *state) · 4 unmodified lines | |||
| 214 | unsigned int i; | 190 | unsigned int i; |
| 215 | 191 | ||
| 216 | for (i = 0; i < state->base.fb->format->num_planes; i++) { | 192 | for (i = 0; i < state->base.fb->format->num_planes; i++) { |
| 217 | struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); | 193 | host1x_bo_unpin(state->map[i]); |
| 218 | struct sg_table *sgt = state->sgt[i]; | ||
| 219 | |||
| 220 | if (sgt) | ||
| 221 | dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); | ||
| 222 | |||
| 223 | host1x_bo_unpin(dc->dev, &bo->base, sgt); | ||
| 224 | state->iova[i] = DMA_MAPPING_ERROR; | 194 | state->iova[i] = DMA_MAPPING_ERROR; |
| 225 | state->sgt[i] = NULL; | 195 | state->map[i] = NULL; |
| 226 | } | 196 | } |
| 227 | } | 197 | } |
| 228 | 198 | ||
| @@ -43,7 +43,7 @@ struct tegra_plane_legacy_blending_state { · 42 unmodified lines | |||
| 43 | struct tegra_plane_state { | 43 | struct tegra_plane_state { |
| 44 | struct drm_plane_state base; | 44 | struct drm_plane_state base; |
| 45 | 45 | ||
| 46 | struct sg_table *sgt[3]; | 46 | struct host1x_bo_mapping *map[3]; |
| 47 | dma_addr_t iova[3]; | 47 | dma_addr_t iova[3]; |
| 48 | 48 | ||
| 49 | struct tegra_bo_tiling tiling; | 49 | struct tegra_bo_tiling tiling; |
| @@ -64,33 +64,61 @@ static void gather_bo_put(struct host1x_bo *host_bo) · 63 unmodified lines | |||
| 64 | kref_put(&bo->ref, gather_bo_release); | 64 | kref_put(&bo->ref, gather_bo_release); |
| 65 | } | 65 | } |
| 66 | 66 | ||
| 67 | static struct sg_table * | 67 | static struct host1x_bo_mapping * |
| 68 | gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys) | 68 | gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) |
| 69 | { | 69 | { |
| 70 | struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); | 70 | struct gather_bo *gather = container_of(bo, struct gather_bo, base); |
| 71 | struct sg_table *sgt; | 71 | struct host1x_bo_mapping *map; |
| 72 | int err; | 72 | int err; |
| 73 | 73 | ||
| 74 | sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); | 74 | map = kzalloc(sizeof(*map), GFP_KERNEL); |
| 75 | if (!sgt) | 75 | if (!map) |
| 76 | return ERR_PTR(-ENOMEM); | 76 | return ERR_PTR(-ENOMEM); |
| 77 | 77 | ||
| 78 | err = dma_get_sgtable(bo->dev, sgt, bo->gather_data, bo->gather_data_dma, | 78 | map->bo = host1x_bo_get(bo); |
| 79 | bo->gather_data_words * 4); | 79 | map->direction = direction; |
| 80 | if (err) { | 80 | map->dev = dev; |
| 81 | kfree(sgt); | 81 | |
| 82 | return ERR_PTR(err); | 82 | map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); |
| 83 | if (!map->sgt) { | ||
| 84 | err = -ENOMEM; | ||
| 85 | goto free; | ||
| 83 | } | 86 | } |
| 84 | 87 | ||
| 85 | return sgt; | 88 | err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, |
| 89 | gather->gather_data_words * 4); | ||
| 90 | if (err) | ||
| 91 | goto free_sgt; | ||
| 92 | |||
| 93 | err = dma_map_sgtable(dev, map->sgt, direction, 0); | ||
| 94 | if (err) | ||
| 95 | goto free_sgt; | ||
| 96 | |||
| 97 | map->phys = sg_dma_address(map->sgt->sgl); | ||
| 98 | map->size = gather->gather_data_words * 4; | ||
| 99 | map->chunks = err; | ||
| 100 | |||
| 101 | return map; | ||
| 102 | |||
| 103 | free_sgt: | ||
| 104 | sg_free_table(map->sgt); | ||
| 105 | kfree(map->sgt); | ||
| 106 | free: | ||
| 107 | kfree(map); | ||
| 108 | return ERR_PTR(err); | ||
| 86 | } | 109 | } |
| 87 | 110 | ||
| 88 | static void gather_bo_unpin(struct device *dev, struct sg_table *sgt) | 111 | static void gather_bo_unpin(struct host1x_bo_mapping *map) |
| 89 | { | 112 | { |
| 90 | if (sgt) { | 113 | if (!map) |
| 91 | sg_free_table(sgt); | 114 | return; |
| 92 | kfree(sgt); | 115 | |
| 93 | } | 116 | dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); |
| 117 | sg_free_table(map->sgt); | ||
| 118 | kfree(map->sgt); | ||
| 119 | host1x_bo_put(map->bo); | ||
| 120 | |||
| 121 | kfree(map); | ||
| 94 | } | 122 | } |
| 95 | 123 | ||
| 96 | static void *gather_bo_mmap(struct host1x_bo *host_bo) | 124 | static void *gather_bo_mmap(struct host1x_bo *host_bo) |
| @@ -17,11 +17,7 @@ static void tegra_drm_mapping_release(struct kref *ref) · 16 unmodified lines | |||
| 17 | struct tegra_drm_mapping *mapping = | 17 | struct tegra_drm_mapping *mapping = |
| 18 | container_of(ref, struct tegra_drm_mapping, ref); | 18 | container_of(ref, struct tegra_drm_mapping, ref); |
| 19 | 19 | ||
| 20 | if (mapping->sgt) | 20 | host1x_bo_unpin(mapping->map); |
| 21 | dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, | ||
| 22 | DMA_ATTR_SKIP_CPU_SYNC); | ||
| 23 | |||
| 24 | host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); | ||
| 25 | host1x_bo_put(mapping->bo); | 21 | host1x_bo_put(mapping->bo); |
| 26 | 22 | ||
| 27 | kfree(mapping); | 23 | kfree(mapping); |
| @@ -159,6 +155,7 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f · 131 unmodified lines | |||
| 159 | struct drm_tegra_channel_map *args = data; | 155 | struct drm_tegra_channel_map *args = data; |
| 160 | struct tegra_drm_mapping *mapping; | 156 | struct tegra_drm_mapping *mapping; |
| 161 | struct tegra_drm_context *context; | 157 | struct tegra_drm_context *context; |
| 158 | enum dma_data_direction direction; | ||
| 162 | int err = 0; | 159 | int err = 0; |
| 163 | 160 | ||
| 164 | if (args->flags & ~DRM_TEGRA_CHANNEL_MAP_READ_WRITE) | 161 | if (args->flags & ~DRM_TEGRA_CHANNEL_MAP_READ_WRITE) |
| @@ -180,68 +177,53 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f · 15 unmodified lines | |||
| 180 | 177 | ||
| 181 | kref_init(&mapping->ref); | 178 | kref_init(&mapping->ref); |
| 182 | 179 | ||
| 183 | mapping->dev = context->client->base.dev; | ||
| 184 | mapping->bo = tegra_gem_lookup(file, args->handle); | 180 | mapping->bo = tegra_gem_lookup(file, args->handle); |
| 185 | if (!mapping->bo) { | 181 | if (!mapping->bo) { |
| 186 | err = -EINVAL; | 182 | err = -EINVAL; |
| 187 | goto unlock; | 183 | goto free; |
| 188 | } | 184 | } |
| 189 | 185 | ||
| 190 | if (context->client->base.group) { | 186 | switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { |
| 191 | /* IOMMU domain managed directly using IOMMU API */ | 187 | case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: |
| 192 | host1x_bo_pin(mapping->dev, mapping->bo, &mapping->iova); | 188 | direction = DMA_BIDIRECTIONAL; |
| 193 | } else { | 189 | break; |
| 194 | switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { | ||
| 195 | case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: | ||
| 196 | mapping->direction = DMA_BIDIRECTIONAL; | ||
| 197 | break; | ||
| 198 | |||
| 199 | case DRM_TEGRA_CHANNEL_MAP_WRITE: | ||
| 200 | mapping->direction = DMA_FROM_DEVICE; | ||
| 201 | break; | ||
| 202 | |||
| 203 | case DRM_TEGRA_CHANNEL_MAP_READ: | ||
| 204 | mapping->direction = DMA_TO_DEVICE; | ||
| 205 | break; | ||
| 206 | 190 | ||
| 207 | default: | 191 | case DRM_TEGRA_CHANNEL_MAP_WRITE: |
| 208 | return -EINVAL; | 192 | direction = DMA_FROM_DEVICE; |
| 209 | } | 193 | break; |
| 210 | 194 | ||
| 211 | mapping->sgt = host1x_bo_pin(mapping->dev, mapping->bo, NULL); | 195 | case DRM_TEGRA_CHANNEL_MAP_READ: |
| 212 | if (IS_ERR(mapping->sgt)) { | 196 | direction = DMA_TO_DEVICE; |
| 213 | err = PTR_ERR(mapping->sgt); | 197 | break; |
| 214 | goto put_gem; | ||
| 215 | } | ||
| 216 | 198 | ||
| 217 | err = dma_map_sgtable(mapping->dev, mapping->sgt, mapping->direction, | 199 | default: |
| 218 | DMA_ATTR_SKIP_CPU_SYNC); | 200 | err = -EINVAL; |
| 219 | if (err) | 201 | goto put_gem; |
| 220 | goto unpin; | 202 | } |
| 221 | 203 | ||
| 222 | mapping->iova = sg_dma_address(mapping->sgt->sgl); | 204 | mapping->map = host1x_bo_pin(context->client->base.dev, mapping->bo, direction); |
| 205 | if (IS_ERR(mapping->map)) { | ||
| 206 | err = PTR_ERR(mapping->map); | ||
| 207 | goto put_gem; | ||
| 223 | } | 208 | } |
| 224 | 209 | ||
| 210 | mapping->iova = mapping->map->phys; | ||
| 225 | mapping->iova_end = mapping->iova + host1x_to_tegra_bo(mapping->bo)->gem.size; | 211 | mapping->iova_end = mapping->iova + host1x_to_tegra_bo(mapping->bo)->gem.size; |
| 226 | 212 | ||
| 227 | err = xa_alloc(&context->mappings, &args->mapping, mapping, XA_LIMIT(1, U32_MAX), | 213 | err = xa_alloc(&context->mappings, &args->mapping, mapping, XA_LIMIT(1, U32_MAX), |
| 228 | GFP_KERNEL); | 214 | GFP_KERNEL); |
| 229 | if (err < 0) | 215 | if (err < 0) |
| 230 | goto unmap; | 216 | goto unpin; |
| 231 | 217 | ||
| 232 | mutex_unlock(&fpriv->lock); | 218 | mutex_unlock(&fpriv->lock); |
| 233 | 219 | ||
| 234 | return 0; | 220 | return 0; |
| 235 | 221 | ||
| 236 | unmap: | ||
| 237 | if (mapping->sgt) { | ||
| 238 | dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, | ||
| 239 | DMA_ATTR_SKIP_CPU_SYNC); | ||
| 240 | } | ||
| 241 | unpin: | 222 | unpin: |
| 242 | host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); | 223 | host1x_bo_unpin(mapping->map); |
| 243 | put_gem: | 224 | put_gem: |
| 244 | host1x_bo_put(mapping->bo); | 225 | host1x_bo_put(mapping->bo); |
| 226 | free: | ||
| 245 | kfree(mapping); | 227 | kfree(mapping); |
| 246 | unlock: | 228 | unlock: |
| 247 | mutex_unlock(&fpriv->lock); | 229 | mutex_unlock(&fpriv->lock); |
| @@ -27,10 +27,9 @@ struct tegra_drm_file { · 26 unmodified lines | |||
| 27 | struct tegra_drm_mapping { | 27 | struct tegra_drm_mapping { |
| 28 | struct kref ref; | 28 | struct kref ref; |
| 29 | 29 | ||
| 30 | struct device *dev; | 30 | struct host1x_bo_mapping *map; |
| 31 | struct host1x_bo *bo; | 31 | struct host1x_bo *bo; |
| 32 | struct sg_table *sgt; | 32 | |
| 33 | enum dma_data_direction direction; | ||
| 34 | dma_addr_t iova; | 33 | dma_addr_t iova; |
| 35 | dma_addr_t iova_end; | 34 | dma_addr_t iova_end; |
| 36 | }; | 35 | }; |
| @@ -134,20 +134,20 @@ EXPORT_SYMBOL(host1x_job_add_wait); · 133 unmodified lines | |||
| 134 | 134 | ||
| 135 | static unsigned int pin_job(struct host1x *host, struct host1x_job *job) | 135 | static unsigned int pin_job(struct host1x *host, struct host1x_job *job) |
| 136 | { | 136 | { |
| 137 | unsigned long mask = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE; | ||
| 137 | struct host1x_client *client = job->client; | 138 | struct host1x_client *client = job->client; |
| 138 | struct device *dev = client->dev; | 139 | struct device *dev = client->dev; |
| 139 | struct host1x_job_gather *g; | 140 | struct host1x_job_gather *g; |
| 140 | struct iommu_domain *domain; | ||
| 141 | struct sg_table *sgt; | ||
| 142 | unsigned int i; | 141 | unsigned int i; |
| 143 | int err; | 142 | int err; |
| 144 | 143 | ||
| 145 | domain = iommu_get_domain_for_dev(dev); | ||
| 146 | job->num_unpins = 0; | 144 | job->num_unpins = 0; |
| 147 | 145 | ||
| 148 | for (i = 0; i < job->num_relocs; i++) { | 146 | for (i = 0; i < job->num_relocs; i++) { |
| 149 | struct host1x_reloc *reloc = &job->relocs[i]; | 147 | struct host1x_reloc *reloc = &job->relocs[i]; |
| 150 | dma_addr_t phys_addr, *phys; | 148 | enum dma_data_direction direction; |
| 149 | struct host1x_bo_mapping *map; | ||
| 150 | struct host1x_bo *bo; | ||
| 151 | 151 | ||
| 152 | reloc->target.bo = host1x_bo_get(reloc->target.bo); | 152 | reloc->target.bo = host1x_bo_get(reloc->target.bo); |
| 153 | if (!reloc->target.bo) { | 153 | if (!reloc->target.bo) { |
| @@ -155,64 +155,44 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 1 unmodified line | |||
| 155 | goto unpin; | 155 | goto unpin; |
| 156 | } | 156 | } |
| 157 | 157 | ||
| 158 | /* | 158 | bo = reloc->target.bo; |
| 159 | * If the client device is not attached to an IOMMU, the | ||
| 160 | * physical address of the buffer object can be used. | ||
| 161 | * | ||
| 162 | * Similarly, when an IOMMU domain is shared between all | ||
| 163 | * host1x clients, the IOVA is already available, so no | ||
| 164 | * need to map the buffer object again. | ||
| 165 | * | ||
| 166 | * XXX Note that this isn't always safe to do because it | ||
| 167 | * relies on an assumption that no cache maintenance is | ||
| 168 | * needed on the buffer objects. | ||
| 169 | */ | ||
| 170 | if (!domain || client->group) | ||
| 171 | phys = &phys_addr; | ||
| 172 | else | ||
| 173 | phys = NULL; | ||
| 174 | |||
| 175 | sgt = host1x_bo_pin(dev, reloc->target.bo, phys); | ||
| 176 | if (IS_ERR(sgt)) { | ||
| 177 | err = PTR_ERR(sgt); | ||
| 178 | goto unpin; | ||
| 179 | } | ||
| 180 | 159 | ||
| 181 | if (sgt) { | 160 | switch (reloc->flags & mask) { |
| 182 | unsigned long mask = HOST1X_RELOC_READ | | 161 | case HOST1X_RELOC_READ: |
| 183 | HOST1X_RELOC_WRITE; | 162 | direction = DMA_TO_DEVICE; |
| 184 | enum dma_data_direction dir; | 163 | break; |
| 185 | |||
| 186 | switch (reloc->flags & mask) { | ||
| 187 | case HOST1X_RELOC_READ: | ||
| 188 | dir = DMA_TO_DEVICE; | ||
| 189 | break; | ||
| 190 | 164 | ||
| 191 | case HOST1X_RELOC_WRITE: | 165 | case HOST1X_RELOC_WRITE: |
| 192 | dir = DMA_FROM_DEVICE; | 166 | direction = DMA_FROM_DEVICE; |
| 193 | break; | 167 | break; |
| 194 | 168 | ||
| 195 | case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: | 169 | case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: |
| 196 | dir = DMA_BIDIRECTIONAL; | 170 | direction = DMA_BIDIRECTIONAL; |
| 197 | break; | 171 | break; |
| 198 | 172 | ||
| 199 | default: | 173 | default: |
| 200 | err = -EINVAL; | 174 | err = -EINVAL; |
| 201 | goto unpin; | 175 | goto unpin; |
| 202 | } | 176 | } |
| 203 | 177 | ||
| 204 | err = dma_map_sgtable(dev, sgt, dir, 0); | 178 | map = host1x_bo_pin(dev, bo, direction); |
| 205 | if (err) | 179 | if (IS_ERR(map)) { |
| 206 | goto unpin; | 180 | err = PTR_ERR(map); |
| 181 | goto unpin; | ||
| 182 | } | ||
| 207 | 183 | ||
| 208 | job->unpins[job->num_unpins].dev = dev; | 184 | /* |
| 209 | job->unpins[job->num_unpins].dir = dir; | 185 | * host1x clients are generally not able to do scatter-gather themselves, so fail |
| 210 | phys_addr = sg_dma_address(sgt->sgl); | 186 | * if the buffer is discontiguous and we fail to map its SG table to a single |
| 187 | * contiguous chunk of I/O virtual memory. | ||
| 188 | */ | ||
| 189 | if (map->chunks > 1) { | ||
| 190 | err = -EINVAL; | ||
| 191 | goto unpin; | ||
| 211 | } | 192 | } |
| 212 | 193 | ||
| 213 | job->addr_phys[job->num_unpins] = phys_addr; | 194 | job->addr_phys[job->num_unpins] = map->phys; |
| 214 | job->unpins[job->num_unpins].bo = reloc->target.bo; | 195 | job->unpins[job->num_unpins].map = map; |
| 215 | job->unpins[job->num_unpins].sgt = sgt; | ||
| 216 | job->num_unpins++; | 196 | job->num_unpins++; |
| 217 | } | 197 | } |
| 218 | 198 | ||
| @@ -224,12 +204,11 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 5 unmodified lines | |||
| 224 | return 0; | 204 | return 0; |
| 225 | 205 | ||
| 226 | for (i = 0; i < job->num_cmds; i++) { | 206 | for (i = 0; i < job->num_cmds; i++) { |
| 207 | struct host1x_bo_mapping *map; | ||
| 227 | size_t gather_size = 0; | 208 | size_t gather_size = 0; |
| 228 | struct scatterlist *sg; | 209 | struct scatterlist *sg; |
| 229 | dma_addr_t phys_addr; | ||
| 230 | unsigned long shift; | 210 | unsigned long shift; |
| 231 | struct iova *alloc; | 211 | struct iova *alloc; |
| 232 | dma_addr_t *phys; | ||
| 233 | unsigned int j; | 212 | unsigned int j; |
| 234 | 213 | ||
| 235 | if (job->cmds[i].is_wait) | 214 | if (job->cmds[i].is_wait) |
| @@ -243,25 +222,16 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 7 unmodified lines | |||
| 243 | goto unpin; | 222 | goto unpin; |
| 244 | } | 223 | } |
| 245 | 224 | ||
| 246 | /** | 225 | map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE); |
| 247 | * If the host1x is not attached to an IOMMU, there is no need | 226 | if (IS_ERR(map)) { |
| 248 | * to map the buffer object for the host1x, since the physical | 227 | err = PTR_ERR(map); |
| 249 | * address can simply be used. | 228 | goto unpin; |
| 250 | */ | ||
| 251 | if (!iommu_get_domain_for_dev(host->dev)) | ||
| 252 | phys = &phys_addr; | ||
| 253 | else | ||
| 254 | phys = NULL; | ||
| 255 | |||
| 256 | sgt = host1x_bo_pin(host->dev, g->bo, phys); | ||
| 257 | if (IS_ERR(sgt)) { | ||
| 258 | err = PTR_ERR(sgt); | ||
| 259 | goto put; | ||
| 260 | } | 229 | } |
| 261 | 230 | ||
| 262 | if (host->domain) { | 231 | if (host->domain) { |
| 263 | for_each_sgtable_sg(sgt, sg, j) | 232 | for_each_sgtable_sg(map->sgt, sg, j) |
| 264 | gather_size += sg->length; | 233 | gather_size += sg->length; |
| 234 | |||
| 265 | gather_size = iova_align(&host->iova, gather_size); | 235 | gather_size = iova_align(&host->iova, gather_size); |
| 266 | 236 | ||
| 267 | shift = iova_shift(&host->iova); | 237 | shift = iova_shift(&host->iova); |
| @@ -272,33 +242,23 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) · 4 unmodified lines | |||
| 272 | goto put; | 242 | goto put; |
| 273 | } | 243 | } |
| 274 | 244 | ||
| 275 | err = iommu_map_sgtable(host->domain, | 245 | err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), |
| 276 | iova_dma_addr(&host->iova, alloc), | 246 | map->sgt, IOMMU_READ); |
| 277 | sgt, IOMMU_READ); | ||
| 278 | if (err == 0) { | 247 | if (err == 0) { |
| 279 | __free_iova(&host->iova, alloc); | 248 | __free_iova(&host->iova, alloc); |
| 280 | err = -EINVAL; | 249 | err = -EINVAL; |
| 281 | goto put; | 250 | goto put; |
| 282 | } | 251 | } |
| 283 | 252 | ||
| 284 | job->unpins[job->num_unpins].size = gather_size; | 253 | map->phys = iova_dma_addr(&host->iova, alloc); |
| 285 | phys_addr = iova_dma_addr(&host->iova, alloc); | 254 | map->size = gather_size; |
| 286 | } else if (sgt) { | ||
| 287 | err = dma_map_sgtable(host->dev, sgt, DMA_TO_DEVICE, 0); | ||
| 288 | if (err) | ||
| 289 | goto put; | ||
| 290 | |||
| 291 | job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; | ||
| 292 | job->unpins[job->num_unpins].dev = host->dev; | ||
| 293 | phys_addr = sg_dma_address(sgt->sgl); | ||
| 294 | } | 255 | } |
| 295 | 256 | ||
| 296 | job->addr_phys[job->num_unpins] = phys_addr; | 257 | job->addr_phys[job->num_unpins] = map->phys; |
| 297 | job->gather_addr_phys[i] = phys_addr; | 258 | job->unpins[job->num_unpins].map = map; |
| 298 | |||
| 299 | job->unpins[job->num_unpins].bo = g->bo; | ||
| 300 | job->unpins[job->num_unpins].sgt = sgt; | ||
| 301 | job->num_unpins++; | 259 | job->num_unpins++; |
| 260 | |||
| 261 | job->gather_addr_phys[i] = map->phys; | ||
| 302 | } | 262 | } |
| 303 | 263 | ||
| 304 | return 0; | 264 | return 0; |
| @@ -690,22 +650,16 @@ void host1x_job_unpin(struct host1x_job *job) · 385 unmodified lines | |||
| 690 | unsigned int i; | 650 | unsigned int i; |
| 691 | 651 | ||
| 692 | for (i = 0; i < job->num_unpins; i++) { | 652 | for (i = 0; i < job->num_unpins; i++) { |
| 693 | struct host1x_job_unpin_data *unpin = &job->unpins[i]; | 653 | struct host1x_bo_mapping *map = job->unpins[i].map; |
| 694 | struct device *dev = unpin->dev ?: host->dev; | 654 | struct host1x_bo *bo = map->bo; |
| 695 | struct sg_table *sgt = unpin->sgt; | ||
| 696 | |||
| 697 | if (!job->enable_firewall && unpin->size && host->domain) { | ||
| 698 | iommu_unmap(host->domain, job->addr_phys[i], | ||
| 699 | unpin->size); | ||
| 700 | free_iova(&host->iova, | ||
| 701 | iova_pfn(&host->iova, job->addr_phys[i])); | ||
| 702 | } | ||
| 703 | 655 | ||
| 704 | if (unpin->dev && sgt) | 656 | if (!job->enable_firewall && map->size && host->domain) { |
| 705 | dma_unmap_sgtable(unpin->dev, sgt, unpin->dir, 0); | 657 | iommu_unmap(host->domain, job->addr_phys[i], map->size); |
| 658 | free_iova(&host->iova, iova_pfn(&host->iova, job->addr_phys[i])); | ||
| 659 | } | ||
| 706 | 660 | ||
| 707 | host1x_bo_unpin(dev, unpin->bo, sgt); | 661 | host1x_bo_unpin(map); |
| 708 | host1x_bo_put(unpin->bo); | 662 | host1x_bo_put(bo); |
| 709 | } | 663 | } |
| 710 | 664 | ||
| 711 | job->num_unpins = 0; | 665 | job->num_unpins = 0; |
| @@ -35,11 +35,7 @@ struct host1x_job_cmd { · 34 unmodified lines | |||
| 35 | }; | 35 | }; |
| 36 | 36 | ||
| 37 | struct host1x_job_unpin_data { | 37 | struct host1x_job_unpin_data { |
| 38 | struct host1x_bo *bo; | 38 | struct host1x_bo_mapping *map; |
| 39 | struct sg_table *sgt; | ||
| 40 | struct device *dev; | ||
| 41 | size_t size; | ||
| 42 | enum dma_data_direction dir; | ||
| 43 | }; | 39 | }; |
| 44 | 40 | ||
| 45 | /* | 41 | /* |
| @@ -7,6 +7,7 @@ · 6 unmodified lines | |||
| 7 | #define __LINUX_HOST1X_H | 7 | #define __LINUX_HOST1X_H |
| 8 | 8 | ||
| 9 | #include <linux/device.h> | 9 | #include <linux/device.h> |
| 10 | #include <linux/dma-direction.h> | ||
| 10 | #include <linux/types.h> | 11 | #include <linux/types.h> |
| 11 | 12 | ||
| 12 | enum host1x_class { | 13 | enum host1x_class { |
| @@ -82,12 +83,23 @@ struct host1x_client { · 69 unmodified lines | |||
| 82 | struct host1x_bo; | 83 | struct host1x_bo; |
| 83 | struct sg_table; | 84 | struct sg_table; |
| 84 | 85 | ||
| 86 | struct host1x_bo_mapping { | ||
| 87 | struct dma_buf_attachment *attach; | ||
| 88 | enum dma_data_direction direction; | ||
| 89 | struct host1x_bo *bo; | ||
| 90 | struct sg_table *sgt; | ||
| 91 | unsigned int chunks; | ||
| 92 | struct device *dev; | ||
| 93 | dma_addr_t phys; | ||
| 94 | size_t size; | ||
| 95 | }; | ||
| 96 | |||
| 85 | struct host1x_bo_ops { | 97 | struct host1x_bo_ops { |
| 86 | struct host1x_bo *(*get)(struct host1x_bo *bo); | 98 | struct host1x_bo *(*get)(struct host1x_bo *bo); |
| 87 | void (*put)(struct host1x_bo *bo); | 99 | void (*put)(struct host1x_bo *bo); |
| 88 | struct sg_table *(*pin)(struct device *dev, struct host1x_bo *bo, | 100 | struct host1x_bo_mapping *(*pin)(struct device *dev, struct host1x_bo *bo, |
| 89 | dma_addr_t *phys); | 101 | enum dma_data_direction dir); |
| 90 | void (*unpin)(struct device *dev, struct sg_table *sgt); | 102 | void (*unpin)(struct host1x_bo_mapping *map); |
| 91 | void *(*mmap)(struct host1x_bo *bo); | 103 | void *(*mmap)(struct host1x_bo *bo); |
| 92 | void (*munmap)(struct host1x_bo *bo, void *addr); | 104 | void (*munmap)(struct host1x_bo *bo, void *addr); |
| 93 | }; | 105 | }; |
| @@ -112,17 +124,15 @@ static inline void host1x_bo_put(struct host1x_bo *bo) · 18 unmodified lines | |||
| 112 | bo->ops->put(bo); | 124 | bo->ops->put(bo); |
| 113 | } | 125 | } |
| 114 | 126 | ||
| 115 | static inline struct sg_table *host1x_bo_pin(struct device *dev, | 127 | static inline struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, |
| 116 | struct host1x_bo *bo, | 128 | enum dma_data_direction dir) |
| 117 | dma_addr_t *phys) | ||
| 118 | { | 129 | { |
| 119 | return bo->ops->pin(dev, bo, phys); | 130 | return bo->ops->pin(dev, bo, dir); |
| 120 | } | 131 | } |
| 121 | 132 | ||
| 122 | static inline void host1x_bo_unpin(struct device *dev, struct host1x_bo *bo, | 133 | static inline void host1x_bo_unpin(struct host1x_bo_mapping *map) |
| 123 | struct sg_table *sgt) | ||
| 124 | { | 134 | { |
| 125 | bo->ops->unpin(dev, sgt); | 135 | map->bo->ops->unpin(map); |
| 126 | } | 136 | } |
| 127 | 137 | ||
| 128 | static inline void *host1x_bo_mmap(struct host1x_bo *bo) | 138 | static inline void *host1x_bo_mmap(struct host1x_bo *bo) |