// SPDX-License-Identifier: GPL-2.0-only/* * Tegra host1x Command DMA * * Copyright (c) 2010-2013, NVIDIA Corporation. */ #include <asm/cacheflush.h>#include <linux/device.h>#include <linux/dma-mapping.h>#include <linux/host1x.h>#include <linux/interrupt.h>#include <linux/kernel.h>#include <linux/kfifo.h>#include <linux/overflow.h>#include <linux/slab.h>#include <trace/events/host1x.h> #include "cdma.h"#include "channel.h"#include "dev.h"#include "debug.h"#include "job.h" /* * push_buffer * * The push buffer is a circular array of words to be fetched by command DMA. * Note that it works slightly differently to the sync queue; fence == pos * means that the push buffer is full, not empty. */ /* * Typically the commands written into the push buffer are a pair of words. We * use slots to represent each of these pairs and to simplify things. Note the * strange number of slots allocated here. 512 slots will fit exactly within a * single memory page. We also need one additional word at the end of the push * buffer for the RESTART opcode that will instruct the CDMA to jump back to * the beginning of the push buffer. With 512 slots, this means that we'll use * 2 memory pages and waste 4092 bytes of the second page that will never be * used. */#define HOST1X_PUSHBUFFER_SLOTS 511 /* * Clean up push buffer resources */static void host1x_pushbuffer_destroy(struct push_buffer *pb){ struct host1x_cdma *cdma = pb_to_cdma(pb); struct host1x *host1x = cdma_to_host1x(cdma); if (!pb->mapped) return; if (host1x->domain) { iommu_unmap(host1x->domain, pb->dma, pb->alloc_size); free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); } dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys); pb->mapped = NULL; pb->phys = 0;} /* * Init push buffer resources */static int host1x_pushbuffer_init(struct push_buffer *pb){ struct host1x_cdma *cdma = pb_to_cdma(pb); struct host1x *host1x = cdma_to_host1x(cdma); struct iova *alloc; u32 size; int err; pb->mapped = NULL; pb->phys = 0; pb->size = HOST1X_PUSHBUFFER_SLOTS * 8; size = pb->size + 4; /* initialize buffer pointers */ pb->fence = pb->size - 8; pb->pos = 0; if (host1x->domain) { unsigned long shift; size = iova_align(&host1x->iova, size); pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, GFP_KERNEL); if (!pb->mapped) return -ENOMEM; shift = iova_shift(&host1x->iova); alloc = alloc_iova(&host1x->iova, size >> shift, host1x->iova_end >> shift, true); if (!alloc) { err = -ENOMEM; goto iommu_free_mem; } pb->dma = iova_dma_addr(&host1x->iova, alloc); err = iommu_map(host1x->domain, pb->dma, pb->phys, size, IOMMU_READ, GFP_KERNEL); if (err) goto iommu_free_iova; } else { pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, GFP_KERNEL); if (!pb->mapped) return -ENOMEM; pb->dma = pb->phys; } pb->alloc_size = size; host1x_hw_pushbuffer_init(host1x, pb); return 0; iommu_free_iova: __free_iova(&host1x->iova, alloc);iommu_free_mem: dma_free_wc(host1x->dev, size, pb->mapped, pb->phys); return err;} /* * Push two words to the push buffer * Caller must ensure push buffer is not full */static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2){ u32 *p = (u32 *)((void *)pb->mapped + pb->pos); WARN_ON(pb->pos == pb->fence); *(p++) = op1; *(p++) = op2; pb->pos += 8; if (pb->pos >= pb->size) pb->pos -= pb->size;} /* * Pop a number of two word slots from the push buffer * Caller must ensure push buffer is not empty */static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots){ /* Advance the next write position */ pb->fence += slots * 8; if (pb->fence >= pb->size) pb->fence -= pb->size;} /* * Return the number of two word slots free in the push buffer */static u32 host1x_pushbuffer_space(struct push_buffer *pb){ unsigned int fence = pb->fence; if (pb->fence < pb->pos) fence += pb->size; return (fence - pb->pos) / 8;} /* * Sleep (if necessary) until the requested event happens * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. * - Returns 1 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer * - Return the amount of space (> 0) * Must be called with the cdma lock held. */unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, enum cdma_event event){ for (;;) { struct push_buffer *pb = &cdma->push_buffer; unsigned int space; switch (event) { case CDMA_EVENT_SYNC_QUEUE_EMPTY: space = list_empty(&cdma->sync_queue) ? 1 : 0; break; case CDMA_EVENT_PUSH_BUFFER_SPACE: space = host1x_pushbuffer_space(pb); break; default: WARN_ON(1); return -EINVAL; } if (space) return space; trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), event); /* If somebody has managed to already start waiting, yield */ if (cdma->event != CDMA_EVENT_NONE) { mutex_unlock(&cdma->lock); schedule(); mutex_lock(&cdma->lock); continue; } cdma->event = event; mutex_unlock(&cdma->lock); wait_for_completion(&cdma->complete); mutex_lock(&cdma->lock); } return 0;} /* * Sleep (if necessary) until the push buffer has enough free space. * * Must be called with the cdma lock held. */static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x, struct host1x_cdma *cdma, unsigned int needed){ while (true) { struct push_buffer *pb = &cdma->push_buffer; unsigned int space; space = host1x_pushbuffer_space(pb); if (space >= needed) break; trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), CDMA_EVENT_PUSH_BUFFER_SPACE); /* If somebody has managed to already start waiting, yield */ if (cdma->event != CDMA_EVENT_NONE) { mutex_unlock(&cdma->lock); schedule(); mutex_lock(&cdma->lock); continue; } cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE; mutex_unlock(&cdma->lock); wait_for_completion(&cdma->complete); mutex_lock(&cdma->lock); } return 0;}/* * Start timer that tracks the time spent by the job. * Must be called with the cdma lock held. */static void cdma_start_timer_locked(struct host1x_cdma *cdma, struct host1x_job *job){ if (cdma->timeout.client) { /* timer already started */ return; } cdma->timeout.client = job->client; cdma->timeout.syncpt = job->syncpt; cdma->timeout.syncpt_val = job->syncpt_end; cdma->timeout.start_ktime = ktime_get(); schedule_delayed_work(&cdma->timeout.wq, msecs_to_jiffies(job->timeout));} /* * Stop timer when a buffer submission completes. * Must be called with the cdma lock held. */static void stop_cdma_timer_locked(struct host1x_cdma *cdma){ cancel_delayed_work(&cdma->timeout.wq); cdma->timeout.client = NULL;} /* * For all sync queue entries that have already finished according to the * current sync point registers: * - unpin & unref their mems * - pop their push buffer slots * - remove them from the sync queue * This is normally called from the host code's worker thread, but can be * called manually if necessary. * Must be called with the cdma lock held. */static void update_cdma_locked(struct host1x_cdma *cdma){ bool signal = false; struct host1x_job *job, *n; /* * Walk the sync queue, reading the sync point registers as necessary, * to consume as many sync queue entries as possible without blocking */ list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { struct host1x_syncpt *sp = job->syncpt; /* Check whether this syncpt has completed, and bail if not */ if (!host1x_syncpt_is_expired(sp, job->syncpt_end) && !job->cancelled) { /* Start timer on next pending syncpt */ if (job->timeout) cdma_start_timer_locked(cdma, job); break; } /* Cancel timeout, when a buffer completes */ if (cdma->timeout.client) stop_cdma_timer_locked(cdma); /* Unpin the memory */ host1x_job_unpin(job); /* Pop push buffer slots */ if (job->num_slots) { struct push_buffer *pb = &cdma->push_buffer; host1x_pushbuffer_pop(pb, job->num_slots); if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) signal = true; } list_del(&job->list); host1x_job_put(job); } if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && list_empty(&cdma->sync_queue)) signal = true; if (signal) { cdma->event = CDMA_EVENT_NONE; complete(&cdma->complete); }} void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, struct device *dev){ struct host1x *host1x = cdma_to_host1x(cdma); u32 restart_addr, syncpt_incrs, syncpt_val; struct host1x_job *job, *next_job = NULL; syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", __func__, syncpt_val); /* * Move the sync_queue read pointer to the first entry that hasn't * completed based on the current HW syncpt value. It's likely there * won't be any (i.e. we're still at the head), but covers the case * where a syncpt incr happens just prior/during the teardown. */ dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", __func__); list_for_each_entry(job, &cdma->sync_queue, list) { if (syncpt_val < job->syncpt_end) { if (!list_is_last(&job->list, &cdma->sync_queue)) next_job = list_next_entry(job, list); goto syncpt_incr; } host1x_job_dump(dev, job); } /* all jobs have been completed */ job = NULL; syncpt_incr: /* * Increment with CPU the remaining syncpts of a partially executed job. * * CDMA will continue execution starting with the next job or will get * into idle state. */ if (next_job) restart_addr = next_job->first_get; else restart_addr = cdma->last_pos; if (!job) goto resume; /* do CPU increments for the remaining syncpts */ if (job->syncpt_recovery) { dev_dbg(dev, "%s: perform CPU incr on pending buffers\n", __func__); /* won't need a timeout when replayed */ job->timeout = 0; syncpt_incrs = wrapping_sub(u32, job->syncpt_end, syncpt_val); dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); host1x_job_dump(dev, job); /* safe to use CPU to incr syncpts */ host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, syncpt_incrs, job->syncpt_end, job->num_slots); dev_dbg(dev, "%s: finished sync_queue modification\n", __func__); } else { struct host1x_job *failed_job = job; host1x_job_dump(dev, job); host1x_syncpt_set_locked(job->syncpt); failed_job->cancelled = true; list_for_each_entry_continue(job, &cdma->sync_queue, list) { unsigned int i; if (job->syncpt != failed_job->syncpt) continue; for (i = 0; i < job->num_slots; i++) { unsigned int slot = (job->first_get/8 + i) % HOST1X_PUSHBUFFER_SLOTS; u32 *mapped = cdma->push_buffer.mapped; /* * Overwrite opcodes with 0 word writes * to offset 0xbad. This does nothing but * has a easily detected signature in debug * traces. * * On systems with MLOCK enforcement enabled, * the above 0 word writes would fall foul of * the enforcement. As such, in the first slot * put a RESTART_W opcode to the beginning * of the next job. We don't use this for older * chips since those only support the RESTART * opcode with inconvenient alignment requirements. */ if (i == 0 && host1x->info->has_wide_gather) { unsigned int next_job = (job->first_get/8 + job->num_slots) % HOST1X_PUSHBUFFER_SLOTS; mapped[2*slot+0] = (0xd << 28) | (next_job * 2); mapped[2*slot+1] = 0x0; } else { mapped[2*slot+0] = 0x1bad0000; mapped[2*slot+1] = 0x1bad0000; } } job->cancelled = true; } wmb(); update_cdma_locked(cdma); } resume: /* roll back DMAGET and start up channel again */ host1x_hw_cdma_resume(host1x, cdma, restart_addr);} static void cdma_update_work(struct work_struct *work){ struct host1x_cdma *cdma = container_of(work, struct host1x_cdma, update_work); mutex_lock(&cdma->lock); update_cdma_locked(cdma); mutex_unlock(&cdma->lock);} Lines 1–500 of 688. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Tegra host1x Command DMA * * Copyright (c) 2010-2013, NVIDIA Corporation. */ #include <asm/cacheflush.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/host1x.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/kfifo.h> #include <linux/overflow.h> #include <linux/slab.h> #include <trace/events/host1x.h> #include "cdma.h" #include "channel.h" #include "dev.h" #include "debug.h" #include "job.h" /* * push_buffer * * The push buffer is a circular array of words to be fetched by command DMA. * Note that it works slightly differently to the sync queue; fence == pos * means that the push buffer is full, not empty. */ /* * Typically the commands written into the push buffer are a pair of words. We * use slots to represent each of these pairs and to simplify things. Note the * strange number of slots allocated here. 512 slots will fit exactly within a * single memory page. We also need one additional word at the end of the push * buffer for the RESTART opcode that will instruct the CDMA to jump back to * the beginning of the push buffer. With 512 slots, this means that we'll use * 2 memory pages and waste 4092 bytes of the second page that will never be * used. */ #define HOST1X_PUSHBUFFER_SLOTS 511 /* * Clean up push buffer resources */ static void host1x_pushbuffer_destroy(struct push_buffer *pb) { struct host1x_cdma *cdma = pb_to_cdma(pb); struct host1x *host1x = cdma_to_host1x(cdma); if (!pb->mapped) return; if (host1x->domain) { iommu_unmap(host1x->domain, pb->dma, pb->alloc_size); free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); } dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys); pb->mapped = NULL; pb->phys = 0; } /* * Init push buffer resources */ static int host1x_pushbuffer_init(struct push_buffer *pb) { struct host1x_cdma *cdma = pb_to_cdma(pb); struct host1x *host1x = cdma_to_host1x(cdma); struct iova *alloc; u32 size; int err; pb->mapped = NULL; pb->phys = 0; pb->size = HOST1X_PUSHBUFFER_SLOTS * 8; size = pb->size + 4; /* initialize buffer pointers */ pb->fence = pb->size - 8; pb->pos = 0; if (host1x->domain) { unsigned long shift; size = iova_align(&host1x->iova, size); pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, GFP_KERNEL); if (!pb->mapped) return -ENOMEM; shift = iova_shift(&host1x->iova); alloc = alloc_iova(&host1x->iova, size >> shift, host1x->iova_end >> shift, true); if (!alloc) { err = -ENOMEM; goto iommu_free_mem; } pb->dma = iova_dma_addr(&host1x->iova, alloc); err = iommu_map(host1x->domain, pb->dma, pb->phys, size, IOMMU_READ, GFP_KERNEL); if (err) goto iommu_free_iova; } else { pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, GFP_KERNEL); if (!pb->mapped) return -ENOMEM; pb->dma = pb->phys; } pb->alloc_size = size; host1x_hw_pushbuffer_init(host1x, pb); return 0; iommu_free_iova: __free_iova(&host1x->iova, alloc); iommu_free_mem: dma_free_wc(host1x->dev, size, pb->mapped, pb->phys); return err; } /* * Push two words to the push buffer * Caller must ensure push buffer is not full */ static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) { u32 *p = (u32 *)((void *)pb->mapped + pb->pos); WARN_ON(pb->pos == pb->fence); *(p++) = op1; *(p++) = op2; pb->pos += 8; if (pb->pos >= pb->size) pb->pos -= pb->size; } /* * Pop a number of two word slots from the push buffer * Caller must ensure push buffer is not empty */ static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) { /* Advance the next write position */ pb->fence += slots * 8; if (pb->fence >= pb->size) pb->fence -= pb->size; } /* * Return the number of two word slots free in the push buffer */ static u32 host1x_pushbuffer_space(struct push_buffer *pb) { unsigned int fence = pb->fence; if (pb->fence < pb->pos) fence += pb->size; return (fence - pb->pos) / 8; } /* * Sleep (if necessary) until the requested event happens * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. * - Returns 1 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer * - Return the amount of space (> 0) * Must be called with the cdma lock held. */ unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, enum cdma_event event) { for (;;) { struct push_buffer *pb = &cdma->push_buffer; unsigned int space; switch (event) { case CDMA_EVENT_SYNC_QUEUE_EMPTY: space = list_empty(&cdma->sync_queue) ? 1 : 0; break; case CDMA_EVENT_PUSH_BUFFER_SPACE: space = host1x_pushbuffer_space(pb); break; default: WARN_ON(1); return -EINVAL; } if (space) return space; trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), event); /* If somebody has managed to already start waiting, yield */ if (cdma->event != CDMA_EVENT_NONE) { mutex_unlock(&cdma->lock); schedule(); mutex_lock(&cdma->lock); continue; } cdma->event = event; mutex_unlock(&cdma->lock); wait_for_completion(&cdma->complete); mutex_lock(&cdma->lock); } return 0; } /* * Sleep (if necessary) until the push buffer has enough free space. * * Must be called with the cdma lock held. */ static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x, struct host1x_cdma *cdma, unsigned int needed) { while (true) { struct push_buffer *pb = &cdma->push_buffer; unsigned int space; space = host1x_pushbuffer_space(pb); if (space >= needed) break; trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), CDMA_EVENT_PUSH_BUFFER_SPACE); /* If somebody has managed to already start waiting, yield */ if (cdma->event != CDMA_EVENT_NONE) { mutex_unlock(&cdma->lock); schedule(); mutex_lock(&cdma->lock); continue; } cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE; mutex_unlock(&cdma->lock); wait_for_completion(&cdma->complete); mutex_lock(&cdma->lock); } return 0; } /* * Start timer that tracks the time spent by the job. * Must be called with the cdma lock held. */ static void cdma_start_timer_locked(struct host1x_cdma *cdma, struct host1x_job *job) { if (cdma->timeout.client) { /* timer already started */ return; } cdma->timeout.client = job->client; cdma->timeout.syncpt = job->syncpt; cdma->timeout.syncpt_val = job->syncpt_end; cdma->timeout.start_ktime = ktime_get(); schedule_delayed_work(&cdma->timeout.wq, msecs_to_jiffies(job->timeout)); } /* * Stop timer when a buffer submission completes. * Must be called with the cdma lock held. */ static void stop_cdma_timer_locked(struct host1x_cdma *cdma) { cancel_delayed_work(&cdma->timeout.wq); cdma->timeout.client = NULL; } /* * For all sync queue entries that have already finished according to the * current sync point registers: * - unpin & unref their mems * - pop their push buffer slots * - remove them from the sync queue * This is normally called from the host code's worker thread, but can be * called manually if necessary. * Must be called with the cdma lock held. */ static void update_cdma_locked(struct host1x_cdma *cdma) { bool signal = false; struct host1x_job *job, *n; /* * Walk the sync queue, reading the sync point registers as necessary, * to consume as many sync queue entries as possible without blocking */ list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { struct host1x_syncpt *sp = job->syncpt; /* Check whether this syncpt has completed, and bail if not */ if (!host1x_syncpt_is_expired(sp, job->syncpt_end) && !job->cancelled) { /* Start timer on next pending syncpt */ if (job->timeout) cdma_start_timer_locked(cdma, job); break; } /* Cancel timeout, when a buffer completes */ if (cdma->timeout.client) stop_cdma_timer_locked(cdma); /* Unpin the memory */ host1x_job_unpin(job); /* Pop push buffer slots */ if (job->num_slots) { struct push_buffer *pb = &cdma->push_buffer; host1x_pushbuffer_pop(pb, job->num_slots); if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) signal = true; } list_del(&job->list); host1x_job_put(job); } if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && list_empty(&cdma->sync_queue)) signal = true; if (signal) { cdma->event = CDMA_EVENT_NONE; complete(&cdma->complete); } } void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, struct device *dev) { struct host1x *host1x = cdma_to_host1x(cdma); u32 restart_addr, syncpt_incrs, syncpt_val; struct host1x_job *job, *next_job = NULL; syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", __func__, syncpt_val); /* * Move the sync_queue read pointer to the first entry that hasn't * completed based on the current HW syncpt value. It's likely there * won't be any (i.e. we're still at the head), but covers the case * where a syncpt incr happens just prior/during the teardown. */ dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", __func__); list_for_each_entry(job, &cdma->sync_queue, list) { if (syncpt_val < job->syncpt_end) { if (!list_is_last(&job->list, &cdma->sync_queue)) next_job = list_next_entry(job, list); goto syncpt_incr; } host1x_job_dump(dev, job); } /* all jobs have been completed */ job = NULL; syncpt_incr: /* * Increment with CPU the remaining syncpts of a partially executed job. * * CDMA will continue execution starting with the next job or will get * into idle state. */ if (next_job) restart_addr = next_job->first_get; else restart_addr = cdma->last_pos; if (!job) goto resume; /* do CPU increments for the remaining syncpts */ if (job->syncpt_recovery) { dev_dbg(dev, "%s: perform CPU incr on pending buffers\n", __func__); /* won't need a timeout when replayed */ job->timeout = 0; syncpt_incrs = wrapping_sub(u32, job->syncpt_end, syncpt_val); dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); host1x_job_dump(dev, job); /* safe to use CPU to incr syncpts */ host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, syncpt_incrs, job->syncpt_end, job->num_slots); dev_dbg(dev, "%s: finished sync_queue modification\n", __func__); } else { struct host1x_job *failed_job = job; host1x_job_dump(dev, job); host1x_syncpt_set_locked(job->syncpt); failed_job->cancelled = true; list_for_each_entry_continue(job, &cdma->sync_queue, list) { unsigned int i; if (job->syncpt != failed_job->syncpt) continue; for (i = 0; i < job->num_slots; i++) { unsigned int slot = (job->first_get/8 + i) % HOST1X_PUSHBUFFER_SLOTS; u32 *mapped = cdma->push_buffer.mapped; /* * Overwrite opcodes with 0 word writes * to offset 0xbad. This does nothing but * has a easily detected signature in debug * traces. * * On systems with MLOCK enforcement enabled, * the above 0 word writes would fall foul of * the enforcement. As such, in the first slot * put a RESTART_W opcode to the beginning * of the next job. We don't use this for older * chips since those only support the RESTART * opcode with inconvenient alignment requirements. */ if (i == 0 && host1x->info->has_wide_gather) { unsigned int next_job = (job->first_get/8 + job->num_slots) % HOST1X_PUSHBUFFER_SLOTS; mapped[2*slot+0] = (0xd << 28) | (next_job * 2); mapped[2*slot+1] = 0x0; } else { mapped[2*slot+0] = 0x1bad0000; mapped[2*slot+1] = 0x1bad0000; } } job->cancelled = true; } wmb(); update_cdma_locked(cdma); } resume: /* roll back DMAGET and start up channel again */ host1x_hw_cdma_resume(host1x, cdma, restart_addr); } static void cdma_update_work(struct work_struct *work) { struct host1x_cdma *cdma = container_of(work, struct host1x_cdma, update_work); mutex_lock(&cdma->lock); update_cdma_locked(cdma); mutex_unlock(&cdma->lock); }