// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2013 Red Hat * Author: Rob Clark <robdclark@gmail.com> */ #include "drm/drm_drv.h" #include "msm_gpu.h"#include "msm_gem.h"#include "msm_mmu.h"#include "msm_fence.h"#include "msm_gpu_trace.h"//#include "adreno/adreno_gpu.h" #include <linux/string_helpers.h>#include <linux/devcoredump.h>#include <linux/sched/task.h>#include <linux/sched/mm.h>#include <linux/utsname.h> /* * Power Management: */ static int enable_pwrrail(struct msm_gpu *gpu){	struct drm_device *dev = gpu->dev;	int ret = 0; 	if (gpu->gpu_reg) {		ret = regulator_enable(gpu->gpu_reg);		if (ret) {			DRM_DEV_ERROR(dev->dev, "failed to enable 'gpu_reg': %d\n", ret);			return ret;		}	} 	if (gpu->gpu_cx) {		ret = regulator_enable(gpu->gpu_cx);		if (ret) {			DRM_DEV_ERROR(dev->dev, "failed to enable 'gpu_cx': %d\n", ret);			return ret;		}	} 	return 0;} static int disable_pwrrail(struct msm_gpu *gpu){	if (gpu->gpu_cx)		regulator_disable(gpu->gpu_cx);	if (gpu->gpu_reg)		regulator_disable(gpu->gpu_reg);	return 0;} static int enable_clk(struct msm_gpu *gpu){	if (gpu->core_clk && gpu->fast_rate)		dev_pm_opp_set_rate(&gpu->pdev->dev, gpu->fast_rate); 	/* Set the RBBM timer rate to 19.2Mhz */	if (gpu->rbbmtimer_clk)		clk_set_rate(gpu->rbbmtimer_clk, 19200000); 	return clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks);} static int disable_clk(struct msm_gpu *gpu){	clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); 	/*	 * Set the clock to a deliberately low rate. On older targets the clock	 * speed had to be non zero to avoid problems. On newer targets this	 * will be rounded down to zero anyway so it all works out.	 */	if (gpu->core_clk)		dev_pm_opp_set_rate(&gpu->pdev->dev, 27000000); 	if (gpu->rbbmtimer_clk)		clk_set_rate(gpu->rbbmtimer_clk, 0); 	return 0;} static int enable_axi(struct msm_gpu *gpu){	return clk_prepare_enable(gpu->ebi1_clk);} static int disable_axi(struct msm_gpu *gpu){	clk_disable_unprepare(gpu->ebi1_clk);	return 0;} int msm_gpu_pm_resume(struct msm_gpu *gpu){	int ret; 	DBG("%s", gpu->name);	trace_msm_gpu_resume(0); 	ret = enable_pwrrail(gpu);	if (ret)		return ret; 	ret = enable_clk(gpu);	if (ret)		return ret; 	ret = enable_axi(gpu);	if (ret)		return ret; 	msm_devfreq_resume(gpu); 	gpu->needs_hw_init = true; 	return 0;} int msm_gpu_pm_suspend(struct msm_gpu *gpu){	int ret; 	DBG("%s", gpu->name);	trace_msm_gpu_suspend(0); 	msm_devfreq_suspend(gpu); 	ret = disable_axi(gpu);	if (ret)		return ret; 	ret = disable_clk(gpu);	if (ret)		return ret; 	ret = disable_pwrrail(gpu);	if (ret)		return ret; 	gpu->suspend_count++; 	return 0;} void msm_gpu_show_fdinfo(struct msm_gpu *gpu, struct msm_context *ctx,			 struct drm_printer *p){	drm_printf(p, "drm-engine-gpu:\t%llu ns\n", ctx->elapsed_ns);	drm_printf(p, "drm-cycles-gpu:\t%llu\n", ctx->cycles);	drm_printf(p, "drm-maxfreq-gpu:\t%u Hz\n", gpu->fast_rate);} int msm_gpu_hw_init(struct msm_gpu *gpu){	int ret; 	WARN_ON(!mutex_is_locked(&gpu->lock)); 	if (!gpu->needs_hw_init)		return 0; 	disable_irq(gpu->irq);	ret = gpu->funcs->hw_init(gpu);	if (!ret)		gpu->needs_hw_init = false;	enable_irq(gpu->irq); 	return ret;} #ifdef CONFIG_DEV_COREDUMPstatic ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset,		size_t count, void *data, size_t datalen){	struct msm_gpu *gpu = data;	struct drm_print_iterator iter;	struct drm_printer p;	struct msm_gpu_state *state; 	state = msm_gpu_crashstate_get(gpu);	if (!state)		return 0; 	iter.data = buffer;	iter.offset = 0;	iter.start = offset;	iter.remain = count; 	p = drm_coredump_printer(&iter); 	drm_printf(&p, "---\n");	drm_printf(&p, "kernel: %s\n", init_utsname()->release);	drm_printf(&p, "module: " KBUILD_MODNAME "\n");	drm_printf(&p, "time: %ptSp\n", &state->time);	if (state->comm)		drm_printf(&p, "comm: %s\n", state->comm);	if (state->cmd)		drm_printf(&p, "cmdline: %s\n", state->cmd); 	gpu->funcs->show(gpu, state, &p); 	msm_gpu_crashstate_put(gpu); 	return count - iter.remain;} static void msm_gpu_devcoredump_free(void *data){	struct msm_gpu *gpu = data; 	msm_gpu_crashstate_put(gpu);} static void msm_gpu_crashstate_get_bo(struct msm_gpu_state *state,				      struct drm_gem_object *obj, u64 iova,				      bool full, size_t offset, size_t size){	struct msm_gpu_state_bo *state_bo = &state->bos[state->nr_bos];	struct msm_gem_object *msm_obj = to_msm_bo(obj); 	/* Don't record write only objects */	state_bo->size = size;	state_bo->flags = msm_obj->flags;	state_bo->iova = iova; 	BUILD_BUG_ON(sizeof(state_bo->name) != sizeof(msm_obj->name)); 	memcpy(state_bo->name, msm_obj->name, sizeof(state_bo->name)); 	if (full) {		void *ptr; 		state_bo->data = kvmalloc(size, GFP_KERNEL);		if (!state_bo->data)			goto out; 		ptr = msm_gem_get_vaddr_active(obj);		if (IS_ERR(ptr)) {			kvfree(state_bo->data);			state_bo->data = NULL;			goto out;		} 		memcpy(state_bo->data, ptr + offset, size);		msm_gem_put_vaddr_locked(obj);	}out:	state->nr_bos++;} static void crashstate_get_bos(struct msm_gpu_state *state, struct msm_gem_submit *submit){	extern bool rd_full; 	if (msm_context_is_vmbind(submit->queue->ctx)) {		struct drm_exec exec;		struct drm_gpuva *vma;		unsigned cnt = 0; 		drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);		drm_exec_until_all_locked(&exec) {			cnt = 0; 			drm_exec_lock_obj(&exec, drm_gpuvm_resv_obj(submit->vm));			drm_exec_retry_on_contention(&exec); 			drm_gpuvm_for_each_va (vma, submit->vm) {				if (!vma->gem.obj)					continue; 				cnt++;				drm_exec_lock_obj(&exec, vma->gem.obj);				drm_exec_retry_on_contention(&exec);			} 		} 		drm_gpuvm_for_each_va (vma, submit->vm)			cnt++; 		state->bos = kcalloc(cnt, sizeof(struct msm_gpu_state_bo), GFP_KERNEL); 		if (state->bos)			drm_gpuvm_for_each_va(vma, submit->vm) {				bool dump = rd_full || (vma->flags & MSM_VMA_DUMP); 				/* Skip MAP_NULL/PRR VMAs: */				if (!vma->gem.obj)					continue; 				msm_gpu_crashstate_get_bo(state, vma->gem.obj, vma->va.addr,							  dump, vma->gem.offset, vma->va.range);			} 		drm_exec_fini(&exec);	} else {		state->bos = kcalloc(submit->nr_bos,			sizeof(struct msm_gpu_state_bo), GFP_KERNEL); 		for (int i = 0; state->bos && i < submit->nr_bos; i++) {			struct drm_gem_object *obj = submit->bos[i].obj;			bool dump = rd_full || (submit->bos[i].flags & MSM_SUBMIT_BO_DUMP); 			msm_gem_lock(obj);			msm_gpu_crashstate_get_bo(state, obj, submit->bos[i].iova,						  dump, 0, obj->size);			msm_gem_unlock(obj);		}	}} static void crashstate_get_vm_logs(struct msm_gpu_state *state, struct msm_gem_vm *vm){	uint32_t vm_log_len = (1 << vm->log_shift);	uint32_t vm_log_mask = vm_log_len - 1;	int first; 	/* Bail if no log, or empty log: */	if (!vm->log || !vm->log[0].op)		return; 	mutex_lock(&vm->mmu_lock); 	/*	 * log_idx is the next entry to overwrite, meaning it is the oldest, or	 * first, entry (other than the special case handled below where the	 * log hasn't wrapped around yet)	 */	first = vm->log_idx; 	if (!vm->log[first].op) {		/*		 * If the next log entry has not been written yet, then only		 * entries 0 to idx-1 are valid (ie. we haven't wrapped around		 * yet)		 */		state->nr_vm_logs = MAX(0, first - 1);		first = 0;	} else {		state->nr_vm_logs = vm_log_len;	} 	state->vm_logs = kmalloc_objs(vm->log[0], state->nr_vm_logs);	if (!state->vm_logs) {		state->nr_vm_logs = 0;	} 	for (int i = 0; i < state->nr_vm_logs; i++) {		int idx = (i + first) & vm_log_mask; 		state->vm_logs[i] = vm->log[idx];	} 	mutex_unlock(&vm->mmu_lock);} static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,		struct msm_gem_submit *submit, struct msm_gpu_fault_info *fault_info,		char *comm, char *cmd){	struct msm_gpu_state *state; 	/* Check if the target supports capturing crash state */	if (!gpu->funcs->gpu_state_get)		return; 	/* Only save one crash state at a time */	if (gpu->crashstate)		return; 	state = gpu->funcs->gpu_state_get(gpu);	if (IS_ERR_OR_NULL(state))		return; 	/* Fill in the additional crash state information */	state->comm = kstrdup(comm, GFP_KERNEL);	state->cmd = kstrdup(cmd, GFP_KERNEL);	if (fault_info)		state->fault_info = *fault_info; 	if (submit && state->fault_info.ttbr0) {		struct msm_gpu_fault_info *info = &state->fault_info;		struct msm_mmu *mmu = to_msm_vm(submit->vm)->mmu; 		msm_iommu_pagetable_params(mmu, &info->pgtbl_ttbr0,					   &info->asid);		msm_iommu_pagetable_walk(mmu, info->iova, info->ptes);	} 	if (submit) {		crashstate_get_vm_logs(state, to_msm_vm(submit->vm));		crashstate_get_bos(state, submit);	} 	/* Set the active crash state to be dumped on failure */	gpu->crashstate = state; 	dev_coredumpm(&gpu->pdev->dev, THIS_MODULE, gpu, 0, GFP_KERNEL,		msm_gpu_devcoredump_read, msm_gpu_devcoredump_free);}#elsestatic void msm_gpu_crashstate_capture(struct msm_gpu *gpu,		struct msm_gem_submit *submit, struct msm_gpu_fault_info *fault_info,		char *comm, char *cmd){}#endif /* * Hangcheck detection for locked gpu: */ static struct msm_gem_submit *find_submit(struct msm_ringbuffer *ring, uint32_t fence){	struct msm_gem_submit *submit;	unsigned long flags; 	spin_lock_irqsave(&ring->submit_lock, flags);	list_for_each_entry(submit, &ring->submits, node) {		if (submit->seqno == fence) {			spin_unlock_irqrestore(&ring->submit_lock, flags);			return submit;		}	}	spin_unlock_irqrestore(&ring->submit_lock, flags); 	return NULL;} static void retire_submits(struct msm_gpu *gpu); static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd){	struct msm_context *ctx = submit->queue->ctx;	struct task_struct *task; 	WARN_ON(!mutex_is_locked(&submit->gpu->lock)); 	/* Note that kstrdup will return NULL if argument is NULL: */	*comm = kstrdup(ctx->comm, GFP_KERNEL);	*cmd  = kstrdup(ctx->cmdline, GFP_KERNEL); 	task = get_pid_task(submit->pid, PIDTYPE_PID);	if (!task)		return; 	if (!*comm)		*comm = kstrdup(task->comm, GFP_KERNEL); 	if (!*cmd)		*cmd = kstrdup_quotable_cmdline(task, GFP_KERNEL); 	put_task_struct(task);} static void recover_worker(struct kthread_work *work){	struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);	struct drm_device *dev = gpu->dev;	struct msm_drm_private *priv = dev->dev_private;	struct msm_gem_submit *submit;	struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);	char *comm = NULL, *cmd = NULL;	unsigned int noreclaim_flag;	struct task_struct *task;	int i; 	mutex_lock(&gpu->lock); 	DRM_DEV_ERROR(dev->dev, "%s: hangcheck recover!\n", gpu->name); 	submit = find_submit(cur_ring, cur_ring->memptrs->fence + 1); 	/*	 * If the submit retired while we were waiting for the worker to run,	 * or waiting to acquire the gpu lock, then nothing more to do.	 */	if (!submit)		goto out_unlock; 	/* Increment the fault counts */	submit->queue->faults++; 	task = get_pid_task(submit->pid, PIDTYPE_PID);	if (!task)		gpu->global_faults++;	else {		struct msm_gem_vm *vm = to_msm_vm(submit->vm); 		vm->faults++; 		/*

Lines 1–500 of 1069. Load more