// SPDX-License-Identifier: GPL-2.0+/* * SSH request transport layer. * * Copyright (C) 2019-2022 Maximilian Luz <luzmaximilian@gmail.com> */ #include <linux/unaligned.h>#include <linux/atomic.h>#include <linux/completion.h>#include <linux/error-injection.h>#include <linux/ktime.h>#include <linux/limits.h>#include <linux/list.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/types.h>#include <linux/workqueue.h> #include <linux/surface_aggregator/serial_hub.h>#include <linux/surface_aggregator/controller.h> #include "ssh_packet_layer.h"#include "ssh_request_layer.h" #include "trace.h" /* * SSH_RTL_REQUEST_TIMEOUT - Request timeout. * * Timeout as ktime_t delta for request responses. If we have not received a * response in this time-frame after finishing the underlying packet * transmission, the request will be completed with %-ETIMEDOUT as status * code. */#define SSH_RTL_REQUEST_TIMEOUT ms_to_ktime(3000) /* * SSH_RTL_REQUEST_TIMEOUT_RESOLUTION - Request timeout granularity. * * Time-resolution for timeouts. Should be larger than one jiffy to avoid * direct re-scheduling of reaper work_struct. */#define SSH_RTL_REQUEST_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50)) /* * SSH_RTL_MAX_PENDING - Maximum number of pending requests. * * Maximum number of requests concurrently waiting to be completed (i.e. * waiting for the corresponding packet transmission to finish if they don't * have a response or waiting for a response if they have one). */#define SSH_RTL_MAX_PENDING 3 /* * SSH_RTL_TX_BATCH - Maximum number of requests processed per work execution. * Used to prevent livelocking of the workqueue. Value chosen via educated * guess, may be adjusted. */#define SSH_RTL_TX_BATCH 10 #ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION /** * ssh_rtl_should_drop_response() - Error injection hook to drop request * responses. * * Useful to cause request transmission timeouts in the driver by dropping the * response to a request. */static noinline bool ssh_rtl_should_drop_response(void){ return false;}ALLOW_ERROR_INJECTION(ssh_rtl_should_drop_response, TRUE); #else static inline bool ssh_rtl_should_drop_response(void){ return false;} #endif static u16 ssh_request_get_rqid(struct ssh_request *rqst){ return get_unaligned_le16(rqst->packet.data.ptr + SSH_MSGOFFSET_COMMAND(rqid));} static u32 ssh_request_get_rqid_safe(struct ssh_request *rqst){ if (!rqst->packet.data.ptr) return U32_MAX; return ssh_request_get_rqid(rqst);} static void ssh_rtl_queue_remove(struct ssh_request *rqst){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); spin_lock(&rtl->queue.lock); if (!test_and_clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &rqst->state)) { spin_unlock(&rtl->queue.lock); return; } list_del(&rqst->node); spin_unlock(&rtl->queue.lock); ssh_request_put(rqst);} static bool ssh_rtl_queue_empty(struct ssh_rtl *rtl){ bool empty; spin_lock(&rtl->queue.lock); empty = list_empty(&rtl->queue.head); spin_unlock(&rtl->queue.lock); return empty;} static void ssh_rtl_pending_remove(struct ssh_request *rqst){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); spin_lock(&rtl->pending.lock); if (!test_and_clear_bit(SSH_REQUEST_SF_PENDING_BIT, &rqst->state)) { spin_unlock(&rtl->pending.lock); return; } atomic_dec(&rtl->pending.count); list_del(&rqst->node); spin_unlock(&rtl->pending.lock); ssh_request_put(rqst);} static int ssh_rtl_tx_pending_push(struct ssh_request *rqst){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); spin_lock(&rtl->pending.lock); if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) { spin_unlock(&rtl->pending.lock); return -EINVAL; } if (test_and_set_bit(SSH_REQUEST_SF_PENDING_BIT, &rqst->state)) { spin_unlock(&rtl->pending.lock); return -EALREADY; } atomic_inc(&rtl->pending.count); list_add_tail(&ssh_request_get(rqst)->node, &rtl->pending.head); spin_unlock(&rtl->pending.lock); return 0;} static void ssh_rtl_complete_with_status(struct ssh_request *rqst, int status){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); trace_ssam_request_complete(rqst, status); /* rtl/ptl may not be set if we're canceling before submitting. */ rtl_dbg_cond(rtl, "rtl: completing request (rqid: %#06x, status: %d)\n", ssh_request_get_rqid_safe(rqst), status); rqst->ops->complete(rqst, NULL, NULL, status);} static void ssh_rtl_complete_with_rsp(struct ssh_request *rqst, const struct ssh_command *cmd, const struct ssam_span *data){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); trace_ssam_request_complete(rqst, 0); rtl_dbg(rtl, "rtl: completing request with response (rqid: %#06x)\n", ssh_request_get_rqid(rqst)); rqst->ops->complete(rqst, cmd, data, 0);} static bool ssh_rtl_tx_can_process(struct ssh_request *rqst){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); if (test_bit(SSH_REQUEST_TY_FLUSH_BIT, &rqst->state)) return !atomic_read(&rtl->pending.count); return atomic_read(&rtl->pending.count) < SSH_RTL_MAX_PENDING;} static struct ssh_request *ssh_rtl_tx_next(struct ssh_rtl *rtl){ struct ssh_request *rqst = ERR_PTR(-ENOENT); struct ssh_request *p, *n; spin_lock(&rtl->queue.lock); /* Find first non-locked request and remove it. */ list_for_each_entry_safe(p, n, &rtl->queue.head, node) { if (unlikely(test_bit(SSH_REQUEST_SF_LOCKED_BIT, &p->state))) continue; if (!ssh_rtl_tx_can_process(p)) { rqst = ERR_PTR(-EBUSY); break; } /* Remove from queue and mark as transmitting. */ set_bit(SSH_REQUEST_SF_TRANSMITTING_BIT, &p->state); /* Ensure state never gets zero. */ smp_mb__before_atomic(); clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &p->state); list_del(&p->node); rqst = p; break; } spin_unlock(&rtl->queue.lock); return rqst;} static int ssh_rtl_tx_try_process_one(struct ssh_rtl *rtl){ struct ssh_request *rqst; int status; /* Get and prepare next request for transmit. */ rqst = ssh_rtl_tx_next(rtl); if (IS_ERR(rqst)) return PTR_ERR(rqst); /* Add it to/mark it as pending. */ status = ssh_rtl_tx_pending_push(rqst); if (status) { ssh_request_put(rqst); return -EAGAIN; } /* Submit packet. */ status = ssh_ptl_submit(&rtl->ptl, &rqst->packet); if (status == -ESHUTDOWN) { /* * Packet has been refused due to the packet layer shutting * down. Complete it here. */ set_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state); /* * Note: A barrier is not required here, as there are only two * references in the system at this point: The one that we have, * and the other one that belongs to the pending set. Due to the * request being marked as "transmitting", our process is the * only one allowed to remove the pending node and change the * state. Normally, the task would fall to the packet callback, * but as this is a path where submission failed, this callback * will never be executed. */ ssh_rtl_pending_remove(rqst); ssh_rtl_complete_with_status(rqst, -ESHUTDOWN); ssh_request_put(rqst); return -ESHUTDOWN; } else if (status) { /* * If submitting the packet failed and the packet layer isn't * shutting down, the packet has either been submitted/queued * before (-EALREADY, which cannot happen as we have * guaranteed that requests cannot be re-submitted), or the * packet was marked as locked (-EINVAL). To mark the packet * locked at this stage, the request, and thus the packets * itself, had to have been canceled. Simply drop the * reference. Cancellation itself will remove it from the set * of pending requests. */ WARN_ON(status != -EINVAL); ssh_request_put(rqst); return -EAGAIN; } ssh_request_put(rqst); return 0;} static bool ssh_rtl_tx_schedule(struct ssh_rtl *rtl){ if (atomic_read(&rtl->pending.count) >= SSH_RTL_MAX_PENDING) return false; if (ssh_rtl_queue_empty(rtl)) return false; return schedule_work(&rtl->tx.work);} static void ssh_rtl_tx_work_fn(struct work_struct *work){ struct ssh_rtl *rtl = to_ssh_rtl(work, tx.work); unsigned int iterations = SSH_RTL_TX_BATCH; int status; /* * Try to be nice and not block/live-lock the workqueue: Run a maximum * of 10 tries, then re-submit if necessary. This should not be * necessary for normal execution, but guarantee it anyway. */ do { status = ssh_rtl_tx_try_process_one(rtl); if (status == -ENOENT || status == -EBUSY) return; /* No more requests to process. */ if (status == -ESHUTDOWN) { /* * Packet system shutting down. No new packets can be * transmitted. Return silently, the party initiating * the shutdown should handle the rest. */ return; } WARN_ON(status != 0 && status != -EAGAIN); } while (--iterations); /* Out of tries, reschedule. */ ssh_rtl_tx_schedule(rtl);} /** * ssh_rtl_submit() - Submit a request to the transport layer. * @rtl: The request transport layer. * @rqst: The request to submit. * * Submits a request to the transport layer. A single request may not be * submitted multiple times without reinitializing it. * * Return: Returns zero on success, %-EINVAL if the request type is invalid or * the request has been canceled prior to submission, %-EALREADY if the * request has already been submitted, or %-ESHUTDOWN in case the request * transport layer has been shut down. */int ssh_rtl_submit(struct ssh_rtl *rtl, struct ssh_request *rqst){ trace_ssam_request_submit(rqst); /* * Ensure that requests expecting a response are sequenced. If this * invariant ever changes, see the comment in ssh_rtl_complete() on what * is required to be changed in the code. */ if (test_bit(SSH_REQUEST_TY_HAS_RESPONSE_BIT, &rqst->state)) if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &rqst->packet.state)) return -EINVAL; spin_lock(&rtl->queue.lock); /* * Try to set ptl and check if this request has already been submitted. * * Must be inside lock as we might run into a lost update problem * otherwise: If this were outside of the lock, cancellation in * ssh_rtl_cancel_nonpending() may run after we've set the ptl * reference but before we enter the lock. In that case, we'd detect * that the request is being added to the queue and would try to remove * it from that, but removal might fail because it hasn't actually been * added yet. By putting this cmpxchg in the critical section, we * ensure that the queuing detection only triggers when we are already * in the critical section and the remove process will wait until the * push operation has been completed (via lock) due to that. Only then, * we can safely try to remove it. */ if (cmpxchg(&rqst->packet.ptl, NULL, &rtl->ptl)) { spin_unlock(&rtl->queue.lock); return -EALREADY; } /* * Ensure that we set ptl reference before we continue modifying state. * This is required for non-pending cancellation. This barrier is paired * with the one in ssh_rtl_cancel_nonpending(). * * By setting the ptl reference before we test for "locked", we can * check if the "locked" test may have already run. See comments in * ssh_rtl_cancel_nonpending() for more detail. */ smp_mb__after_atomic(); if (test_bit(SSH_RTL_SF_SHUTDOWN_BIT, &rtl->state)) { spin_unlock(&rtl->queue.lock); return -ESHUTDOWN; } if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) { spin_unlock(&rtl->queue.lock); return -EINVAL; } set_bit(SSH_REQUEST_SF_QUEUED_BIT, &rqst->state); list_add_tail(&ssh_request_get(rqst)->node, &rtl->queue.head); spin_unlock(&rtl->queue.lock); ssh_rtl_tx_schedule(rtl); return 0;} static void ssh_rtl_timeout_reaper_mod(struct ssh_rtl *rtl, ktime_t now, ktime_t expires){ unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now)); ktime_t aexp = ktime_add(expires, SSH_RTL_REQUEST_TIMEOUT_RESOLUTION); spin_lock(&rtl->rtx_timeout.lock); /* Re-adjust / schedule reaper only if it is above resolution delta. */ if (ktime_before(aexp, rtl->rtx_timeout.expires)) { rtl->rtx_timeout.expires = expires; mod_delayed_work(system_percpu_wq, &rtl->rtx_timeout.reaper, delta); } spin_unlock(&rtl->rtx_timeout.lock);} static void ssh_rtl_timeout_start(struct ssh_request *rqst){ struct ssh_rtl *rtl = ssh_request_rtl(rqst); ktime_t timestamp = ktime_get_coarse_boottime(); ktime_t timeout = rtl->rtx_timeout.timeout; if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) return; /* * Note: The timestamp gets set only once. This happens on the packet * callback. All other access to it is read-only. */ WRITE_ONCE(rqst->timestamp, timestamp); /* * Ensure timestamp is set before starting the reaper. Paired with * implicit barrier following check on ssh_request_get_expiration() in * ssh_rtl_timeout_reap. */ smp_mb__after_atomic(); ssh_rtl_timeout_reaper_mod(rtl, timestamp, timestamp + timeout);} static void ssh_rtl_complete(struct ssh_rtl *rtl, const struct ssh_command *command, const struct ssam_span *command_data){ struct ssh_request *r = NULL; struct ssh_request *p, *n; u16 rqid = get_unaligned_le16(&command->rqid); trace_ssam_rx_response_received(command, command_data->len); /* * Get request from pending based on request ID and mark it as response * received and locked. */ spin_lock(&rtl->pending.lock); list_for_each_entry_safe(p, n, &rtl->pending.head, node) { /* We generally expect requests to be processed in order. */ if (unlikely(ssh_request_get_rqid(p) != rqid)) continue; /* Simulate response timeout. */ if (ssh_rtl_should_drop_response()) { spin_unlock(&rtl->pending.lock); trace_ssam_ei_rx_drop_response(p); rtl_info(rtl, "request error injection: dropping response for request %p\n", &p->packet); return; } /* * Mark as "response received" and "locked" as we're going to * complete it. */Lines 1–500 of 1274. Load more
// SPDX-License-Identifier: GPL-2.0+ /* * SSH request transport layer. * * Copyright (C) 2019-2022 Maximilian Luz <luzmaximilian@gmail.com> */ #include <linux/unaligned.h> #include <linux/atomic.h> #include <linux/completion.h> #include <linux/error-injection.h> #include <linux/ktime.h> #include <linux/limits.h> #include <linux/list.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/types.h> #include <linux/workqueue.h> #include <linux/surface_aggregator/serial_hub.h> #include <linux/surface_aggregator/controller.h> #include "ssh_packet_layer.h" #include "ssh_request_layer.h" #include "trace.h" /* * SSH_RTL_REQUEST_TIMEOUT - Request timeout. * * Timeout as ktime_t delta for request responses. If we have not received a * response in this time-frame after finishing the underlying packet * transmission, the request will be completed with %-ETIMEDOUT as status * code. */ #define SSH_RTL_REQUEST_TIMEOUT ms_to_ktime(3000) /* * SSH_RTL_REQUEST_TIMEOUT_RESOLUTION - Request timeout granularity. * * Time-resolution for timeouts. Should be larger than one jiffy to avoid * direct re-scheduling of reaper work_struct. */ #define SSH_RTL_REQUEST_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50)) /* * SSH_RTL_MAX_PENDING - Maximum number of pending requests. * * Maximum number of requests concurrently waiting to be completed (i.e. * waiting for the corresponding packet transmission to finish if they don't * have a response or waiting for a response if they have one). */ #define SSH_RTL_MAX_PENDING 3 /* * SSH_RTL_TX_BATCH - Maximum number of requests processed per work execution. * Used to prevent livelocking of the workqueue. Value chosen via educated * guess, may be adjusted. */ #define SSH_RTL_TX_BATCH 10 #ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION /** * ssh_rtl_should_drop_response() - Error injection hook to drop request * responses. * * Useful to cause request transmission timeouts in the driver by dropping the * response to a request. */ static noinline bool ssh_rtl_should_drop_response(void) { return false; } ALLOW_ERROR_INJECTION(ssh_rtl_should_drop_response, TRUE); #else static inline bool ssh_rtl_should_drop_response(void) { return false; } #endif static u16 ssh_request_get_rqid(struct ssh_request *rqst) { return get_unaligned_le16(rqst->packet.data.ptr + SSH_MSGOFFSET_COMMAND(rqid)); } static u32 ssh_request_get_rqid_safe(struct ssh_request *rqst) { if (!rqst->packet.data.ptr) return U32_MAX; return ssh_request_get_rqid(rqst); } static void ssh_rtl_queue_remove(struct ssh_request *rqst) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); spin_lock(&rtl->queue.lock); if (!test_and_clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &rqst->state)) { spin_unlock(&rtl->queue.lock); return; } list_del(&rqst->node); spin_unlock(&rtl->queue.lock); ssh_request_put(rqst); } static bool ssh_rtl_queue_empty(struct ssh_rtl *rtl) { bool empty; spin_lock(&rtl->queue.lock); empty = list_empty(&rtl->queue.head); spin_unlock(&rtl->queue.lock); return empty; } static void ssh_rtl_pending_remove(struct ssh_request *rqst) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); spin_lock(&rtl->pending.lock); if (!test_and_clear_bit(SSH_REQUEST_SF_PENDING_BIT, &rqst->state)) { spin_unlock(&rtl->pending.lock); return; } atomic_dec(&rtl->pending.count); list_del(&rqst->node); spin_unlock(&rtl->pending.lock); ssh_request_put(rqst); } static int ssh_rtl_tx_pending_push(struct ssh_request *rqst) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); spin_lock(&rtl->pending.lock); if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) { spin_unlock(&rtl->pending.lock); return -EINVAL; } if (test_and_set_bit(SSH_REQUEST_SF_PENDING_BIT, &rqst->state)) { spin_unlock(&rtl->pending.lock); return -EALREADY; } atomic_inc(&rtl->pending.count); list_add_tail(&ssh_request_get(rqst)->node, &rtl->pending.head); spin_unlock(&rtl->pending.lock); return 0; } static void ssh_rtl_complete_with_status(struct ssh_request *rqst, int status) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); trace_ssam_request_complete(rqst, status); /* rtl/ptl may not be set if we're canceling before submitting. */ rtl_dbg_cond(rtl, "rtl: completing request (rqid: %#06x, status: %d)\n", ssh_request_get_rqid_safe(rqst), status); rqst->ops->complete(rqst, NULL, NULL, status); } static void ssh_rtl_complete_with_rsp(struct ssh_request *rqst, const struct ssh_command *cmd, const struct ssam_span *data) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); trace_ssam_request_complete(rqst, 0); rtl_dbg(rtl, "rtl: completing request with response (rqid: %#06x)\n", ssh_request_get_rqid(rqst)); rqst->ops->complete(rqst, cmd, data, 0); } static bool ssh_rtl_tx_can_process(struct ssh_request *rqst) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); if (test_bit(SSH_REQUEST_TY_FLUSH_BIT, &rqst->state)) return !atomic_read(&rtl->pending.count); return atomic_read(&rtl->pending.count) < SSH_RTL_MAX_PENDING; } static struct ssh_request *ssh_rtl_tx_next(struct ssh_rtl *rtl) { struct ssh_request *rqst = ERR_PTR(-ENOENT); struct ssh_request *p, *n; spin_lock(&rtl->queue.lock); /* Find first non-locked request and remove it. */ list_for_each_entry_safe(p, n, &rtl->queue.head, node) { if (unlikely(test_bit(SSH_REQUEST_SF_LOCKED_BIT, &p->state))) continue; if (!ssh_rtl_tx_can_process(p)) { rqst = ERR_PTR(-EBUSY); break; } /* Remove from queue and mark as transmitting. */ set_bit(SSH_REQUEST_SF_TRANSMITTING_BIT, &p->state); /* Ensure state never gets zero. */ smp_mb__before_atomic(); clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &p->state); list_del(&p->node); rqst = p; break; } spin_unlock(&rtl->queue.lock); return rqst; } static int ssh_rtl_tx_try_process_one(struct ssh_rtl *rtl) { struct ssh_request *rqst; int status; /* Get and prepare next request for transmit. */ rqst = ssh_rtl_tx_next(rtl); if (IS_ERR(rqst)) return PTR_ERR(rqst); /* Add it to/mark it as pending. */ status = ssh_rtl_tx_pending_push(rqst); if (status) { ssh_request_put(rqst); return -EAGAIN; } /* Submit packet. */ status = ssh_ptl_submit(&rtl->ptl, &rqst->packet); if (status == -ESHUTDOWN) { /* * Packet has been refused due to the packet layer shutting * down. Complete it here. */ set_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state); /* * Note: A barrier is not required here, as there are only two * references in the system at this point: The one that we have, * and the other one that belongs to the pending set. Due to the * request being marked as "transmitting", our process is the * only one allowed to remove the pending node and change the * state. Normally, the task would fall to the packet callback, * but as this is a path where submission failed, this callback * will never be executed. */ ssh_rtl_pending_remove(rqst); ssh_rtl_complete_with_status(rqst, -ESHUTDOWN); ssh_request_put(rqst); return -ESHUTDOWN; } else if (status) { /* * If submitting the packet failed and the packet layer isn't * shutting down, the packet has either been submitted/queued * before (-EALREADY, which cannot happen as we have * guaranteed that requests cannot be re-submitted), or the * packet was marked as locked (-EINVAL). To mark the packet * locked at this stage, the request, and thus the packets * itself, had to have been canceled. Simply drop the * reference. Cancellation itself will remove it from the set * of pending requests. */ WARN_ON(status != -EINVAL); ssh_request_put(rqst); return -EAGAIN; } ssh_request_put(rqst); return 0; } static bool ssh_rtl_tx_schedule(struct ssh_rtl *rtl) { if (atomic_read(&rtl->pending.count) >= SSH_RTL_MAX_PENDING) return false; if (ssh_rtl_queue_empty(rtl)) return false; return schedule_work(&rtl->tx.work); } static void ssh_rtl_tx_work_fn(struct work_struct *work) { struct ssh_rtl *rtl = to_ssh_rtl(work, tx.work); unsigned int iterations = SSH_RTL_TX_BATCH; int status; /* * Try to be nice and not block/live-lock the workqueue: Run a maximum * of 10 tries, then re-submit if necessary. This should not be * necessary for normal execution, but guarantee it anyway. */ do { status = ssh_rtl_tx_try_process_one(rtl); if (status == -ENOENT || status == -EBUSY) return; /* No more requests to process. */ if (status == -ESHUTDOWN) { /* * Packet system shutting down. No new packets can be * transmitted. Return silently, the party initiating * the shutdown should handle the rest. */ return; } WARN_ON(status != 0 && status != -EAGAIN); } while (--iterations); /* Out of tries, reschedule. */ ssh_rtl_tx_schedule(rtl); } /** * ssh_rtl_submit() - Submit a request to the transport layer. * @rtl: The request transport layer. * @rqst: The request to submit. * * Submits a request to the transport layer. A single request may not be * submitted multiple times without reinitializing it. * * Return: Returns zero on success, %-EINVAL if the request type is invalid or * the request has been canceled prior to submission, %-EALREADY if the * request has already been submitted, or %-ESHUTDOWN in case the request * transport layer has been shut down. */ int ssh_rtl_submit(struct ssh_rtl *rtl, struct ssh_request *rqst) { trace_ssam_request_submit(rqst); /* * Ensure that requests expecting a response are sequenced. If this * invariant ever changes, see the comment in ssh_rtl_complete() on what * is required to be changed in the code. */ if (test_bit(SSH_REQUEST_TY_HAS_RESPONSE_BIT, &rqst->state)) if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &rqst->packet.state)) return -EINVAL; spin_lock(&rtl->queue.lock); /* * Try to set ptl and check if this request has already been submitted. * * Must be inside lock as we might run into a lost update problem * otherwise: If this were outside of the lock, cancellation in * ssh_rtl_cancel_nonpending() may run after we've set the ptl * reference but before we enter the lock. In that case, we'd detect * that the request is being added to the queue and would try to remove * it from that, but removal might fail because it hasn't actually been * added yet. By putting this cmpxchg in the critical section, we * ensure that the queuing detection only triggers when we are already * in the critical section and the remove process will wait until the * push operation has been completed (via lock) due to that. Only then, * we can safely try to remove it. */ if (cmpxchg(&rqst->packet.ptl, NULL, &rtl->ptl)) { spin_unlock(&rtl->queue.lock); return -EALREADY; } /* * Ensure that we set ptl reference before we continue modifying state. * This is required for non-pending cancellation. This barrier is paired * with the one in ssh_rtl_cancel_nonpending(). * * By setting the ptl reference before we test for "locked", we can * check if the "locked" test may have already run. See comments in * ssh_rtl_cancel_nonpending() for more detail. */ smp_mb__after_atomic(); if (test_bit(SSH_RTL_SF_SHUTDOWN_BIT, &rtl->state)) { spin_unlock(&rtl->queue.lock); return -ESHUTDOWN; } if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) { spin_unlock(&rtl->queue.lock); return -EINVAL; } set_bit(SSH_REQUEST_SF_QUEUED_BIT, &rqst->state); list_add_tail(&ssh_request_get(rqst)->node, &rtl->queue.head); spin_unlock(&rtl->queue.lock); ssh_rtl_tx_schedule(rtl); return 0; } static void ssh_rtl_timeout_reaper_mod(struct ssh_rtl *rtl, ktime_t now, ktime_t expires) { unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now)); ktime_t aexp = ktime_add(expires, SSH_RTL_REQUEST_TIMEOUT_RESOLUTION); spin_lock(&rtl->rtx_timeout.lock); /* Re-adjust / schedule reaper only if it is above resolution delta. */ if (ktime_before(aexp, rtl->rtx_timeout.expires)) { rtl->rtx_timeout.expires = expires; mod_delayed_work(system_percpu_wq, &rtl->rtx_timeout.reaper, delta); } spin_unlock(&rtl->rtx_timeout.lock); } static void ssh_rtl_timeout_start(struct ssh_request *rqst) { struct ssh_rtl *rtl = ssh_request_rtl(rqst); ktime_t timestamp = ktime_get_coarse_boottime(); ktime_t timeout = rtl->rtx_timeout.timeout; if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) return; /* * Note: The timestamp gets set only once. This happens on the packet * callback. All other access to it is read-only. */ WRITE_ONCE(rqst->timestamp, timestamp); /* * Ensure timestamp is set before starting the reaper. Paired with * implicit barrier following check on ssh_request_get_expiration() in * ssh_rtl_timeout_reap. */ smp_mb__after_atomic(); ssh_rtl_timeout_reaper_mod(rtl, timestamp, timestamp + timeout); } static void ssh_rtl_complete(struct ssh_rtl *rtl, const struct ssh_command *command, const struct ssam_span *command_data) { struct ssh_request *r = NULL; struct ssh_request *p, *n; u16 rqid = get_unaligned_le16(&command->rqid); trace_ssam_rx_response_received(command, command_data->len); /* * Get request from pending based on request ID and mark it as response * received and locked. */ spin_lock(&rtl->pending.lock); list_for_each_entry_safe(p, n, &rtl->pending.head, node) { /* We generally expect requests to be processed in order. */ if (unlikely(ssh_request_get_rqid(p) != rqid)) continue; /* Simulate response timeout. */ if (ssh_rtl_should_drop_response()) { spin_unlock(&rtl->pending.lock); trace_ssam_ei_rx_drop_response(p); rtl_info(rtl, "request error injection: dropping response for request %p\n", &p->packet); return; } /* * Mark as "response received" and "locked" as we're going to * complete it. */