// SPDX-License-Identifier: GPL-2.0-only/* * Copyright 2016 Broadcom */ #include <linux/err.h>#include <linux/module.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/kernel.h>#include <linux/interrupt.h>#include <linux/platform_device.h>#include <linux/scatterlist.h>#include <linux/crypto.h>#include <linux/kthread.h>#include <linux/rtnetlink.h>#include <linux/sched.h>#include <linux/string_choices.h>#include <linux/of.h>#include <linux/io.h>#include <linux/bitops.h> #include <crypto/algapi.h>#include <crypto/aead.h>#include <crypto/internal/aead.h>#include <crypto/aes.h>#include <crypto/internal/des.h>#include <crypto/hmac.h>#include <crypto/md5.h>#include <crypto/authenc.h>#include <crypto/skcipher.h>#include <crypto/hash.h>#include <crypto/sha1.h>#include <crypto/sha2.h>#include <crypto/sha3.h> #include "util.h"#include "cipher.h"#include "spu.h"#include "spum.h"#include "spu2.h" /* ================= Device Structure ================== */ struct bcm_device_private iproc_priv; /* ==================== Parameters ===================== */ int flow_debug_logging;module_param(flow_debug_logging, int, 0644);MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging"); int packet_debug_logging;module_param(packet_debug_logging, int, 0644);MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging"); int debug_logging_sleep;module_param(debug_logging_sleep, int, 0644);MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep"); /* * The value of these module parameters is used to set the priority for each * algo type when this driver registers algos with the kernel crypto API. * To use a priority other than the default, set the priority in the insmod or * modprobe. Changing the module priority after init time has no effect. * * The default priorities are chosen to be lower (less preferred) than ARMv8 CE * algos, but more preferred than generic software algos. */static int cipher_pri = 150;module_param(cipher_pri, int, 0644);MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos"); static int hash_pri = 100;module_param(hash_pri, int, 0644);MODULE_PARM_DESC(hash_pri, "Priority for hash algos"); static int aead_pri = 150;module_param(aead_pri, int, 0644);MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); /* A type 3 BCM header, expected to precede the SPU header for SPU-M. * Bits 3 and 4 in the first byte encode the channel number (the dma ringset). * 0x60 - ring 0 * 0x68 - ring 1 * 0x70 - ring 2 * 0x78 - ring 3 */static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };/* * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN * is set dynamically after reading SPU type from device tree. */#define BCM_HDR_LEN iproc_priv.bcm_hdr_len /* min and max time to sleep before retrying when mbox queue is full. usec */#define MBOX_SLEEP_MIN 800#define MBOX_SLEEP_MAX 1000 /** * select_channel() - Select a SPU channel to handle a crypto request. Selects * channel in round robin order. * * Return: channel index */static u8 select_channel(void){ u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan); return chan_idx % iproc_priv.spu.num_chan;} /** * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to * receive a SPU response message for an skcipher request. Includes buffers to * catch SPU message headers and the response data. * @mssg: mailbox message containing the receive sg * @rctx: crypto request context * @rx_frag_num: number of scatterlist elements required to hold the * SPU response message * @chunksize: Number of bytes of response data expected * @stat_pad_len: Number of bytes required to pad the STAT field to * a 4-byte boundary * * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() * when the request completes, whether the request is handled successfully or * there is an error. * * Returns: * 0 if successful * < 0 if an error */static intspu_skcipher_rx_sg_create(struct brcm_message *mssg, struct iproc_reqctx_s *rctx, u8 rx_frag_num, unsigned int chunksize, u32 stat_pad_len){ struct spu_hw *spu = &iproc_priv.spu; struct scatterlist *sg; /* used to build sgs in mbox message */ struct iproc_ctx_s *ctx = rctx->ctx; u32 datalen; /* Number of bytes of response data expected */ mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp); if (!mssg->spu.dst) return -ENOMEM; sg = mssg->spu.dst; sg_init_table(sg, rx_frag_num); /* Space for SPU message header */ sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); /* If XTS tweak in payload, add buffer to receive encrypted tweak */ if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_XTS_TWEAK_SIZE); /* Copy in each dst sg entry from request, up to chunksize */ datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, rctx->dst_nents, chunksize); if (datalen < chunksize) { pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u", __func__, chunksize, datalen); return -EFAULT; } if (stat_pad_len) sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); return 0;} /** * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to * send a SPU request message for an skcipher request. Includes SPU message * headers and the request data. * @mssg: mailbox message containing the transmit sg * @rctx: crypto request context * @tx_frag_num: number of scatterlist elements required to construct the * SPU request message * @chunksize: Number of bytes of request data * @pad_len: Number of pad bytes * * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() * when the request completes, whether the request is handled successfully or * there is an error. * * Returns: * 0 if successful * < 0 if an error */static intspu_skcipher_tx_sg_create(struct brcm_message *mssg, struct iproc_reqctx_s *rctx, u8 tx_frag_num, unsigned int chunksize, u32 pad_len){ struct spu_hw *spu = &iproc_priv.spu; struct scatterlist *sg; /* used to build sgs in mbox message */ struct iproc_ctx_s *ctx = rctx->ctx; u32 datalen; /* Number of bytes of response data expected */ u32 stat_len; mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp); if (unlikely(!mssg->spu.src)) return -ENOMEM; sg = mssg->spu.src; sg_init_table(sg, tx_frag_num); sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, BCM_HDR_LEN + ctx->spu_req_hdr_len); /* if XTS tweak in payload, copy from IV (where crypto API puts it) */ if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE); /* Copy in each src sg entry from request, up to chunksize */ datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, rctx->src_nents, chunksize); if (unlikely(datalen < chunksize)) { pr_err("%s(): failed to copy src sg to mbox msg", __func__); return -EFAULT; } if (pad_len) sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); stat_len = spu->spu_tx_status_len(); if (stat_len) { memset(rctx->msg_buf.tx_stat, 0, stat_len); sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); } return 0;} static int mailbox_send_message(struct brcm_message *mssg, u32 flags, u8 chan_idx){ int err; int retry_cnt = 0; struct device *dev = &(iproc_priv.pdev->dev); err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) { while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { /* * Mailbox queue is full. Since MAY_SLEEP is set, assume * not in atomic context and we can wait and try again. */ retry_cnt++; usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); atomic_inc(&iproc_priv.mb_no_spc); } } if (err < 0) { atomic_inc(&iproc_priv.mb_send_fail); return err; } /* Check error returned by mailbox controller */ err = mssg->error; if (unlikely(err < 0)) { dev_err(dev, "message error %d", err); /* Signal txdone for mailbox channel */ } /* Signal txdone for mailbox channel */ mbox_client_txdone(iproc_priv.mbox[chan_idx], err); return err;} /** * handle_skcipher_req() - Submit as much of a block cipher request as fits in * a single SPU request message, starting at the current position in the request * data. * @rctx: Crypto request context * * This may be called on the crypto API thread, or, when a request is so large * it must be broken into multiple SPU messages, on the thread used to invoke * the response callback. When requests are broken into multiple SPU * messages, we assume subsequent messages depend on previous results, and * thus always wait for previous results before submitting the next message. * Because requests are submitted in lock step like this, there is no need * to synchronize access to request data structures. * * Return: -EINPROGRESS: request has been accepted and result will be returned * asynchronously * Any other value indicates an error */static int handle_skcipher_req(struct iproc_reqctx_s *rctx){ struct spu_hw *spu = &iproc_priv.spu; struct crypto_async_request *areq = rctx->parent; struct skcipher_request *req = container_of(areq, struct skcipher_request, base); struct iproc_ctx_s *ctx = rctx->ctx; struct spu_cipher_parms cipher_parms; int err; unsigned int chunksize; /* Num bytes of request to submit */ int remaining; /* Bytes of request still to process */ int chunk_start; /* Beginning of data for current SPU msg */ /* IV or ctr value to use in this SPU msg */ u8 local_iv_ctr[MAX_IV_SIZE]; u32 stat_pad_len; /* num bytes to align status field */ u32 pad_len; /* total length of all padding */ struct brcm_message *mssg; /* mailbox message */ /* number of entries in src and dst sg in mailbox message. */ u8 rx_frag_num = 2; /* response header and STATUS */ u8 tx_frag_num = 1; /* request header */ flow_log("%s\n", __func__); cipher_parms.alg = ctx->cipher.alg; cipher_parms.mode = ctx->cipher.mode; cipher_parms.type = ctx->cipher_type; cipher_parms.key_len = ctx->enckeylen; cipher_parms.key_buf = ctx->enckey; cipher_parms.iv_buf = local_iv_ctr; cipher_parms.iv_len = rctx->iv_ctr_len; mssg = &rctx->mb_mssg; chunk_start = rctx->src_sent; remaining = rctx->total_todo - chunk_start; /* determine the chunk we are breaking off and update the indexes */ if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) && (remaining > ctx->max_payload)) chunksize = ctx->max_payload; else chunksize = remaining; rctx->src_sent += chunksize; rctx->total_sent = rctx->src_sent; /* Count number of sg entries to be included in this request */ rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); if ((ctx->cipher.mode == CIPHER_MODE_CBC) && rctx->is_encrypt && chunk_start) /* * Encrypting non-first first chunk. Copy last block of * previous result to IV for this chunk. */ sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr, rctx->iv_ctr_len, chunk_start - rctx->iv_ctr_len); if (rctx->iv_ctr_len) { /* get our local copy of the iv */ __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr, rctx->iv_ctr_len); /* generate the next IV if possible */ if ((ctx->cipher.mode == CIPHER_MODE_CBC) && !rctx->is_encrypt) { /* * CBC Decrypt: next IV is the last ciphertext block in * this chunk */ sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr, rctx->iv_ctr_len, rctx->src_sent - rctx->iv_ctr_len); } else if (ctx->cipher.mode == CIPHER_MODE_CTR) { /* * The SPU hardware increments the counter once for * each AES block of 16 bytes. So update the counter * for the next chunk, if there is one. Note that for * this chunk, the counter has already been copied to * local_iv_ctr. We can assume a block size of 16, * because we only support CTR mode for AES, not for * any other cipher alg. */ add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4); } } if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) flow_log("max_payload infinite\n"); else flow_log("max_payload %u\n", ctx->max_payload); flow_log("sent:%u start:%u remains:%u size:%u\n", rctx->src_sent, chunk_start, remaining, chunksize); /* Copy SPU header template created at setkey time */ memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, sizeof(rctx->msg_buf.bcm_spu_req_hdr)); spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, ctx->spu_req_hdr_len, !(rctx->is_encrypt), &cipher_parms, chunksize); atomic64_add(chunksize, &iproc_priv.bytes_out); stat_pad_len = spu->spu_wordalign_padlen(chunksize); if (stat_pad_len) rx_frag_num++; pad_len = stat_pad_len; if (pad_len) { tx_frag_num++; spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0, 0, ctx->auth.alg, ctx->auth.mode, rctx->total_sent, stat_pad_len); } spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, ctx->spu_req_hdr_len); packet_log("payload:\n"); dump_sg(rctx->src_sg, rctx->src_skip, chunksize); packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); /* * Build mailbox message containing SPU request msg and rx buffers * to catch response message */ memset(mssg, 0, sizeof(*mssg)); mssg->type = BRCM_MESSAGE_SPU; mssg->ctx = rctx; /* Will be returned in response */ /* Create rx scatterlist to catch result */ rx_frag_num += rctx->dst_nents; if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) rx_frag_num++; /* extra sg to insert tweak */ err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize, stat_pad_len); if (err) return err; /* Create tx scatterlist containing SPU request message */ tx_frag_num += rctx->src_nents; if (spu->spu_tx_status_len()) tx_frag_num++; if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) tx_frag_num++; /* extra sg to insert tweak */ err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize, pad_len); if (err) return err; err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); if (unlikely(err < 0)) return err; return -EINPROGRESS;} /** * handle_skcipher_resp() - Process a block cipher SPU response. Updates the * total received count for the request and updates global stats. * @rctx: Crypto request context */static void handle_skcipher_resp(struct iproc_reqctx_s *rctx){ struct spu_hw *spu = &iproc_priv.spu; struct crypto_async_request *areq = rctx->parent; struct skcipher_request *req = skcipher_request_cast(areq); struct iproc_ctx_s *ctx = rctx->ctx; u32 payload_len; /* See how much data was returned */ payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); /* * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the * encrypted tweak ("i") value; we don't count those. */ if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload() && (payload_len >= SPU_XTS_TWEAK_SIZE)) payload_len -= SPU_XTS_TWEAK_SIZE; atomic64_add(payload_len, &iproc_priv.bytes_in); flow_log("%s() offset: %u, bd_len: %u BD:\n", __func__, rctx->total_received, payload_len); dump_sg(req->dst, rctx->total_received, payload_len); rctx->total_received += payload_len; if (rctx->total_received == rctx->total_todo) { atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]); atomic_inc( &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);Lines 1–500 of 4709. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2016 Broadcom */ #include <linux/err.h> #include <linux/module.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/kernel.h> #include <linux/interrupt.h> #include <linux/platform_device.h> #include <linux/scatterlist.h> #include <linux/crypto.h> #include <linux/kthread.h> #include <linux/rtnetlink.h> #include <linux/sched.h> #include <linux/string_choices.h> #include <linux/of.h> #include <linux/io.h> #include <linux/bitops.h> #include <crypto/algapi.h> #include <crypto/aead.h> #include <crypto/internal/aead.h> #include <crypto/aes.h> #include <crypto/internal/des.h> #include <crypto/hmac.h> #include <crypto/md5.h> #include <crypto/authenc.h> #include <crypto/skcipher.h> #include <crypto/hash.h> #include <crypto/sha1.h> #include <crypto/sha2.h> #include <crypto/sha3.h> #include "util.h" #include "cipher.h" #include "spu.h" #include "spum.h" #include "spu2.h" /* ================= Device Structure ================== */ struct bcm_device_private iproc_priv; /* ==================== Parameters ===================== */ int flow_debug_logging; module_param(flow_debug_logging, int, 0644); MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging"); int packet_debug_logging; module_param(packet_debug_logging, int, 0644); MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging"); int debug_logging_sleep; module_param(debug_logging_sleep, int, 0644); MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep"); /* * The value of these module parameters is used to set the priority for each * algo type when this driver registers algos with the kernel crypto API. * To use a priority other than the default, set the priority in the insmod or * modprobe. Changing the module priority after init time has no effect. * * The default priorities are chosen to be lower (less preferred) than ARMv8 CE * algos, but more preferred than generic software algos. */ static int cipher_pri = 150; module_param(cipher_pri, int, 0644); MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos"); static int hash_pri = 100; module_param(hash_pri, int, 0644); MODULE_PARM_DESC(hash_pri, "Priority for hash algos"); static int aead_pri = 150; module_param(aead_pri, int, 0644); MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); /* A type 3 BCM header, expected to precede the SPU header for SPU-M. * Bits 3 and 4 in the first byte encode the channel number (the dma ringset). * 0x60 - ring 0 * 0x68 - ring 1 * 0x70 - ring 2 * 0x78 - ring 3 */ static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 }; /* * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN * is set dynamically after reading SPU type from device tree. */ #define BCM_HDR_LEN iproc_priv.bcm_hdr_len /* min and max time to sleep before retrying when mbox queue is full. usec */ #define MBOX_SLEEP_MIN 800 #define MBOX_SLEEP_MAX 1000 /** * select_channel() - Select a SPU channel to handle a crypto request. Selects * channel in round robin order. * * Return: channel index */ static u8 select_channel(void) { u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan); return chan_idx % iproc_priv.spu.num_chan; } /** * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to * receive a SPU response message for an skcipher request. Includes buffers to * catch SPU message headers and the response data. * @mssg: mailbox message containing the receive sg * @rctx: crypto request context * @rx_frag_num: number of scatterlist elements required to hold the * SPU response message * @chunksize: Number of bytes of response data expected * @stat_pad_len: Number of bytes required to pad the STAT field to * a 4-byte boundary * * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() * when the request completes, whether the request is handled successfully or * there is an error. * * Returns: * 0 if successful * < 0 if an error */ static int spu_skcipher_rx_sg_create(struct brcm_message *mssg, struct iproc_reqctx_s *rctx, u8 rx_frag_num, unsigned int chunksize, u32 stat_pad_len) { struct spu_hw *spu = &iproc_priv.spu; struct scatterlist *sg; /* used to build sgs in mbox message */ struct iproc_ctx_s *ctx = rctx->ctx; u32 datalen; /* Number of bytes of response data expected */ mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp); if (!mssg->spu.dst) return -ENOMEM; sg = mssg->spu.dst; sg_init_table(sg, rx_frag_num); /* Space for SPU message header */ sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); /* If XTS tweak in payload, add buffer to receive encrypted tweak */ if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_XTS_TWEAK_SIZE); /* Copy in each dst sg entry from request, up to chunksize */ datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, rctx->dst_nents, chunksize); if (datalen < chunksize) { pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u", __func__, chunksize, datalen); return -EFAULT; } if (stat_pad_len) sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); return 0; } /** * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to * send a SPU request message for an skcipher request. Includes SPU message * headers and the request data. * @mssg: mailbox message containing the transmit sg * @rctx: crypto request context * @tx_frag_num: number of scatterlist elements required to construct the * SPU request message * @chunksize: Number of bytes of request data * @pad_len: Number of pad bytes * * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() * when the request completes, whether the request is handled successfully or * there is an error. * * Returns: * 0 if successful * < 0 if an error */ static int spu_skcipher_tx_sg_create(struct brcm_message *mssg, struct iproc_reqctx_s *rctx, u8 tx_frag_num, unsigned int chunksize, u32 pad_len) { struct spu_hw *spu = &iproc_priv.spu; struct scatterlist *sg; /* used to build sgs in mbox message */ struct iproc_ctx_s *ctx = rctx->ctx; u32 datalen; /* Number of bytes of response data expected */ u32 stat_len; mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp); if (unlikely(!mssg->spu.src)) return -ENOMEM; sg = mssg->spu.src; sg_init_table(sg, tx_frag_num); sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, BCM_HDR_LEN + ctx->spu_req_hdr_len); /* if XTS tweak in payload, copy from IV (where crypto API puts it) */ if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE); /* Copy in each src sg entry from request, up to chunksize */ datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, rctx->src_nents, chunksize); if (unlikely(datalen < chunksize)) { pr_err("%s(): failed to copy src sg to mbox msg", __func__); return -EFAULT; } if (pad_len) sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); stat_len = spu->spu_tx_status_len(); if (stat_len) { memset(rctx->msg_buf.tx_stat, 0, stat_len); sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); } return 0; } static int mailbox_send_message(struct brcm_message *mssg, u32 flags, u8 chan_idx) { int err; int retry_cnt = 0; struct device *dev = &(iproc_priv.pdev->dev); err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) { while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { /* * Mailbox queue is full. Since MAY_SLEEP is set, assume * not in atomic context and we can wait and try again. */ retry_cnt++; usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); atomic_inc(&iproc_priv.mb_no_spc); } } if (err < 0) { atomic_inc(&iproc_priv.mb_send_fail); return err; } /* Check error returned by mailbox controller */ err = mssg->error; if (unlikely(err < 0)) { dev_err(dev, "message error %d", err); /* Signal txdone for mailbox channel */ } /* Signal txdone for mailbox channel */ mbox_client_txdone(iproc_priv.mbox[chan_idx], err); return err; } /** * handle_skcipher_req() - Submit as much of a block cipher request as fits in * a single SPU request message, starting at the current position in the request * data. * @rctx: Crypto request context * * This may be called on the crypto API thread, or, when a request is so large * it must be broken into multiple SPU messages, on the thread used to invoke * the response callback. When requests are broken into multiple SPU * messages, we assume subsequent messages depend on previous results, and * thus always wait for previous results before submitting the next message. * Because requests are submitted in lock step like this, there is no need * to synchronize access to request data structures. * * Return: -EINPROGRESS: request has been accepted and result will be returned * asynchronously * Any other value indicates an error */ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) { struct spu_hw *spu = &iproc_priv.spu; struct crypto_async_request *areq = rctx->parent; struct skcipher_request *req = container_of(areq, struct skcipher_request, base); struct iproc_ctx_s *ctx = rctx->ctx; struct spu_cipher_parms cipher_parms; int err; unsigned int chunksize; /* Num bytes of request to submit */ int remaining; /* Bytes of request still to process */ int chunk_start; /* Beginning of data for current SPU msg */ /* IV or ctr value to use in this SPU msg */ u8 local_iv_ctr[MAX_IV_SIZE]; u32 stat_pad_len; /* num bytes to align status field */ u32 pad_len; /* total length of all padding */ struct brcm_message *mssg; /* mailbox message */ /* number of entries in src and dst sg in mailbox message. */ u8 rx_frag_num = 2; /* response header and STATUS */ u8 tx_frag_num = 1; /* request header */ flow_log("%s\n", __func__); cipher_parms.alg = ctx->cipher.alg; cipher_parms.mode = ctx->cipher.mode; cipher_parms.type = ctx->cipher_type; cipher_parms.key_len = ctx->enckeylen; cipher_parms.key_buf = ctx->enckey; cipher_parms.iv_buf = local_iv_ctr; cipher_parms.iv_len = rctx->iv_ctr_len; mssg = &rctx->mb_mssg; chunk_start = rctx->src_sent; remaining = rctx->total_todo - chunk_start; /* determine the chunk we are breaking off and update the indexes */ if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) && (remaining > ctx->max_payload)) chunksize = ctx->max_payload; else chunksize = remaining; rctx->src_sent += chunksize; rctx->total_sent = rctx->src_sent; /* Count number of sg entries to be included in this request */ rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); if ((ctx->cipher.mode == CIPHER_MODE_CBC) && rctx->is_encrypt && chunk_start) /* * Encrypting non-first first chunk. Copy last block of * previous result to IV for this chunk. */ sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr, rctx->iv_ctr_len, chunk_start - rctx->iv_ctr_len); if (rctx->iv_ctr_len) { /* get our local copy of the iv */ __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr, rctx->iv_ctr_len); /* generate the next IV if possible */ if ((ctx->cipher.mode == CIPHER_MODE_CBC) && !rctx->is_encrypt) { /* * CBC Decrypt: next IV is the last ciphertext block in * this chunk */ sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr, rctx->iv_ctr_len, rctx->src_sent - rctx->iv_ctr_len); } else if (ctx->cipher.mode == CIPHER_MODE_CTR) { /* * The SPU hardware increments the counter once for * each AES block of 16 bytes. So update the counter * for the next chunk, if there is one. Note that for * this chunk, the counter has already been copied to * local_iv_ctr. We can assume a block size of 16, * because we only support CTR mode for AES, not for * any other cipher alg. */ add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4); } } if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) flow_log("max_payload infinite\n"); else flow_log("max_payload %u\n", ctx->max_payload); flow_log("sent:%u start:%u remains:%u size:%u\n", rctx->src_sent, chunk_start, remaining, chunksize); /* Copy SPU header template created at setkey time */ memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, sizeof(rctx->msg_buf.bcm_spu_req_hdr)); spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, ctx->spu_req_hdr_len, !(rctx->is_encrypt), &cipher_parms, chunksize); atomic64_add(chunksize, &iproc_priv.bytes_out); stat_pad_len = spu->spu_wordalign_padlen(chunksize); if (stat_pad_len) rx_frag_num++; pad_len = stat_pad_len; if (pad_len) { tx_frag_num++; spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0, 0, ctx->auth.alg, ctx->auth.mode, rctx->total_sent, stat_pad_len); } spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, ctx->spu_req_hdr_len); packet_log("payload:\n"); dump_sg(rctx->src_sg, rctx->src_skip, chunksize); packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); /* * Build mailbox message containing SPU request msg and rx buffers * to catch response message */ memset(mssg, 0, sizeof(*mssg)); mssg->type = BRCM_MESSAGE_SPU; mssg->ctx = rctx; /* Will be returned in response */ /* Create rx scatterlist to catch result */ rx_frag_num += rctx->dst_nents; if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) rx_frag_num++; /* extra sg to insert tweak */ err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize, stat_pad_len); if (err) return err; /* Create tx scatterlist containing SPU request message */ tx_frag_num += rctx->src_nents; if (spu->spu_tx_status_len()) tx_frag_num++; if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload()) tx_frag_num++; /* extra sg to insert tweak */ err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize, pad_len); if (err) return err; err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); if (unlikely(err < 0)) return err; return -EINPROGRESS; } /** * handle_skcipher_resp() - Process a block cipher SPU response. Updates the * total received count for the request and updates global stats. * @rctx: Crypto request context */ static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) { struct spu_hw *spu = &iproc_priv.spu; struct crypto_async_request *areq = rctx->parent; struct skcipher_request *req = skcipher_request_cast(areq); struct iproc_ctx_s *ctx = rctx->ctx; u32 payload_len; /* See how much data was returned */ payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); /* * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the * encrypted tweak ("i") value; we don't count those. */ if ((ctx->cipher.mode == CIPHER_MODE_XTS) && spu->spu_xts_tweak_in_payload() && (payload_len >= SPU_XTS_TWEAK_SIZE)) payload_len -= SPU_XTS_TWEAK_SIZE; atomic64_add(payload_len, &iproc_priv.bytes_in); flow_log("%s() offset: %u, bd_len: %u BD:\n", __func__, rctx->total_received, payload_len); dump_sg(req->dst, rctx->total_received, payload_len); rctx->total_received += payload_len; if (rctx->total_received == rctx->total_todo) { atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]); atomic_inc( &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);