// SPDX-License-Identifier: GPL-2.0+/* * caam - Freescale FSL CAAM support for crypto API * * Copyright 2008-2011 Freescale Semiconductor, Inc. * Copyright 2016-2019, 2023, 2025 NXP * * Based on talitos crypto API driver. * * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): * * --------------- --------------- * | JobDesc #1 |-------------------->| ShareDesc | * | *(packet 1) | | (PDB) | * --------------- |------------->| (hashKey) | * . | | (cipherKey) | * . | |-------->| (operation) | * --------------- | | --------------- * | JobDesc #2 |------| | * | *(packet 2) | | * --------------- | * . | * . | * --------------- | * | JobDesc #3 |------------ * | *(packet 3) | * --------------- * * The SharedDesc never changes for a connection unless rekeyed, but * each packet will likely be in a different place. So all we need * to know to process the packet is where the input is, where the * output goes, and what context we want to process with. Context is * in the SharedDesc, packet references in the JobDesc. * * So, a job desc looks like: * * --------------------- * | Header | * | ShareDesc Pointer | * | SEQ_OUT_PTR | * | (output buffer) | * | (output length) | * | SEQ_IN_PTR | * | (input buffer) | * | (input length) | * --------------------- */ #include "compat.h" #include "regs.h"#include "intern.h"#include "desc_constr.h"#include "jr.h"#include "error.h"#include "sg_sw_sec4.h"#include "key_gen.h"#include "caamalg_desc.h"#include <linux/unaligned.h>#include <crypto/internal/aead.h>#include <crypto/internal/engine.h>#include <crypto/internal/skcipher.h>#include <crypto/xts.h>#include <keys/trusted-type.h>#include <linux/dma-mapping.h>#include <linux/device.h>#include <linux/err.h>#include <linux/module.h>#include <linux/kernel.h>#include <linux/key-type.h>#include <linux/slab.h>#include <linux/string.h>#include <soc/fsl/caam-blob.h> /* * crypto alg */#define CAAM_CRA_PRIORITY 3000/* max key is sum of AES_MAX_KEY_SIZE, max split key size */#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ CTR_RFC3686_NONCE_SIZE + \ SHA512_DIGEST_SIZE * 2) #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ CAAM_CMD_SZ * 4)#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ CAAM_CMD_SZ * 5) #define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) struct caam_alg_entry { int class1_alg_type; int class2_alg_type; bool rfc3686; bool geniv; bool nodkp;}; struct caam_aead_alg { struct aead_engine_alg aead; struct caam_alg_entry caam; bool registered;}; struct caam_skcipher_alg { struct skcipher_engine_alg skcipher; struct caam_alg_entry caam; bool registered;}; /* * per-session context */struct caam_ctx { u32 sh_desc_enc[DESC_MAX_USED_LEN]; u32 sh_desc_dec[DESC_MAX_USED_LEN]; u8 key[CAAM_MAX_KEY_SIZE]; dma_addr_t sh_desc_enc_dma; dma_addr_t sh_desc_dec_dma; dma_addr_t key_dma; u8 protected_key[CAAM_MAX_KEY_SIZE]; dma_addr_t protected_key_dma; enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; unsigned int authsize; bool xts_key_fallback; bool is_blob; struct crypto_skcipher *fallback;}; struct caam_skcipher_req_ctx { struct skcipher_edesc *edesc; struct skcipher_request fallback_req;}; struct caam_aead_req_ctx { struct aead_edesc *edesc;}; static int aead_null_set_sh_desc(struct crypto_aead *aead){ struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - ctx->adata.keylen_pad; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { ctx->adata.key_inline = true; ctx->adata.key_virt = ctx->key; } else { ctx->adata.key_inline = false; ctx->adata.key_dma = ctx->key_dma; } /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { ctx->adata.key_inline = true; ctx->adata.key_virt = ctx->key; } else { ctx->adata.key_inline = false; ctx->adata.key_dma = ctx->key_dma; } /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); return 0;} static int aead_set_sh_desc(struct crypto_aead *aead){ struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), struct caam_aead_alg, aead.base); unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 ctx1_iv_off = 0; u32 *desc, *nonce = NULL; u32 inl_mask; unsigned int data_len[2]; const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; if (!ctx->authsize) return 0; /* NULL encryption / decryption */ if (!ctx->cdata.keylen) return aead_null_set_sh_desc(aead); /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) * CONTEXT1[255:128] = IV */ if (ctr_mode) ctx1_iv_off = 16; /* * RFC3686 specific: * CONTEXT1[255:128] = {NONCE, IV, COUNTER} */ if (is_rfc3686) { ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); } /* * In case |user key| > |derived key|, using DKP<imm,imm> * would result in invalid opcodes (last bytes of user key) in * the resulting descriptor. Use DKP<ptr,imm> instead => both * virtual and dma key addresses are needed. */ ctx->adata.key_virt = ctx->key; ctx->adata.key_dma = ctx->key_dma; ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; data_len[0] = ctx->adata.keylen_pad; data_len[1] = ctx->cdata.keylen; if (alg->caam.geniv) goto skip_enc; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (desc_inline_query(DESC_AEAD_ENC_LEN + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, ARRAY_SIZE(data_len)) < 0) return -EINVAL; ctx->adata.key_inline = !!(inl_mask & 1); ctx->cdata.key_inline = !!(inl_mask & 2); /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); skip_enc: /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (desc_inline_query(DESC_AEAD_DEC_LEN + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, ARRAY_SIZE(data_len)) < 0) return -EINVAL; ctx->adata.key_inline = !!(inl_mask & 1); ctx->cdata.key_inline = !!(inl_mask & 2); /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, alg->caam.geniv, is_rfc3686, nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); if (!alg->caam.geniv) goto skip_givenc; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (desc_inline_query(DESC_AEAD_GIVENC_LEN + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, ARRAY_SIZE(data_len)) < 0) return -EINVAL; ctx->adata.key_inline = !!(inl_mask & 1); ctx->cdata.key_inline = !!(inl_mask & 2); /* aead_givencrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); skip_givenc: return 0;} static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize){ struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); ctx->authsize = authsize; aead_set_sh_desc(authenc); return 0;} static int gcm_set_sh_desc(struct crypto_aead *aead){ struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - ctx->cdata.keylen; if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* * AES GCM encrypt shared descriptor * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_GCM_ENC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_enc; cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_GCM_DEC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_dec; cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); return 0;} static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize){ struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); int err; err = crypto_gcm_check_authsize(authsize); if (err) return err; ctx->authsize = authsize; gcm_set_sh_desc(authenc); return 0;} static int rfc4106_set_sh_desc(struct crypto_aead *aead){ struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - ctx->cdata.keylen; if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* * RFC4106 encrypt shared descriptor * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_RFC4106_ENC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_RFC4106_DEC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_dec; cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); return 0;} static int rfc4106_setauthsize(struct crypto_aead *authenc, unsigned int authsize){ struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); int err; err = crypto_rfc4106_check_authsize(authsize); if (err) return err; ctx->authsize = authsize; rfc4106_set_sh_desc(authenc); return 0;} static int rfc4543_set_sh_desc(struct crypto_aead *aead){ struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - ctx->cdata.keylen; if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* * RFC4543 encrypt shared descriptor * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_RFC4543_ENC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir);Lines 1–500 of 3988. Load more
// SPDX-License-Identifier: GPL-2.0+ /* * caam - Freescale FSL CAAM support for crypto API * * Copyright 2008-2011 Freescale Semiconductor, Inc. * Copyright 2016-2019, 2023, 2025 NXP * * Based on talitos crypto API driver. * * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): * * --------------- --------------- * | JobDesc #1 |-------------------->| ShareDesc | * | *(packet 1) | | (PDB) | * --------------- |------------->| (hashKey) | * . | | (cipherKey) | * . | |-------->| (operation) | * --------------- | | --------------- * | JobDesc #2 |------| | * | *(packet 2) | | * --------------- | * . | * . | * --------------- | * | JobDesc #3 |------------ * | *(packet 3) | * --------------- * * The SharedDesc never changes for a connection unless rekeyed, but * each packet will likely be in a different place. So all we need * to know to process the packet is where the input is, where the * output goes, and what context we want to process with. Context is * in the SharedDesc, packet references in the JobDesc. * * So, a job desc looks like: * * --------------------- * | Header | * | ShareDesc Pointer | * | SEQ_OUT_PTR | * | (output buffer) | * | (output length) | * | SEQ_IN_PTR | * | (input buffer) | * | (input length) | * --------------------- */ #include "compat.h" #include "regs.h" #include "intern.h" #include "desc_constr.h" #include "jr.h" #include "error.h" #include "sg_sw_sec4.h" #include "key_gen.h" #include "caamalg_desc.h" #include <linux/unaligned.h> #include <crypto/internal/aead.h> #include <crypto/internal/engine.h> #include <crypto/internal/skcipher.h> #include <crypto/xts.h> #include <keys/trusted-type.h> #include <linux/dma-mapping.h> #include <linux/device.h> #include <linux/err.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/key-type.h> #include <linux/slab.h> #include <linux/string.h> #include <soc/fsl/caam-blob.h> /* * crypto alg */ #define CAAM_CRA_PRIORITY 3000 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ CTR_RFC3686_NONCE_SIZE + \ SHA512_DIGEST_SIZE * 2) #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ CAAM_CMD_SZ * 4) #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ CAAM_CMD_SZ * 5) #define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) struct caam_alg_entry { int class1_alg_type; int class2_alg_type; bool rfc3686; bool geniv; bool nodkp; }; struct caam_aead_alg { struct aead_engine_alg aead; struct caam_alg_entry caam; bool registered; }; struct caam_skcipher_alg { struct skcipher_engine_alg skcipher; struct caam_alg_entry caam; bool registered; }; /* * per-session context */ struct caam_ctx { u32 sh_desc_enc[DESC_MAX_USED_LEN]; u32 sh_desc_dec[DESC_MAX_USED_LEN]; u8 key[CAAM_MAX_KEY_SIZE]; dma_addr_t sh_desc_enc_dma; dma_addr_t sh_desc_dec_dma; dma_addr_t key_dma; u8 protected_key[CAAM_MAX_KEY_SIZE]; dma_addr_t protected_key_dma; enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; unsigned int authsize; bool xts_key_fallback; bool is_blob; struct crypto_skcipher *fallback; }; struct caam_skcipher_req_ctx { struct skcipher_edesc *edesc; struct skcipher_request fallback_req; }; struct caam_aead_req_ctx { struct aead_edesc *edesc; }; static int aead_null_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - ctx->adata.keylen_pad; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { ctx->adata.key_inline = true; ctx->adata.key_virt = ctx->key; } else { ctx->adata.key_inline = false; ctx->adata.key_dma = ctx->key_dma; } /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { ctx->adata.key_inline = true; ctx->adata.key_virt = ctx->key; } else { ctx->adata.key_inline = false; ctx->adata.key_dma = ctx->key_dma; } /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); return 0; } static int aead_set_sh_desc(struct crypto_aead *aead) { struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), struct caam_aead_alg, aead.base); unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 ctx1_iv_off = 0; u32 *desc, *nonce = NULL; u32 inl_mask; unsigned int data_len[2]; const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; if (!ctx->authsize) return 0; /* NULL encryption / decryption */ if (!ctx->cdata.keylen) return aead_null_set_sh_desc(aead); /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) * CONTEXT1[255:128] = IV */ if (ctr_mode) ctx1_iv_off = 16; /* * RFC3686 specific: * CONTEXT1[255:128] = {NONCE, IV, COUNTER} */ if (is_rfc3686) { ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); } /* * In case |user key| > |derived key|, using DKP<imm,imm> * would result in invalid opcodes (last bytes of user key) in * the resulting descriptor. Use DKP<ptr,imm> instead => both * virtual and dma key addresses are needed. */ ctx->adata.key_virt = ctx->key; ctx->adata.key_dma = ctx->key_dma; ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; data_len[0] = ctx->adata.keylen_pad; data_len[1] = ctx->cdata.keylen; if (alg->caam.geniv) goto skip_enc; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (desc_inline_query(DESC_AEAD_ENC_LEN + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, ARRAY_SIZE(data_len)) < 0) return -EINVAL; ctx->adata.key_inline = !!(inl_mask & 1); ctx->cdata.key_inline = !!(inl_mask & 2); /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); skip_enc: /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (desc_inline_query(DESC_AEAD_DEC_LEN + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, ARRAY_SIZE(data_len)) < 0) return -EINVAL; ctx->adata.key_inline = !!(inl_mask & 1); ctx->cdata.key_inline = !!(inl_mask & 2); /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, alg->caam.geniv, is_rfc3686, nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); if (!alg->caam.geniv) goto skip_givenc; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (desc_inline_query(DESC_AEAD_GIVENC_LEN + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, ARRAY_SIZE(data_len)) < 0) return -EINVAL; ctx->adata.key_inline = !!(inl_mask & 1); ctx->cdata.key_inline = !!(inl_mask & 2); /* aead_givencrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); skip_givenc: return 0; } static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) { struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); ctx->authsize = authsize; aead_set_sh_desc(authenc); return 0; } static int gcm_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - ctx->cdata.keylen; if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* * AES GCM encrypt shared descriptor * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_GCM_ENC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_enc; cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_GCM_DEC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_dec; cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); return 0; } static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) { struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); int err; err = crypto_gcm_check_authsize(authsize); if (err) return err; ctx->authsize = authsize; gcm_set_sh_desc(authenc); return 0; } static int rfc4106_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - ctx->cdata.keylen; if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* * RFC4106 encrypt shared descriptor * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_RFC4106_ENC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_RFC4106_DEC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_dec; cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), ctx->dir); return 0; } static int rfc4106_setauthsize(struct crypto_aead *authenc, unsigned int authsize) { struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); int err; err = crypto_rfc4106_check_authsize(authsize); if (err) return err; ctx->authsize = authsize; rfc4106_set_sh_desc(authenc); return 0; } static int rfc4543_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); struct device *jrdev = ctx->jrdev; unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - ctx->cdata.keylen; if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* * RFC4543 encrypt shared descriptor * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ if (rem_bytes >= DESC_RFC4543_ENC_LEN) { ctx->cdata.key_inline = true; ctx->cdata.key_virt = ctx->key; } else { ctx->cdata.key_inline = false; ctx->cdata.key_dma = ctx->key_dma; } desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), ctx->dir);