// SPDX-License-Identifier: GPL-2.0-or-later/* * Quick & dirty crypto benchmarking module. * * This will only exist until we have a better benchmarking mechanism * (e.g. a char device). * * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org> * Copyright (c) 2007 Nokia Siemens Networks * * Updated RFC4106 AES-GCM testing. * Authors: Aidan O'Mahony (aidan.o.mahony@intel.com) * Adrian Hoban <adrian.hoban@intel.com> * Gabriele Paoloni <gabriele.paoloni@intel.com> * Tadeusz Struk (tadeusz.struk@intel.com) * Copyright (c) 2010, Intel Corporation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/aead.h>#include <crypto/hash.h>#include <crypto/skcipher.h>#include <linux/err.h>#include <linux/fips.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/jiffies.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/moduleparam.h>#include <linux/scatterlist.h>#include <linux/slab.h>#include <linux/string.h>#include <linux/timex.h> #include "internal.h"#include "tcrypt.h" /* * Need slab memory for benchmarking (size in number of pages). */#define TVMEMSIZE 4 /** Used by test_cipher_speed()*/#define ENCRYPT 1#define DECRYPT 0 #define MAX_DIGEST_SIZE 64 /* * return a string with the driver name */#define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm)) /* * Used by test_cipher_speed() */static unsigned int sec; static char *alg;static u32 type;static u32 mask;static int mode;static u32 num_mb = 8;static unsigned int klen;static char *tvmem[TVMEMSIZE]; static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 };static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 }; #define XBUFSIZE 8#define MAX_IVLEN 32 static int testmgr_alloc_buf(char *buf[XBUFSIZE]){ int i; for (i = 0; i < XBUFSIZE; i++) { buf[i] = (void *)__get_free_page(GFP_KERNEL); if (!buf[i]) goto err_free_buf; } return 0; err_free_buf: while (i-- > 0) free_page((unsigned long)buf[i]); return -ENOMEM;} static void testmgr_free_buf(char *buf[XBUFSIZE]){ int i; for (i = 0; i < XBUFSIZE; i++) free_page((unsigned long)buf[i]);} static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], unsigned int buflen, const void *assoc, unsigned int aad_size){ int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; int k, rem; if (np > XBUFSIZE) { rem = PAGE_SIZE; np = XBUFSIZE; } else { rem = buflen % PAGE_SIZE; } sg_init_table(sg, np + 1); sg_set_buf(&sg[0], assoc, aad_size); if (rem) np--; for (k = 0; k < np; k++) sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE); if (rem) sg_set_buf(&sg[k + 1], xbuf[k], rem);} static inline int do_one_aead_op(struct aead_request *req, int ret){ struct crypto_wait *wait = req->base.data; return crypto_wait_req(ret, wait);} struct test_mb_aead_data { struct scatterlist sg[XBUFSIZE]; struct scatterlist sgout[XBUFSIZE]; struct aead_request *req; struct crypto_wait wait; char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; char *axbuf[XBUFSIZE];}; static int do_mult_aead_op(struct test_mb_aead_data *data, int enc, u32 num_mb, int *rc){ int i, err = 0; /* Fire up a bunch of concurrent requests */ for (i = 0; i < num_mb; i++) { if (enc == ENCRYPT) rc[i] = crypto_aead_encrypt(data[i].req); else rc[i] = crypto_aead_decrypt(data[i].req); } /* Wait for all requests to finish */ for (i = 0; i < num_mb; i++) { rc[i] = crypto_wait_req(rc[i], &data[i].wait); if (rc[i]) { pr_info("concurrent request %d error %d\n", i, rc[i]); err = rc[i]; } } return err;} static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc, int blen, int secs, u32 num_mb){ unsigned long start, end; int bcount; int ret = 0; int *rc; rc = kzalloc_objs(*rc, num_mb); if (!rc) return -ENOMEM; for (start = jiffies, end = start + secs * HZ, bcount = 0; time_before(jiffies, end); bcount++) { ret = do_mult_aead_op(data, enc, num_mb, rc); if (ret) goto out; } pr_cont("%d operations in %d seconds (%llu bytes)\n", bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); return ret;} static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc, int blen, u32 num_mb){ unsigned long cycles = 0; int ret = 0; int i; int *rc; rc = kzalloc_objs(*rc, num_mb); if (!rc) return -ENOMEM; /* Warm-up run. */ for (i = 0; i < 4; i++) { ret = do_mult_aead_op(data, enc, num_mb, rc); if (ret) goto out; } /* The real thing. */ for (i = 0; i < 8; i++) { cycles_t start, end; start = get_cycles(); ret = do_mult_aead_op(data, enc, num_mb, rc); end = get_cycles(); if (ret) goto out; cycles += end - start; } pr_cont("1 operation in %lu cycles (%d bytes)\n", (cycles + 4) / (8 * num_mb), blen); out: kfree(rc); return ret;} static void test_mb_aead_speed(const char *algo, int enc, int secs, struct aead_speed_template *template, unsigned int tcount, u8 authsize, unsigned int aad_size, u8 *keysize, u32 num_mb){ struct test_mb_aead_data *data; struct crypto_aead *tfm; unsigned int i, j, iv_len; const int *b_size; const char *key; const char *e; void *assoc; char *iv; int ret; if (aad_size >= PAGE_SIZE) { pr_err("associate data length (%u) too big\n", aad_size); return; } iv = kzalloc(MAX_IVLEN, GFP_KERNEL); if (!iv) return; if (enc == ENCRYPT) e = "encryption"; else e = "decryption"; data = kzalloc_objs(*data, num_mb); if (!data) goto out_free_iv; tfm = crypto_alloc_aead(algo, 0, 0); if (IS_ERR(tfm)) { pr_err("failed to load transform for %s: %ld\n", algo, PTR_ERR(tfm)); goto out_free_data; } ret = crypto_aead_setauthsize(tfm, authsize); if (ret) { pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo, ret); goto out_free_tfm; } for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].xbuf)) { while (i--) testmgr_free_buf(data[i].xbuf); goto out_free_tfm; } for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].axbuf)) { while (i--) testmgr_free_buf(data[i].axbuf); goto out_free_xbuf; } for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].xoutbuf)) { while (i--) testmgr_free_buf(data[i].xoutbuf); goto out_free_axbuf; } for (i = 0; i < num_mb; ++i) { data[i].req = aead_request_alloc(tfm, GFP_KERNEL); if (!data[i].req) { pr_err("alg: aead: Failed to allocate request for %s\n", algo); while (i--) aead_request_free(data[i].req); goto out_free_xoutbuf; } } for (i = 0; i < num_mb; ++i) { crypto_init_wait(&data[i].wait); aead_request_set_callback(data[i].req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &data[i].wait); } pr_info("testing speed of multibuffer %s (%s) %s\n", algo, get_driver_name(crypto_aead, tfm), e); i = 0; do { b_size = aead_sizes; do { int bs = round_up(*b_size, crypto_aead_blocksize(tfm)); if (bs + authsize > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", authsize + bs, XBUFSIZE * PAGE_SIZE); goto out; } pr_info("test %u (%d bit key, %d byte blocks): ", i, *keysize * 8, bs); /* Set up tfm global state, i.e. the key */ memset(tvmem[0], 0xff, PAGE_SIZE); key = tvmem[0]; for (j = 0; j < tcount; j++) { if (template[j].klen == *keysize) { key = template[j].key; break; } } crypto_aead_clear_flags(tfm, ~0); ret = crypto_aead_setkey(tfm, key, *keysize); if (ret) { pr_err("setkey() failed flags=%x\n", crypto_aead_get_flags(tfm)); goto out; } iv_len = crypto_aead_ivsize(tfm); if (iv_len) memset(iv, 0xff, iv_len); /* Now setup per request stuff, i.e. buffers */ for (j = 0; j < num_mb; ++j) { struct test_mb_aead_data *cur = &data[j]; assoc = cur->axbuf[0]; memset(assoc, 0xff, aad_size); sg_init_aead(cur->sg, cur->xbuf, bs + (enc ? 0 : authsize), assoc, aad_size); sg_init_aead(cur->sgout, cur->xoutbuf, bs + (enc ? authsize : 0), assoc, aad_size); aead_request_set_ad(cur->req, aad_size); if (!enc) { aead_request_set_crypt(cur->req, cur->sgout, cur->sg, bs, iv); ret = crypto_aead_encrypt(cur->req); ret = do_one_aead_op(cur->req, ret); if (ret) { pr_err("calculating auth failed (%d)\n", ret); break; } } aead_request_set_crypt(cur->req, cur->sg, cur->sgout, bs + (enc ? 0 : authsize), iv); } if (secs) { ret = test_mb_aead_jiffies(data, enc, bs, secs, num_mb); cond_resched(); } else { ret = test_mb_aead_cycles(data, enc, bs, num_mb); } if (ret) { pr_err("%s() failed return code=%d\n", e, ret); break; } b_size++; i++; } while (*b_size); keysize++; } while (*keysize); out: for (i = 0; i < num_mb; ++i) aead_request_free(data[i].req);out_free_xoutbuf: for (i = 0; i < num_mb; ++i) testmgr_free_buf(data[i].xoutbuf);out_free_axbuf: for (i = 0; i < num_mb; ++i) testmgr_free_buf(data[i].axbuf);out_free_xbuf: for (i = 0; i < num_mb; ++i) testmgr_free_buf(data[i].xbuf);out_free_tfm: crypto_free_aead(tfm);out_free_data: kfree(data);out_free_iv: kfree(iv);} static int test_aead_jiffies(struct aead_request *req, int enc, int blen, int secs){ unsigned long start, end; int bcount; int ret; for (start = jiffies, end = start + secs * HZ, bcount = 0; time_before(jiffies, end); bcount++) { if (enc) ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else ret = do_one_aead_op(req, crypto_aead_decrypt(req)); if (ret) return ret; } pr_cont("%d operations in %d seconds (%llu bytes)\n", bcount, secs, (u64)bcount * blen); return 0;} static int test_aead_cycles(struct aead_request *req, int enc, int blen){ unsigned long cycles = 0; int ret = 0; int i; /* Warm-up run. */ for (i = 0; i < 4; i++) { if (enc) ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else ret = do_one_aead_op(req, crypto_aead_decrypt(req)); if (ret) goto out; } /* The real thing. */ for (i = 0; i < 8; i++) { cycles_t start, end; start = get_cycles(); if (enc) ret = do_one_aead_op(req, crypto_aead_encrypt(req)); elseLines 1–500 of 2854. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * Quick & dirty crypto benchmarking module. * * This will only exist until we have a better benchmarking mechanism * (e.g. a char device). * * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org> * Copyright (c) 2007 Nokia Siemens Networks * * Updated RFC4106 AES-GCM testing. * Authors: Aidan O'Mahony (aidan.o.mahony@intel.com) * Adrian Hoban <adrian.hoban@intel.com> * Gabriele Paoloni <gabriele.paoloni@intel.com> * Tadeusz Struk (tadeusz.struk@intel.com) * Copyright (c) 2010, Intel Corporation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/aead.h> #include <crypto/hash.h> #include <crypto/skcipher.h> #include <linux/err.h> #include <linux/fips.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/jiffies.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/string.h> #include <linux/timex.h> #include "internal.h" #include "tcrypt.h" /* * Need slab memory for benchmarking (size in number of pages). */ #define TVMEMSIZE 4 /* * Used by test_cipher_speed() */ #define ENCRYPT 1 #define DECRYPT 0 #define MAX_DIGEST_SIZE 64 /* * return a string with the driver name */ #define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm)) /* * Used by test_cipher_speed() */ static unsigned int sec; static char *alg; static u32 type; static u32 mask; static int mode; static u32 num_mb = 8; static unsigned int klen; static char *tvmem[TVMEMSIZE]; static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 }; static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 }; #define XBUFSIZE 8 #define MAX_IVLEN 32 static int testmgr_alloc_buf(char *buf[XBUFSIZE]) { int i; for (i = 0; i < XBUFSIZE; i++) { buf[i] = (void *)__get_free_page(GFP_KERNEL); if (!buf[i]) goto err_free_buf; } return 0; err_free_buf: while (i-- > 0) free_page((unsigned long)buf[i]); return -ENOMEM; } static void testmgr_free_buf(char *buf[XBUFSIZE]) { int i; for (i = 0; i < XBUFSIZE; i++) free_page((unsigned long)buf[i]); } static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], unsigned int buflen, const void *assoc, unsigned int aad_size) { int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; int k, rem; if (np > XBUFSIZE) { rem = PAGE_SIZE; np = XBUFSIZE; } else { rem = buflen % PAGE_SIZE; } sg_init_table(sg, np + 1); sg_set_buf(&sg[0], assoc, aad_size); if (rem) np--; for (k = 0; k < np; k++) sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE); if (rem) sg_set_buf(&sg[k + 1], xbuf[k], rem); } static inline int do_one_aead_op(struct aead_request *req, int ret) { struct crypto_wait *wait = req->base.data; return crypto_wait_req(ret, wait); } struct test_mb_aead_data { struct scatterlist sg[XBUFSIZE]; struct scatterlist sgout[XBUFSIZE]; struct aead_request *req; struct crypto_wait wait; char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; char *axbuf[XBUFSIZE]; }; static int do_mult_aead_op(struct test_mb_aead_data *data, int enc, u32 num_mb, int *rc) { int i, err = 0; /* Fire up a bunch of concurrent requests */ for (i = 0; i < num_mb; i++) { if (enc == ENCRYPT) rc[i] = crypto_aead_encrypt(data[i].req); else rc[i] = crypto_aead_decrypt(data[i].req); } /* Wait for all requests to finish */ for (i = 0; i < num_mb; i++) { rc[i] = crypto_wait_req(rc[i], &data[i].wait); if (rc[i]) { pr_info("concurrent request %d error %d\n", i, rc[i]); err = rc[i]; } } return err; } static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc, int blen, int secs, u32 num_mb) { unsigned long start, end; int bcount; int ret = 0; int *rc; rc = kzalloc_objs(*rc, num_mb); if (!rc) return -ENOMEM; for (start = jiffies, end = start + secs * HZ, bcount = 0; time_before(jiffies, end); bcount++) { ret = do_mult_aead_op(data, enc, num_mb, rc); if (ret) goto out; } pr_cont("%d operations in %d seconds (%llu bytes)\n", bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); return ret; } static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc, int blen, u32 num_mb) { unsigned long cycles = 0; int ret = 0; int i; int *rc; rc = kzalloc_objs(*rc, num_mb); if (!rc) return -ENOMEM; /* Warm-up run. */ for (i = 0; i < 4; i++) { ret = do_mult_aead_op(data, enc, num_mb, rc); if (ret) goto out; } /* The real thing. */ for (i = 0; i < 8; i++) { cycles_t start, end; start = get_cycles(); ret = do_mult_aead_op(data, enc, num_mb, rc); end = get_cycles(); if (ret) goto out; cycles += end - start; } pr_cont("1 operation in %lu cycles (%d bytes)\n", (cycles + 4) / (8 * num_mb), blen); out: kfree(rc); return ret; } static void test_mb_aead_speed(const char *algo, int enc, int secs, struct aead_speed_template *template, unsigned int tcount, u8 authsize, unsigned int aad_size, u8 *keysize, u32 num_mb) { struct test_mb_aead_data *data; struct crypto_aead *tfm; unsigned int i, j, iv_len; const int *b_size; const char *key; const char *e; void *assoc; char *iv; int ret; if (aad_size >= PAGE_SIZE) { pr_err("associate data length (%u) too big\n", aad_size); return; } iv = kzalloc(MAX_IVLEN, GFP_KERNEL); if (!iv) return; if (enc == ENCRYPT) e = "encryption"; else e = "decryption"; data = kzalloc_objs(*data, num_mb); if (!data) goto out_free_iv; tfm = crypto_alloc_aead(algo, 0, 0); if (IS_ERR(tfm)) { pr_err("failed to load transform for %s: %ld\n", algo, PTR_ERR(tfm)); goto out_free_data; } ret = crypto_aead_setauthsize(tfm, authsize); if (ret) { pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo, ret); goto out_free_tfm; } for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].xbuf)) { while (i--) testmgr_free_buf(data[i].xbuf); goto out_free_tfm; } for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].axbuf)) { while (i--) testmgr_free_buf(data[i].axbuf); goto out_free_xbuf; } for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].xoutbuf)) { while (i--) testmgr_free_buf(data[i].xoutbuf); goto out_free_axbuf; } for (i = 0; i < num_mb; ++i) { data[i].req = aead_request_alloc(tfm, GFP_KERNEL); if (!data[i].req) { pr_err("alg: aead: Failed to allocate request for %s\n", algo); while (i--) aead_request_free(data[i].req); goto out_free_xoutbuf; } } for (i = 0; i < num_mb; ++i) { crypto_init_wait(&data[i].wait); aead_request_set_callback(data[i].req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &data[i].wait); } pr_info("testing speed of multibuffer %s (%s) %s\n", algo, get_driver_name(crypto_aead, tfm), e); i = 0; do { b_size = aead_sizes; do { int bs = round_up(*b_size, crypto_aead_blocksize(tfm)); if (bs + authsize > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", authsize + bs, XBUFSIZE * PAGE_SIZE); goto out; } pr_info("test %u (%d bit key, %d byte blocks): ", i, *keysize * 8, bs); /* Set up tfm global state, i.e. the key */ memset(tvmem[0], 0xff, PAGE_SIZE); key = tvmem[0]; for (j = 0; j < tcount; j++) { if (template[j].klen == *keysize) { key = template[j].key; break; } } crypto_aead_clear_flags(tfm, ~0); ret = crypto_aead_setkey(tfm, key, *keysize); if (ret) { pr_err("setkey() failed flags=%x\n", crypto_aead_get_flags(tfm)); goto out; } iv_len = crypto_aead_ivsize(tfm); if (iv_len) memset(iv, 0xff, iv_len); /* Now setup per request stuff, i.e. buffers */ for (j = 0; j < num_mb; ++j) { struct test_mb_aead_data *cur = &data[j]; assoc = cur->axbuf[0]; memset(assoc, 0xff, aad_size); sg_init_aead(cur->sg, cur->xbuf, bs + (enc ? 0 : authsize), assoc, aad_size); sg_init_aead(cur->sgout, cur->xoutbuf, bs + (enc ? authsize : 0), assoc, aad_size); aead_request_set_ad(cur->req, aad_size); if (!enc) { aead_request_set_crypt(cur->req, cur->sgout, cur->sg, bs, iv); ret = crypto_aead_encrypt(cur->req); ret = do_one_aead_op(cur->req, ret); if (ret) { pr_err("calculating auth failed (%d)\n", ret); break; } } aead_request_set_crypt(cur->req, cur->sg, cur->sgout, bs + (enc ? 0 : authsize), iv); } if (secs) { ret = test_mb_aead_jiffies(data, enc, bs, secs, num_mb); cond_resched(); } else { ret = test_mb_aead_cycles(data, enc, bs, num_mb); } if (ret) { pr_err("%s() failed return code=%d\n", e, ret); break; } b_size++; i++; } while (*b_size); keysize++; } while (*keysize); out: for (i = 0; i < num_mb; ++i) aead_request_free(data[i].req); out_free_xoutbuf: for (i = 0; i < num_mb; ++i) testmgr_free_buf(data[i].xoutbuf); out_free_axbuf: for (i = 0; i < num_mb; ++i) testmgr_free_buf(data[i].axbuf); out_free_xbuf: for (i = 0; i < num_mb; ++i) testmgr_free_buf(data[i].xbuf); out_free_tfm: crypto_free_aead(tfm); out_free_data: kfree(data); out_free_iv: kfree(iv); } static int test_aead_jiffies(struct aead_request *req, int enc, int blen, int secs) { unsigned long start, end; int bcount; int ret; for (start = jiffies, end = start + secs * HZ, bcount = 0; time_before(jiffies, end); bcount++) { if (enc) ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else ret = do_one_aead_op(req, crypto_aead_decrypt(req)); if (ret) return ret; } pr_cont("%d operations in %d seconds (%llu bytes)\n", bcount, secs, (u64)bcount * blen); return 0; } static int test_aead_cycles(struct aead_request *req, int enc, int blen) { unsigned long cycles = 0; int ret = 0; int i; /* Warm-up run. */ for (i = 0; i < 4; i++) { if (enc) ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else ret = do_one_aead_op(req, crypto_aead_decrypt(req)); if (ret) goto out; } /* The real thing. */ for (i = 0; i < 8; i++) { cycles_t start, end; start = get_cycles(); if (enc) ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else