Commit 199354d7 authored by Herbert Xu's avatar Herbert Xu
Browse files

crypto: caam - Remove GFP_DMA and add DMA alignment padding 



GFP_DMA does not guarantee that the returned memory is aligned
for DMA.  It should be removed where it is superfluous.

However, kmalloc may start returning DMA-unaligned memory in future
so fix this by adding the alignment by hand.

Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent c27b2d20

drivers/crypto/caam/blob_gen.c

+1 −1
Original line number Diff line number Diff line
@@ -83,7 +83,7 @@ int caam_process_blob(struct caam_blob_priv *priv, 
		output_len = info->input_len - CAAM_BLOB_OVERHEAD;

	}



	desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL | GFP_DMA);

	desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL);

	if (!desc)

		return -ENOMEM;

drivers/crypto/caam/caamalg.c

+10 −6
Original line number Diff line number Diff line
@@ -59,6 +59,8 @@ 
#include <crypto/engine.h>

#include <crypto/xts.h>

#include <asm/unaligned.h>

#include <linux/dma-mapping.h>

#include <linux/kernel.h>



/*

 * crypto alg
@@ -1379,8 +1381,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 
	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);



	/* allocate space for base edesc and hw desc commands, link tables */

	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,

			GFP_DMA | flags);

	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, flags);

	if (!edesc) {

		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,

			   0, 0, 0);
@@ -1608,6 +1609,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 
	u8 *iv;

	int ivsize = crypto_skcipher_ivsize(skcipher);

	int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;

	unsigned int aligned_size;



	src_nents = sg_nents_for_len(req->src, req->cryptlen);

	if (unlikely(src_nents < 0)) {
@@ -1681,15 +1683,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 
	/*

	 * allocate space for base edesc and hw desc commands, link tables, IV

	 */

	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,

			GFP_DMA | flags);

	if (!edesc) {

	aligned_size = ALIGN(ivsize, __alignof__(*edesc));

	aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;

	aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());

	iv = kzalloc(aligned_size, flags);

	if (!iv) {

		dev_err(jrdev, "could not allocate extended descriptor\n");

		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,

			   0, 0, 0);

		return ERR_PTR(-ENOMEM);

	}



	edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));

	edesc->src_nents = src_nents;

	edesc->dst_nents = dst_nents;

	edesc->mapped_src_nents = mapped_src_nents;
@@ -1701,7 +1706,6 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 


	/* Make sure IV is located in a DMAable area */

	if (ivsize) {

		iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;

		memcpy(iv, req->iv, ivsize);



		iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);

drivers/crypto/caam/caamalg_qi.c

+10 −6
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ 
#include "caamalg_desc.h"

#include <crypto/xts.h>

#include <asm/unaligned.h>

#include <linux/dma-mapping.h>

#include <linux/kernel.h>



/*

 * crypto alg
@@ -959,7 +961,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 
		return (struct aead_edesc *)drv_ctx;



	/* allocate space for base edesc and hw desc commands, link tables */

	edesc = qi_cache_alloc(GFP_DMA | flags);

	edesc = qi_cache_alloc(flags);

	if (unlikely(!edesc)) {

		dev_err(qidev, "could not allocate extended descriptor\n");

		return ERR_PTR(-ENOMEM);
@@ -1317,8 +1319,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 
		qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);



	qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);

	if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +

		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {

	if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +

		     offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >

		     CAAM_QI_MEMCACHE_SIZE)) {

		dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",

			qm_sg_ents, ivsize);

		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1327,17 +1330,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 
	}



	/* allocate space for base edesc, link tables and IV */

	edesc = qi_cache_alloc(GFP_DMA | flags);

	if (unlikely(!edesc)) {

	iv = qi_cache_alloc(flags);

	if (unlikely(!iv)) {

		dev_err(qidev, "could not allocate extended descriptor\n");

		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,

			   0, DMA_NONE, 0, 0);

		return ERR_PTR(-ENOMEM);

	}



	edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));



	/* Make sure IV is located in a DMAable area */

	sg_table = &edesc->sgt[0];

	iv = (u8 *)(sg_table + qm_sg_ents);

	memcpy(iv, req->iv, ivsize);



	iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);

drivers/crypto/caam/caamalg_qi2.c

+29 −19
Original line number Diff line number Diff line
@@ -16,7 +16,9 @@ 
#include "caamalg_desc.h"

#include "caamhash_desc.h"

#include "dpseci-debugfs.h"

#include <linux/dma-mapping.h>

#include <linux/fsl/mc.h>

#include <linux/kernel.h>

#include <soc/fsl/dpaa2-io.h>

#include <soc/fsl/dpaa2-fd.h>

#include <crypto/xts.h>
@@ -370,7 +372,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 
	struct dpaa2_sg_entry *sg_table;



	/* allocate space for base edesc, link tables and IV */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (unlikely(!edesc)) {

		dev_err(dev, "could not allocate extended descriptor\n");

		return ERR_PTR(-ENOMEM);
@@ -1189,7 +1191,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 
	}



	/* allocate space for base edesc, link tables and IV */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (unlikely(!edesc)) {

		dev_err(dev, "could not allocate extended descriptor\n");

		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -3220,14 +3222,14 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 
	int ret = -ENOMEM;

	struct dpaa2_fl_entry *in_fle, *out_fle;



	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);

	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL);

	if (!req_ctx)

		return -ENOMEM;



	in_fle = &req_ctx->fd_flt[1];

	out_fle = &req_ctx->fd_flt[0];



	flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);

	flc = kzalloc(sizeof(*flc), GFP_KERNEL);

	if (!flc)

		goto err_flc;
@@ -3316,7 +3318,13 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 
	dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);



	if (keylen > blocksize) {

		hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);

		unsigned int aligned_len =

			ALIGN(keylen, dma_get_cache_alignment());



		if (aligned_len < keylen)

			return -EOVERFLOW;



		hashed_key = kmemdup(key, aligned_len, GFP_KERNEL);

		if (!hashed_key)

			return -ENOMEM;

		ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
@@ -3560,7 +3568,7 @@ static int ahash_update_ctx(struct ahash_request *req) 
		}



		/* allocate space for base edesc and link tables */

		edesc = qi_cache_zalloc(GFP_DMA | flags);

		edesc = qi_cache_zalloc(flags);

		if (!edesc) {

			dma_unmap_sg(ctx->dev, req->src, src_nents,

				     DMA_TO_DEVICE);
@@ -3654,7 +3662,7 @@ static int ahash_final_ctx(struct ahash_request *req) 
	int ret;



	/* allocate space for base edesc and link tables */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (!edesc)

		return -ENOMEM;
@@ -3743,7 +3751,7 @@ static int ahash_finup_ctx(struct ahash_request *req) 
	}



	/* allocate space for base edesc and link tables */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (!edesc) {

		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);

		return -ENOMEM;
@@ -3836,7 +3844,7 @@ static int ahash_digest(struct ahash_request *req) 
	}



	/* allocate space for base edesc and link tables */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (!edesc) {

		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);

		return ret;
@@ -3913,7 +3921,7 @@ static int ahash_final_no_ctx(struct ahash_request *req) 
	int ret = -ENOMEM;



	/* allocate space for base edesc and link tables */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (!edesc)

		return ret;
@@ -4012,7 +4020,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) 
		}



		/* allocate space for base edesc and link tables */

		edesc = qi_cache_zalloc(GFP_DMA | flags);

		edesc = qi_cache_zalloc(flags);

		if (!edesc) {

			dma_unmap_sg(ctx->dev, req->src, src_nents,

				     DMA_TO_DEVICE);
@@ -4125,7 +4133,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req) 
	}



	/* allocate space for base edesc and link tables */

	edesc = qi_cache_zalloc(GFP_DMA | flags);

	edesc = qi_cache_zalloc(flags);

	if (!edesc) {

		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);

		return ret;
@@ -4230,7 +4238,7 @@ static int ahash_update_first(struct ahash_request *req) 
		}



		/* allocate space for base edesc and link tables */

		edesc = qi_cache_zalloc(GFP_DMA | flags);

		edesc = qi_cache_zalloc(flags);

		if (!edesc) {

			dma_unmap_sg(ctx->dev, req->src, src_nents,

				     DMA_TO_DEVICE);
@@ -4926,6 +4934,7 @@ static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 
{

	struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };

	struct device *dev = priv->dev;

	unsigned int alignmask;

	int err;



	/*
@@ -4936,13 +4945,14 @@ static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 
	    !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))

		return 0;



	priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,

				 GFP_KERNEL | GFP_DMA);

	alignmask = DPAA2_CSCN_ALIGN - 1;

	alignmask |= dma_get_cache_alignment() - 1;

	priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1),

				 GFP_KERNEL);

	if (!priv->cscn_mem)

		return -ENOMEM;



	priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);

	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,

	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem,

					DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);

	if (dma_mapping_error(dev, priv->cscn_dma)) {

		dev_err(dev, "Error mapping CSCN memory area\n");
@@ -5174,7 +5184,7 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 
	priv->domain = iommu_get_domain_for_dev(dev);



	qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,

				     0, SLAB_CACHE_DMA, NULL);

				     0, 0, NULL);

	if (!qi_cache) {

		dev_err(dev, "Can't allocate SEC cache\n");

		return -ENOMEM;
@@ -5451,7 +5461,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 
		dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,

					DPAA2_CSCN_SIZE,

					DMA_FROM_DEVICE);

		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {

		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) {

			dev_dbg_ratelimited(dev, "Dropping request\n");

			return -EBUSY;

		}

drivers/crypto/caam/caamalg_qi2.h

+4 −6
Original line number Diff line number Diff line
@@ -7,13 +7,14 @@ 
#ifndef _CAAMALG_QI2_H_

#define _CAAMALG_QI2_H_



#include <crypto/internal/skcipher.h>

#include <linux/compiler_attributes.h>

#include <soc/fsl/dpaa2-io.h>

#include <soc/fsl/dpaa2-fd.h>

#include <linux/threads.h>

#include <linux/netdevice.h>

#include "dpseci.h"

#include "desc_constr.h"

#include <crypto/skcipher.h>



#define DPAA2_CAAM_STORE_SIZE	16

/* NAPI weight *must* be a multiple of the store size. */
@@ -36,8 +37,6 @@ 
 * @tx_queue_attr: array of Tx queue attributes

 * @cscn_mem: pointer to memory region containing the congestion SCN

 *	it's size is larger than to accommodate alignment

 * @cscn_mem_aligned: pointer to congestion SCN; it is computed as

 *	PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN)

 * @cscn_dma: dma address used by the QMAN to write CSCN messages

 * @dev: device associated with the DPSECI object

 * @mc_io: pointer to MC portal's I/O object
@@ -58,7 +57,6 @@ struct dpaa2_caam_priv { 


	/* congestion */

	void *cscn_mem;

	void *cscn_mem_aligned;

	dma_addr_t cscn_dma;



	struct device *dev;
@@ -158,7 +156,7 @@ struct ahash_edesc { 
struct caam_flc {

	u32 flc[16];

	u32 sh_desc[MAX_SDLEN];

} ____cacheline_aligned;

} __aligned(CRYPTO_DMA_ALIGN);



enum optype {

	ENCRYPT = 0,
@@ -180,7 +178,7 @@ enum optype { 
 * @edesc: extended descriptor; points to one of {skcipher,aead}_edesc

 */

struct caam_request {

	struct dpaa2_fl_entry fd_flt[2];

	struct dpaa2_fl_entry fd_flt[2] __aligned(CRYPTO_DMA_ALIGN);

	dma_addr_t fd_flt_dma;

	struct caam_flc *flc;

	dma_addr_t flc_dma;