Commit e1ffcc66 authored by Dimitris Michailidis's avatar Dimitris Michailidis Committed by David S. Miller
Browse files

net/fungible: Add service module for Fungible drivers 



Fungible cards have a number of different PCI functions and thus
different drivers, all of which use a common method to initialize and
interact with the device. This commit adds a library module that
collects these common mechanisms. They mainly deal with device
initialization, setting up and destroying queues, and operating an admin
queue. A subset of the FW interface is also included here.

Signed-off-by: default avatarDimitris Michailidis <dmichail@fungible.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e8eb9e32

drivers/net/ethernet/fungible/funcore/Makefile

0 → 100644
+5 −0
Original line number Diff line number Diff line 
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)



obj-$(CONFIG_FUN_CORE) += funcore.o



funcore-y := fun_dev.o fun_queue.o

drivers/net/ethernet/fungible/funcore/fun_dev.c

0 → 100644
+843 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)



#include <linux/aer.h>

#include <linux/bitmap.h>

#include <linux/delay.h>

#include <linux/interrupt.h>

#include <linux/io.h>

#include <linux/io-64-nonatomic-lo-hi.h>

#include <linux/mm.h>

#include <linux/module.h>

#include <linux/nvme.h>

#include <linux/pci.h>

#include <linux/wait.h>

#include <linux/sched/signal.h>



#include "fun_queue.h"

#include "fun_dev.h"



#define FUN_ADMIN_CMD_TO_MS 3000



enum {

	AQA_ASQS_SHIFT = 0,

	AQA_ACQS_SHIFT = 16,

	AQA_MIN_QUEUE_SIZE = 2,

	AQA_MAX_QUEUE_SIZE = 4096

};



/* context for admin commands */

struct fun_cmd_ctx {

	fun_admin_callback_t cb;  /* callback to invoke on completion */

	void *cb_data;            /* user data provided to callback */

	int cpu;                  /* CPU where the cmd's tag was allocated */

};



/* Context for synchronous admin commands. */

struct fun_sync_cmd_ctx {

	struct completion compl;

	u8 *rsp_buf;              /* caller provided response buffer */

	unsigned int rsp_len;     /* response buffer size */

	u8 rsp_status;            /* command response status */

};



/* Wait for the CSTS.RDY bit to match @enabled. */

static int fun_wait_ready(struct fun_dev *fdev, bool enabled)

{

	unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);

	u32 bit = enabled ? NVME_CSTS_RDY : 0;

	unsigned long deadline;



	deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */



	for (;;) {

		u32 csts = readl(fdev->bar + NVME_REG_CSTS);



		if (csts == ~0) {

			dev_err(fdev->dev, "CSTS register read %#x\n", csts);

			return -EIO;

		}



		if ((csts & NVME_CSTS_RDY) == bit)

			return 0;



		if (time_is_before_jiffies(deadline))

			break;



		msleep(100);

	}



	dev_err(fdev->dev,

		"Timed out waiting for device to indicate RDY %u; aborting %s\n",

		enabled, enabled ? "initialization" : "reset");

	return -ETIMEDOUT;

}



/* Check CSTS and return an error if it is unreadable or has unexpected

 * RDY value.

 */

static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)

{

	u32 csts = readl(fdev->bar + NVME_REG_CSTS);

	u32 actual_rdy = csts & NVME_CSTS_RDY;



	if (csts == ~0) {

		dev_err(fdev->dev, "CSTS register read %#x\n", csts);

		return -EIO;

	}

	if (actual_rdy != expected_rdy) {

		dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);

		return -EINVAL;

	}

	return 0;

}



/* Check that CSTS RDY has the expected value. Then write a new value to the CC

 * register and wait for CSTS RDY to match the new CC ENABLE state.

 */

static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)

{

	int rc = fun_check_csts_rdy(fdev, initial_rdy);



	if (rc)

		return rc;

	writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);

	return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));

}



static int fun_disable_ctrl(struct fun_dev *fdev)

{

	fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);

	return fun_update_cc_enable(fdev, 1);

}



static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,

			   u32 admin_sqesz_log2)

{

	fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |

		       (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |

		       ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |

		       NVME_CC_ENABLE;



	return fun_update_cc_enable(fdev, 0);

}



static int fun_map_bars(struct fun_dev *fdev, const char *name)

{

	struct pci_dev *pdev = to_pci_dev(fdev->dev);

	int err;



	err = pci_request_mem_regions(pdev, name);

	if (err) {

		dev_err(&pdev->dev,

			"Couldn't get PCI memory resources, err %d\n", err);

		return err;

	}



	fdev->bar = pci_ioremap_bar(pdev, 0);

	if (!fdev->bar) {

		dev_err(&pdev->dev, "Couldn't map BAR 0\n");

		pci_release_mem_regions(pdev);

		return -ENOMEM;

	}



	return 0;

}



static void fun_unmap_bars(struct fun_dev *fdev)

{

	struct pci_dev *pdev = to_pci_dev(fdev->dev);



	if (fdev->bar) {

		iounmap(fdev->bar);

		fdev->bar = NULL;

		pci_release_mem_regions(pdev);

	}

}



static int fun_set_dma_masks(struct device *dev)

{

	int err;



	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));

	if (err)

		dev_err(dev, "DMA mask configuration failed, err %d\n", err);

	return err;

}



static irqreturn_t fun_admin_irq(int irq, void *data)

{

	struct fun_queue *funq = data;



	return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;

}



static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,

				   void *entry, const struct fun_cqe_info *info)

{

	const struct fun_admin_rsp_common *rsp_common = entry;

	struct fun_dev *fdev = funq->fdev;

	struct fun_cmd_ctx *cmd_ctx;

	int cpu;

	u16 cid;



	if (info->sqhd == cpu_to_be16(0xffff)) {

		dev_dbg(fdev->dev, "adminq event");

		if (fdev->adminq_cb)

			fdev->adminq_cb(fdev, entry);

		return;

	}



	cid = be16_to_cpu(rsp_common->cid);

	dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,

		rsp_common->op, rsp_common->ret);



	cmd_ctx = &fdev->cmd_ctx[cid];

	if (cmd_ctx->cpu < 0) {

		dev_err(fdev->dev,

			"admin CQE with CID=%u, op=%u does not match a pending command\n",

			cid, rsp_common->op);

		return;

	}



	if (cmd_ctx->cb)

		cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));



	cpu = cmd_ctx->cpu;

	cmd_ctx->cpu = -1;

	sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);

}



static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)

{

	unsigned int i;



	fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);

	if (!fdev->cmd_ctx)

		return -ENOMEM;



	for (i = 0; i < ntags; i++)

		fdev->cmd_ctx[i].cpu = -1;



	return 0;

}



/* Allocate and enable an admin queue and assign it the first IRQ vector. */

static int fun_enable_admin_queue(struct fun_dev *fdev,

				  const struct fun_dev_params *areq)

{

	struct fun_queue_alloc_req qreq = {

		.cqe_size_log2 = areq->cqe_size_log2,

		.sqe_size_log2 = areq->sqe_size_log2,

		.cq_depth = areq->cq_depth,

		.sq_depth = areq->sq_depth,

		.rq_depth = areq->rq_depth,

	};

	unsigned int ntags = areq->sq_depth - 1;

	struct fun_queue *funq;

	int rc;



	if (fdev->admin_q)

		return -EEXIST;



	if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||

	    areq->sq_depth > AQA_MAX_QUEUE_SIZE ||

	    areq->cq_depth < AQA_MIN_QUEUE_SIZE ||

	    areq->cq_depth > AQA_MAX_QUEUE_SIZE)

		return -EINVAL;



	fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);

	if (!fdev->admin_q)

		return -ENOMEM;



	rc = fun_init_cmd_ctx(fdev, ntags);

	if (rc)

		goto free_q;



	rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,

				     GFP_KERNEL, dev_to_node(fdev->dev));

	if (rc)

		goto free_cmd_ctx;



	funq = fdev->admin_q;

	funq->cq_vector = 0;

	rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);

	if (rc)

		goto free_sbq;



	fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);

	fdev->adminq_cb = areq->event_cb;



	writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT |

	       (funq->cq_depth - 1) << AQA_ACQS_SHIFT,

	       fdev->bar + NVME_REG_AQA);



	writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);

	writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);



	rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);

	if (rc)

		goto free_irq;



	if (areq->rq_depth) {

		rc = fun_create_rq(funq);

		if (rc)

			goto disable_ctrl;



		funq_rq_post(funq);

	}



	return 0;



disable_ctrl:

	fun_disable_ctrl(fdev);

free_irq:

	fun_free_irq(funq);

free_sbq:

	sbitmap_queue_free(&fdev->admin_sbq);

free_cmd_ctx:

	kvfree(fdev->cmd_ctx);

	fdev->cmd_ctx = NULL;

free_q:

	fun_free_queue(fdev->admin_q);

	fdev->admin_q = NULL;

	return rc;

}



static void fun_disable_admin_queue(struct fun_dev *fdev)

{

	struct fun_queue *admq = fdev->admin_q;



	if (!admq)

		return;



	fun_disable_ctrl(fdev);



	fun_free_irq(admq);

	__fun_process_cq(admq, 0);



	sbitmap_queue_free(&fdev->admin_sbq);



	kvfree(fdev->cmd_ctx);

	fdev->cmd_ctx = NULL;



	fun_free_queue(admq);

	fdev->admin_q = NULL;

}



/* Return %true if the admin queue has stopped servicing commands as can be

 * detected through registers. This isn't exhaustive and may provide false

 * negatives.

 */

static bool fun_adminq_stopped(struct fun_dev *fdev)

{

	u32 csts = readl(fdev->bar + NVME_REG_CSTS);



	return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;

}



static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)

{

	struct sbitmap_queue *sbq = &fdev->admin_sbq;

	struct sbq_wait_state *ws = &sbq->ws[0];

	DEFINE_SBQ_WAIT(wait);

	int tag;



	for (;;) {

		sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);

		if (fdev->suppress_cmds) {

			tag = -ESHUTDOWN;

			break;

		}

		tag = sbitmap_queue_get(sbq, cpup);

		if (tag >= 0)

			break;

		schedule();

	}



	sbitmap_finish_wait(sbq, ws, &wait);

	return tag;

}



/* Submit an asynchronous admin command. Caller is responsible for implementing

 * any waiting or timeout. Upon command completion the callback @cb is called.

 */

int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,

			 fun_admin_callback_t cb, void *cb_data, bool wait_ok)

{

	struct fun_queue *funq = fdev->admin_q;

	unsigned int cmdsize = cmd->len8 * 8;

	struct fun_cmd_ctx *cmd_ctx;

	int tag, cpu, rc = 0;



	if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))

		return -EMSGSIZE;



	tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);

	if (tag < 0) {

		if (!wait_ok)

			return -EAGAIN;

		tag = fun_wait_for_tag(fdev, &cpu);

		if (tag < 0)

			return tag;

	}



	cmd->cid = cpu_to_be16(tag);



	cmd_ctx = &fdev->cmd_ctx[tag];

	cmd_ctx->cb = cb;

	cmd_ctx->cb_data = cb_data;



	spin_lock(&funq->sq_lock);



	if (unlikely(fdev->suppress_cmds)) {

		rc = -ESHUTDOWN;

		sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);

	} else {

		cmd_ctx->cpu = cpu;

		memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);



		dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,

			cmd);



		if (++funq->sq_tail == funq->sq_depth)

			funq->sq_tail = 0;

		writel(funq->sq_tail, funq->sq_db);

	}

	spin_unlock(&funq->sq_lock);

	return rc;

}



/* Abandon a pending admin command by clearing the issuer's callback data.

 * Failure indicates that the command either has already completed or its

 * completion is racing with this call.

 */

static bool fun_abandon_admin_cmd(struct fun_dev *fd,

				  const struct fun_admin_req_common *cmd,

				  void *cb_data)

{

	u16 cid = be16_to_cpu(cmd->cid);

	struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];



	return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;

}



/* Stop submission of new admin commands and wake up any processes waiting for

 * tags. Already submitted commands are left to complete or time out.

 */

static void fun_admin_stop(struct fun_dev *fdev)

{

	spin_lock(&fdev->admin_q->sq_lock);

	fdev->suppress_cmds = true;

	spin_unlock(&fdev->admin_q->sq_lock);

	sbitmap_queue_wake_all(&fdev->admin_sbq);

}



/* The callback for synchronous execution of admin commands. It copies the

 * command response to the caller's buffer and signals completion.

 */

static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)

{

	const struct fun_admin_rsp_common *rsp_common = rsp;

	struct fun_sync_cmd_ctx *ctx = cb_data;



	if (!ctx)

		return;         /* command issuer timed out and left */

	if (ctx->rsp_buf) {

		unsigned int rsp_len = rsp_common->len8 * 8;



		if (unlikely(rsp_len > ctx->rsp_len)) {

			dev_err(fd->dev,

				"response for op %u is %uB > response buffer %uB\n",

				rsp_common->op, rsp_len, ctx->rsp_len);

			rsp_len = ctx->rsp_len;

		}

		memcpy(ctx->rsp_buf, rsp, rsp_len);

	}

	ctx->rsp_status = rsp_common->ret;

	complete(&ctx->compl);

}



/* Submit a synchronous admin command. */

int fun_submit_admin_sync_cmd(struct fun_dev *fdev,

			      struct fun_admin_req_common *cmd, void *rsp,

			      size_t rspsize, unsigned int timeout)

{

	struct fun_sync_cmd_ctx ctx = {

		.compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),

		.rsp_buf = rsp,

		.rsp_len = rspsize,

	};

	unsigned int cmdlen = cmd->len8 * 8;

	unsigned long jiffies_left;

	int ret;



	ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,

				   true);

	if (ret)

		return ret;



	if (!timeout)

		timeout = FUN_ADMIN_CMD_TO_MS;



	jiffies_left = wait_for_completion_timeout(&ctx.compl,

						   msecs_to_jiffies(timeout));

	if (!jiffies_left) {

		/* The command timed out. Attempt to cancel it so we can return.

		 * But if the command is in the process of completing we'll

		 * wait for it.

		 */

		if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {

			dev_err(fdev->dev, "admin command timed out: %*ph\n",

				cmdlen, cmd);

			fun_admin_stop(fdev);

			/* see if the timeout was due to a queue failure */

			if (fun_adminq_stopped(fdev))

				dev_err(fdev->dev,

					"device does not accept admin commands\n");



			return -ETIMEDOUT;

		}

		wait_for_completion(&ctx.compl);

	}



	if (ctx.rsp_status) {

		dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",

			ctx.rsp_status, cmdlen, cmd);

	}



	return -ctx.rsp_status;

}

EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);



/* Return the number of device resources of the requested type. */

int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)

{

	union {

		struct fun_admin_res_count_req req;

		struct fun_admin_res_count_rsp rsp;

	} cmd;

	int rc;



	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));

	cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,

						    0, 0);



	rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,

				       sizeof(cmd), 0);

	return rc ? rc : be32_to_cpu(cmd.rsp.count.data);

}

EXPORT_SYMBOL_GPL(fun_get_res_count);



/* Request that the instance of resource @res with the given id be deleted. */

int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,

		    unsigned int flags, u32 id)

{

	struct fun_admin_generic_destroy_req req = {

		.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),

		.destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,

						       flags, id)

	};



	return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);

}

EXPORT_SYMBOL_GPL(fun_res_destroy);



/* Bind two entities of the given types and IDs. */

int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,

	     unsigned int id0, enum fun_admin_bind_type type1,

	     unsigned int id1)

{

	struct {

		struct fun_admin_bind_req req;

		struct fun_admin_bind_entry entry[2];

	} cmd = {

		.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,

							 sizeof(cmd)),

		.entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0),

		.entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1),

	};



	return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0);

}

EXPORT_SYMBOL_GPL(fun_bind);



static int fun_get_dev_limits(struct fun_dev *fdev)

{

	struct pci_dev *pdev = to_pci_dev(fdev->dev);

	unsigned int cq_count, sq_count, num_dbs;

	int rc;



	rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);

	if (rc < 0)

		return rc;

	cq_count = rc;



	rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);

	if (rc < 0)

		return rc;

	sq_count = rc;



	/* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the

	 * device must provide additional queues.

	 */

	if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)

		return -EINVAL;



	/* Calculate the max QID based on SQ/CQ/doorbell counts.

	 * SQ/CQ doorbells alternate.

	 */

	num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) /

		  (fdev->db_stride * 4);

	fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;

	fdev->kern_end_qid = fdev->max_qid + 1;

	return 0;

}



/* Allocate all MSI-X vectors available on a function and at least @min_vecs. */

static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)

{

	int vecs, num_msix = pci_msix_vec_count(pdev);



	if (num_msix < 0)

		return num_msix;

	if (min_vecs > num_msix)

		return -ERANGE;



	vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);

	if (vecs > 0) {

		dev_info(&pdev->dev,

			 "Allocated %d IRQ vectors of %d requested\n",

			 vecs, num_msix);

	} else {

		dev_err(&pdev->dev,

			"Unable to allocate at least %u IRQ vectors\n",

			min_vecs);

	}

	return vecs;

}



/* Allocate and initialize the IRQ manager state. */

static int fun_alloc_irq_mgr(struct fun_dev *fdev)

{

	fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);

	if (!fdev->irq_map)

		return -ENOMEM;



	spin_lock_init(&fdev->irqmgr_lock);

	/* mark IRQ 0 allocated, it is used by the admin queue */

	__set_bit(0, fdev->irq_map);

	fdev->irqs_avail = fdev->num_irqs - 1;

	return 0;

}



/* Reserve @nirqs of the currently available IRQs and return their indices. */

int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)

{

	unsigned int b, n = 0;

	int err = -ENOSPC;



	if (!nirqs)

		return 0;



	spin_lock(&fdev->irqmgr_lock);

	if (nirqs > fdev->irqs_avail)

		goto unlock;



	for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {

		__set_bit(b, fdev->irq_map);

		irq_indices[n++] = b;

		if (n >= nirqs)

			break;

	}



	WARN_ON(n < nirqs);

	fdev->irqs_avail -= n;

	err = n;

unlock:

	spin_unlock(&fdev->irqmgr_lock);

	return err;

}

EXPORT_SYMBOL(fun_reserve_irqs);



/* Release @nirqs previously allocated IRQS with the supplied indices. */

void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,

		      u16 *irq_indices)

{

	unsigned int i;



	spin_lock(&fdev->irqmgr_lock);

	for (i = 0; i < nirqs; i++)

		__clear_bit(irq_indices[i], fdev->irq_map);

	fdev->irqs_avail += nirqs;

	spin_unlock(&fdev->irqmgr_lock);

}

EXPORT_SYMBOL(fun_release_irqs);



static void fun_serv_handler(struct work_struct *work)

{

	struct fun_dev *fd = container_of(work, struct fun_dev, service_task);



	if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))

		return;

	if (fd->serv_cb)

		fd->serv_cb(fd);

}



void fun_serv_stop(struct fun_dev *fd)

{

	set_bit(FUN_SERV_DISABLED, &fd->service_flags);

	cancel_work_sync(&fd->service_task);

}

EXPORT_SYMBOL_GPL(fun_serv_stop);



void fun_serv_restart(struct fun_dev *fd)

{

	clear_bit(FUN_SERV_DISABLED, &fd->service_flags);

	if (fd->service_flags)

		schedule_work(&fd->service_task);

}

EXPORT_SYMBOL_GPL(fun_serv_restart);



void fun_serv_sched(struct fun_dev *fd)

{

	if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))

		schedule_work(&fd->service_task);

}

EXPORT_SYMBOL_GPL(fun_serv_sched);



/* Check and try to get the device into a proper state for initialization,

 * i.e., CSTS.RDY = CC.EN = 0.

 */

static int sanitize_dev(struct fun_dev *fdev)

{

	int rc;



	fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);

	fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);



	/* First get RDY to agree with the current EN. Give RDY the opportunity

	 * to complete a potential recent EN change.

	 */

	rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);

	if (rc)

		return rc;



	/* Next, reset the device if EN is currently 1. */

	if (fdev->cc_reg & NVME_CC_ENABLE)

		rc = fun_disable_ctrl(fdev);



	return rc;

}



/* Undo the device initialization of fun_dev_enable(). */

void fun_dev_disable(struct fun_dev *fdev)

{

	struct pci_dev *pdev = to_pci_dev(fdev->dev);



	pci_set_drvdata(pdev, NULL);



	if (fdev->fw_handle != FUN_HCI_ID_INVALID) {

		fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,

				fdev->fw_handle);

		fdev->fw_handle = FUN_HCI_ID_INVALID;

	}



	fun_disable_admin_queue(fdev);



	bitmap_free(fdev->irq_map);

	pci_free_irq_vectors(pdev);



	pci_clear_master(pdev);

	pci_disable_pcie_error_reporting(pdev);

	pci_disable_device(pdev);



	fun_unmap_bars(fdev);

}

EXPORT_SYMBOL(fun_dev_disable);



/* Perform basic initialization of a device, including

 * - PCI config space setup and BAR0 mapping

 * - interrupt management initialization

 * - 1 admin queue setup

 * - determination of some device limits, such as number of queues.

 */

int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,

		   const struct fun_dev_params *areq, const char *name)

{

	int rc;



	fdev->dev = &pdev->dev;

	rc = fun_map_bars(fdev, name);

	if (rc)

		return rc;



	rc = fun_set_dma_masks(fdev->dev);

	if (rc)

		goto unmap;



	rc = pci_enable_device_mem(pdev);

	if (rc) {

		dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);

		goto unmap;

	}



	pci_enable_pcie_error_reporting(pdev);



	rc = sanitize_dev(fdev);

	if (rc)

		goto disable_dev;



	fdev->fw_handle = FUN_HCI_ID_INVALID;

	fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;

	fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);

	fdev->dbs = fdev->bar + NVME_REG_DBS;



	INIT_WORK(&fdev->service_task, fun_serv_handler);

	fdev->service_flags = FUN_SERV_DISABLED;

	fdev->serv_cb = areq->serv_cb;



	rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */

	if (rc < 0)

		goto disable_dev;

	fdev->num_irqs = rc;



	rc = fun_alloc_irq_mgr(fdev);

	if (rc)

		goto free_irqs;



	pci_set_master(pdev);

	rc = fun_enable_admin_queue(fdev, areq);

	if (rc)

		goto free_irq_mgr;



	rc = fun_get_dev_limits(fdev);

	if (rc < 0)

		goto disable_admin;



	pci_save_state(pdev);

	pci_set_drvdata(pdev, fdev);

	pcie_print_link_status(pdev);

	dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",

		fdev->q_depth, fdev->db_stride, fdev->max_qid,

		fdev->kern_end_qid);

	return 0;



disable_admin:

	fun_disable_admin_queue(fdev);

free_irq_mgr:

	pci_clear_master(pdev);

	bitmap_free(fdev->irq_map);

free_irqs:

	pci_free_irq_vectors(pdev);

disable_dev:

	pci_disable_pcie_error_reporting(pdev);

	pci_disable_device(pdev);

unmap:

	fun_unmap_bars(fdev);

	return rc;

}

EXPORT_SYMBOL(fun_dev_enable);



MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");

MODULE_DESCRIPTION("Core services driver for Fungible devices");

MODULE_LICENSE("Dual BSD/GPL");

drivers/net/ethernet/fungible/funcore/fun_dev.h

0 → 100644
+150 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/fungible/funcore/fun_hci.h

0 → 100644
+1202 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/fungible/funcore/fun_queue.c

0 → 100644
+601 −0

File added.

Preview size limit exceeded, changes collapsed.