| // SPDX-License-Identifier: GPL-2.0 |
| /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ |
| #include <linux/init.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/pci.h> |
| #include <linux/io-64-nonatomic-lo-hi.h> |
| #include <linux/dmaengine.h> |
| #include <uapi/linux/idxd.h> |
| #include "../dmaengine.h" |
| #include "idxd.h" |
| #include "registers.h" |
| |
| void idxd_device_wqs_clear_state(struct idxd_device *idxd) |
| { |
| int i; |
| |
| lockdep_assert_held(&idxd->dev_lock); |
| for (i = 0; i < idxd->max_wqs; i++) { |
| struct idxd_wq *wq = &idxd->wqs[i]; |
| |
| wq->state = IDXD_WQ_DISABLED; |
| } |
| } |
| |
| static int idxd_restart(struct idxd_device *idxd) |
| { |
| int i, rc; |
| |
| lockdep_assert_held(&idxd->dev_lock); |
| |
| rc = __idxd_device_reset(idxd); |
| if (rc < 0) |
| goto out; |
| |
| rc = idxd_device_config(idxd); |
| if (rc < 0) |
| goto out; |
| |
| rc = idxd_device_enable(idxd); |
| if (rc < 0) |
| goto out; |
| |
| for (i = 0; i < idxd->max_wqs; i++) { |
| struct idxd_wq *wq = &idxd->wqs[i]; |
| |
| if (wq->state == IDXD_WQ_ENABLED) { |
| rc = idxd_wq_enable(wq); |
| if (rc < 0) { |
| dev_warn(&idxd->pdev->dev, |
| "Unable to re-enable wq %s\n", |
| dev_name(&wq->conf_dev)); |
| } |
| } |
| } |
| |
| return 0; |
| |
| out: |
| idxd_device_wqs_clear_state(idxd); |
| idxd->state = IDXD_DEV_HALTED; |
| return rc; |
| } |
| |
| irqreturn_t idxd_irq_handler(int vec, void *data) |
| { |
| struct idxd_irq_entry *irq_entry = data; |
| struct idxd_device *idxd = irq_entry->idxd; |
| |
| idxd_mask_msix_vector(idxd, irq_entry->id); |
| return IRQ_WAKE_THREAD; |
| } |
| |
| irqreturn_t idxd_misc_thread(int vec, void *data) |
| { |
| struct idxd_irq_entry *irq_entry = data; |
| struct idxd_device *idxd = irq_entry->idxd; |
| struct device *dev = &idxd->pdev->dev; |
| union gensts_reg gensts; |
| u32 cause, val = 0; |
| int i, rc; |
| bool err = false; |
| |
| cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); |
| |
| if (cause & IDXD_INTC_ERR) { |
| spin_lock_bh(&idxd->dev_lock); |
| for (i = 0; i < 4; i++) |
| idxd->sw_err.bits[i] = ioread64(idxd->reg_base + |
| IDXD_SWERR_OFFSET + i * sizeof(u64)); |
| iowrite64(IDXD_SWERR_ACK, idxd->reg_base + IDXD_SWERR_OFFSET); |
| |
| if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { |
| int id = idxd->sw_err.wq_idx; |
| struct idxd_wq *wq = &idxd->wqs[id]; |
| |
| if (wq->type == IDXD_WQT_USER) |
| wake_up_interruptible(&wq->idxd_cdev.err_queue); |
| } else { |
| int i; |
| |
| for (i = 0; i < idxd->max_wqs; i++) { |
| struct idxd_wq *wq = &idxd->wqs[i]; |
| |
| if (wq->type == IDXD_WQT_USER) |
| wake_up_interruptible(&wq->idxd_cdev.err_queue); |
| } |
| } |
| |
| spin_unlock_bh(&idxd->dev_lock); |
| val |= IDXD_INTC_ERR; |
| |
| for (i = 0; i < 4; i++) |
| dev_warn(dev, "err[%d]: %#16.16llx\n", |
| i, idxd->sw_err.bits[i]); |
| err = true; |
| } |
| |
| if (cause & IDXD_INTC_CMD) { |
| /* Driver does use command interrupts */ |
| val |= IDXD_INTC_CMD; |
| } |
| |
| if (cause & IDXD_INTC_OCCUPY) { |
| /* Driver does not utilize occupancy interrupt */ |
| val |= IDXD_INTC_OCCUPY; |
| } |
| |
| if (cause & IDXD_INTC_PERFMON_OVFL) { |
| /* |
| * Driver does not utilize perfmon counter overflow interrupt |
| * yet. |
| */ |
| val |= IDXD_INTC_PERFMON_OVFL; |
| } |
| |
| val ^= cause; |
| if (val) |
| dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", |
| val); |
| |
| iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); |
| if (!err) |
| return IRQ_HANDLED; |
| |
| gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); |
| if (gensts.state == IDXD_DEVICE_STATE_HALT) { |
| spin_lock_bh(&idxd->dev_lock); |
| if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { |
| rc = idxd_restart(idxd); |
| if (rc < 0) |
| dev_err(&idxd->pdev->dev, |
| "idxd restart failed, device halt."); |
| } else { |
| idxd_device_wqs_clear_state(idxd); |
| idxd->state = IDXD_DEV_HALTED; |
| dev_err(&idxd->pdev->dev, |
| "idxd halted, need %s.\n", |
| gensts.reset_type == IDXD_DEVICE_RESET_FLR ? |
| "FLR" : "system reset"); |
| } |
| spin_unlock_bh(&idxd->dev_lock); |
| } |
| |
| idxd_unmask_msix_vector(idxd, irq_entry->id); |
| return IRQ_HANDLED; |
| } |
| |
| static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, |
| int *processed) |
| { |
| struct idxd_desc *desc, *t; |
| struct llist_node *head; |
| int queued = 0; |
| |
| *processed = 0; |
| head = llist_del_all(&irq_entry->pending_llist); |
| if (!head) |
| return 0; |
| |
| llist_for_each_entry_safe(desc, t, head, llnode) { |
| if (desc->completion->status) { |
| idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); |
| idxd_free_desc(desc->wq, desc); |
| (*processed)++; |
| } else { |
| list_add_tail(&desc->list, &irq_entry->work_list); |
| queued++; |
| } |
| } |
| |
| return queued; |
| } |
| |
| static int irq_process_work_list(struct idxd_irq_entry *irq_entry, |
| int *processed) |
| { |
| struct list_head *node, *next; |
| int queued = 0; |
| |
| *processed = 0; |
| if (list_empty(&irq_entry->work_list)) |
| return 0; |
| |
| list_for_each_safe(node, next, &irq_entry->work_list) { |
| struct idxd_desc *desc = |
| container_of(node, struct idxd_desc, list); |
| |
| if (desc->completion->status) { |
| list_del(&desc->list); |
| /* process and callback */ |
| idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); |
| idxd_free_desc(desc->wq, desc); |
| (*processed)++; |
| } else { |
| queued++; |
| } |
| } |
| |
| return queued; |
| } |
| |
| static int idxd_desc_process(struct idxd_irq_entry *irq_entry) |
| { |
| int rc, processed, total = 0; |
| |
| /* |
| * There are two lists we are processing. The pending_llist is where |
| * submmiter adds all the submitted descriptor after sending it to |
| * the workqueue. It's a lockless singly linked list. The work_list |
| * is the common linux double linked list. We are in a scenario of |
| * multiple producers and a single consumer. The producers are all |
| * the kernel submitters of descriptors, and the consumer is the |
| * kernel irq handler thread for the msix vector when using threaded |
| * irq. To work with the restrictions of llist to remain lockless, |
| * we are doing the following steps: |
| * 1. Iterate through the work_list and process any completed |
| * descriptor. Delete the completed entries during iteration. |
| * 2. llist_del_all() from the pending list. |
| * 3. Iterate through the llist that was deleted from the pending list |
| * and process the completed entries. |
| * 4. If the entry is still waiting on hardware, list_add_tail() to |
| * the work_list. |
| * 5. Repeat until no more descriptors. |
| */ |
| do { |
| rc = irq_process_work_list(irq_entry, &processed); |
| total += processed; |
| if (rc != 0) |
| continue; |
| |
| rc = irq_process_pending_llist(irq_entry, &processed); |
| total += processed; |
| } while (rc != 0); |
| |
| return total; |
| } |
| |
| irqreturn_t idxd_wq_thread(int irq, void *data) |
| { |
| struct idxd_irq_entry *irq_entry = data; |
| int processed; |
| |
| processed = idxd_desc_process(irq_entry); |
| idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id); |
| /* catch anything unprocessed after unmasking */ |
| processed += idxd_desc_process(irq_entry); |
| |
| if (processed == 0) |
| return IRQ_NONE; |
| |
| return IRQ_HANDLED; |
| } |