blob: 965ba21818b15dd885694962e54aed2f26edbcc1 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0-only
/*
* Tegra host1x Interrupt Management
*
* Copyright (c) 2010-2013, NVIDIA Corporation.
*/
#include <linux/clk.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/irq.h>
#include <trace/events/host1x.h>
#include "channel.h"
#include "dev.h"
#include "fence.h"
#include "intr.h"
/* Wait list management */
enum waitlist_state {
WLS_PENDING,
WLS_REMOVED,
WLS_CANCELLED,
WLS_HANDLED
};
static void waiter_release(struct kref *kref)
{
kfree(container_of(kref, struct host1x_waitlist, refcount));
}
/*
* add a waiter to a waiter queue, sorted by threshold
* returns true if it was added at the head of the queue
*/
static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
struct list_head *queue)
{
struct host1x_waitlist *pos;
u32 thresh = waiter->thresh;
list_for_each_entry_reverse(pos, queue, list)
if ((s32)(pos->thresh - thresh) <= 0) {
list_add(&waiter->list, &pos->list);
return false;
}
list_add(&waiter->list, queue);
return true;
}
/*
* run through a waiter queue for a single sync point ID
* and gather all completed waiters into lists by actions
*/
static void remove_completed_waiters(struct list_head *head, u32 sync,
struct list_head completed[HOST1X_INTR_ACTION_COUNT])
{
struct list_head *dest;
struct host1x_waitlist *waiter, *next, *prev;
list_for_each_entry_safe(waiter, next, head, list) {
if ((s32)(waiter->thresh - sync) > 0)
break;
dest = completed + waiter->action;
/* consolidate submit cleanups */
if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
!list_empty(dest)) {
prev = list_entry(dest->prev,
struct host1x_waitlist, list);
if (prev->data == waiter->data) {
prev->count++;
dest = NULL;
}
}
/* PENDING->REMOVED or CANCELLED->HANDLED */
if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
list_del(&waiter->list);
kref_put(&waiter->refcount, waiter_release);
} else
list_move_tail(&waiter->list, dest);
}
}
static void reset_threshold_interrupt(struct host1x *host,
struct list_head *head,
unsigned int id)
{
u32 thresh =
list_first_entry(head, struct host1x_waitlist, list)->thresh;
host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
host1x_hw_intr_enable_syncpt_intr(host, id);
}
static void action_submit_complete(struct host1x_waitlist *waiter)
{
struct host1x_channel *channel = waiter->data;
host1x_cdma_update(&channel->cdma);
/* Add nr_completed to trace */
trace_host1x_channel_submit_complete(dev_name(channel->dev),
waiter->count, waiter->thresh);
}
static void action_wakeup(struct host1x_waitlist *waiter)
{
wait_queue_head_t *wq = waiter->data;
wake_up(wq);
}
static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
{
wait_queue_head_t *wq = waiter->data;
wake_up_interruptible(wq);
}
static void action_signal_fence(struct host1x_waitlist *waiter)
{
struct host1x_syncpt_fence *f = waiter->data;
host1x_fence_signal(f);
}
typedef void (*action_handler)(struct host1x_waitlist *waiter);
static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
action_submit_complete,
action_wakeup,
action_wakeup_interruptible,
action_signal_fence,
};
static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
{
struct list_head *head = completed;
unsigned int i;
for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
action_handler handler = action_handlers[i];
struct host1x_waitlist *waiter, *next;
list_for_each_entry_safe(waiter, next, head, list) {
list_del(&waiter->list);
handler(waiter);
WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
WLS_REMOVED);
kref_put(&waiter->refcount, waiter_release);
}
}
}
/*
* Remove & handle all waiters that have completed for the given syncpt
*/
static int process_wait_list(struct host1x *host,
struct host1x_syncpt *syncpt,
u32 threshold)
{
struct list_head completed[HOST1X_INTR_ACTION_COUNT];
unsigned int i;
int empty;
for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
INIT_LIST_HEAD(completed + i);
spin_lock(&syncpt->intr.lock);
remove_completed_waiters(&syncpt->intr.wait_head, threshold,
completed);
empty = list_empty(&syncpt->intr.wait_head);
if (empty)
host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
else
reset_threshold_interrupt(host, &syncpt->intr.wait_head,
syncpt->id);
spin_unlock(&syncpt->intr.lock);
run_handlers(completed);
return empty;
}
/*
* Sync point threshold interrupt service thread function
* Handles sync point threshold triggers, in thread context
*/
static void syncpt_thresh_work(struct work_struct *work)
{
struct host1x_syncpt_intr *syncpt_intr =
container_of(work, struct host1x_syncpt_intr, work);
struct host1x_syncpt *syncpt =
container_of(syncpt_intr, struct host1x_syncpt, intr);
unsigned int id = syncpt->id;
struct host1x *host = syncpt->host;
(void)process_wait_list(host, syncpt,
host1x_syncpt_load(host->syncpt + id));
}
int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
u32 thresh, enum host1x_intr_action action,
void *data, struct host1x_waitlist *waiter,
void **ref)
{
int queue_was_empty;
if (waiter == NULL) {
pr_warn("%s: NULL waiter\n", __func__);
return -EINVAL;
}
/* initialize a new waiter */
INIT_LIST_HEAD(&waiter->list);
kref_init(&waiter->refcount);
if (ref)
kref_get(&waiter->refcount);
waiter->thresh = thresh;
waiter->action = action;
atomic_set(&waiter->state, WLS_PENDING);
waiter->data = data;
waiter->count = 1;
spin_lock(&syncpt->intr.lock);
queue_was_empty = list_empty(&syncpt->intr.wait_head);
if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
/* added at head of list - new threshold value */
host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
/* added as first waiter - enable interrupt */
if (queue_was_empty)
host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
}
if (ref)
*ref = waiter;
spin_unlock(&syncpt->intr.lock);
return 0;
}
void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref,
bool flush)
{
struct host1x_waitlist *waiter = ref;
struct host1x_syncpt *syncpt;
atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);
syncpt = host->syncpt + id;
spin_lock(&syncpt->intr.lock);
if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==
WLS_CANCELLED) {
list_del(&waiter->list);
kref_put(&waiter->refcount, waiter_release);
}
spin_unlock(&syncpt->intr.lock);
if (flush) {
/* Wait until any concurrently executing handler has finished. */
while (atomic_read(&waiter->state) != WLS_HANDLED)
schedule();
}
kref_put(&waiter->refcount, waiter_release);
}
int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
{
unsigned int id;
u32 nb_pts = host1x_syncpt_nb_pts(host);
mutex_init(&host->intr_mutex);
host->intr_syncpt_irq = irq_sync;
for (id = 0; id < nb_pts; ++id) {
struct host1x_syncpt *syncpt = host->syncpt + id;
spin_lock_init(&syncpt->intr.lock);
INIT_LIST_HEAD(&syncpt->intr.wait_head);
snprintf(syncpt->intr.thresh_irq_name,
sizeof(syncpt->intr.thresh_irq_name),
"host1x_sp_%02u", id);
}
return 0;
}
void host1x_intr_deinit(struct host1x *host)
{
}
void host1x_intr_start(struct host1x *host)
{
u32 hz = clk_get_rate(host->clk);
int err;
mutex_lock(&host->intr_mutex);
err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
syncpt_thresh_work);
if (err) {
mutex_unlock(&host->intr_mutex);
return;
}
mutex_unlock(&host->intr_mutex);
}
void host1x_intr_stop(struct host1x *host)
{
unsigned int id;
struct host1x_syncpt *syncpt = host->syncpt;
u32 nb_pts = host1x_syncpt_nb_pts(host);
mutex_lock(&host->intr_mutex);
host1x_hw_intr_disable_all_syncpt_intrs(host);
for (id = 0; id < nb_pts; ++id) {
struct host1x_waitlist *waiter, *next;
list_for_each_entry_safe(waiter, next,
&syncpt[id].intr.wait_head, list) {
if (atomic_cmpxchg(&waiter->state,
WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
list_del(&waiter->list);
kref_put(&waiter->refcount, waiter_release);
}
}
if (!list_empty(&syncpt[id].intr.wait_head)) {
/* output diagnostics */
mutex_unlock(&host->intr_mutex);
pr_warn("%s cannot stop syncpt intr id=%u\n",
__func__, id);
return;
}
}
host1x_hw_intr_free_syncpt_irq(host);
mutex_unlock(&host->intr_mutex);
}