arch/riscv/mm/cacheflush.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2017 SiFive
 */

#include <linux/acpi.h>
#include <linux/of.h>
#include <linux/prctl.h>
#include <asm/acpi.h>
#include <asm/cacheflush.h>

#ifdef CONFIG_SMP

#include <asm/sbi.h>

static void ipi_remote_fence_i(void *info)
{
	return local_flush_icache_all();
}

void flush_icache_all(void)
{
	local_flush_icache_all();

	if (num_online_cpus() < 2)
		return;

	/*
	 * Make sure all previous writes to the D$ are ordered before making
	 * the IPI. The RISC-V spec states that a hart must execute a data fence
	 * before triggering a remote fence.i in order to make the modification
	 * visible for remote harts.
	 *
	 * IPIs on RISC-V are triggered by MMIO writes to either CLINT or
	 * S-IMSIC, so the fence ensures previous data writes "happen before"
	 * the MMIO.
	 */
	RISCV_FENCE(w, o);

	if (riscv_use_sbi_for_rfence())
		sbi_remote_fence_i(NULL);
	else
		on_each_cpu(ipi_remote_fence_i, NULL, 1);
}
EXPORT_SYMBOL(flush_icache_all);

/*
 * Performs an icache flush for the given MM context.  RISC-V has no direct
 * mechanism for instruction cache shoot downs, so instead we send an IPI that
 * informs the remote harts they need to flush their local instruction caches.
 * To avoid pathologically slow behavior in a common case (a bunch of
 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
 * IPIs for harts that are not currently executing a MM context and instead
 * schedule a deferred local instruction cache flush to be performed before
 * execution resumes on each hart.
 */
void flush_icache_mm(struct mm_struct *mm, bool local)
{
	unsigned int cpu;
	cpumask_t others, *mask;

	preempt_disable();

	/* Mark every hart's icache as needing a flush for this MM. */
	mask = &mm->context.icache_stale_mask;
	cpumask_setall(mask);
	/* Flush this hart's I$ now, and mark it as flushed. */
	cpu = smp_processor_id();
	cpumask_clear_cpu(cpu, mask);
	local_flush_icache_all();

	/*
	 * Flush the I$ of other harts concurrently executing, and mark them as
	 * flushed.
	 */
	cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
	local |= cpumask_empty(&others);
	if (mm == current->active_mm && local) {
		/*
		 * It's assumed that at least one strongly ordered operation is
		 * performed on this hart between setting a hart's cpumask bit
		 * and scheduling this MM context on that hart.  Sending an SBI
		 * remote message will do this, but in the case where no
		 * messages are sent we still need to order this hart's writes
		 * with flush_icache_deferred().
		 */
		smp_mb();
	} else if (riscv_use_sbi_for_rfence()) {
		sbi_remote_fence_i(&others);
	} else {
		on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
	}

	preempt_enable();
}

#endif /* CONFIG_SMP */

#ifdef CONFIG_MMU
void flush_icache_pte(struct mm_struct *mm, pte_t pte)
{
	struct folio *folio = page_folio(pte_page(pte));

	if (!test_bit(PG_dcache_clean, &folio->flags.f)) {
		flush_icache_mm(mm, false);
		set_bit(PG_dcache_clean, &folio->flags.f);
	}
}
#endif /* CONFIG_MMU */

unsigned int riscv_cbom_block_size;
EXPORT_SYMBOL_GPL(riscv_cbom_block_size);

unsigned int riscv_cboz_block_size;
EXPORT_SYMBOL_GPL(riscv_cboz_block_size);

unsigned int riscv_cbop_block_size;
EXPORT_SYMBOL_GPL(riscv_cbop_block_size);

static void __init cbo_get_block_size(struct device_node *node,
				      const char *name, u32 *block_size,
				      unsigned long *first_hartid)
{
	unsigned long hartid;
	u32 val;

	if (riscv_of_processor_hartid(node, &hartid))
		return;

	if (of_property_read_u32(node, name, &val))
		return;

	if (!*block_size) {
		*block_size = val;
		*first_hartid = hartid;
	} else if (*block_size != val) {
		pr_warn("%s mismatched between harts %lu and %lu\n",
			name, *first_hartid, hartid);
	}
}

void __init riscv_init_cbo_blocksizes(void)
{
	unsigned long cbom_hartid, cboz_hartid, cbop_hartid;
	u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0;
	struct device_node *node;
	struct acpi_table_header *rhct;
	acpi_status status;

	if (acpi_disabled) {
		for_each_of_cpu_node(node) {
			/* set block-size for cbom and/or cboz extension if available */
			cbo_get_block_size(node, "riscv,cbom-block-size",
					   &cbom_block_size, &cbom_hartid);
			cbo_get_block_size(node, "riscv,cboz-block-size",
					   &cboz_block_size, &cboz_hartid);
			cbo_get_block_size(node, "riscv,cbop-block-size",
					   &cbop_block_size, &cbop_hartid);
		}
	} else {
		status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
		if (ACPI_FAILURE(status))
			return;

		acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size);
		acpi_put_table((struct acpi_table_header *)rhct);
	}

	if (cbom_block_size)
		riscv_cbom_block_size = cbom_block_size;

	if (cboz_block_size)
		riscv_cboz_block_size = cboz_block_size;

	if (cbop_block_size)
		riscv_cbop_block_size = cbop_block_size;
}

#ifdef CONFIG_SMP
static void set_icache_stale_mask(void)
{
	int cpu = get_cpu();
	cpumask_t *mask;
	bool stale_cpu;

	/*
	 * Mark every other hart's icache as needing a flush for
	 * this MM. Maintain the previous value of the current
	 * cpu to handle the case when this function is called
	 * concurrently on different harts.
	 */
	mask = &current->mm->context.icache_stale_mask;
	stale_cpu = cpumask_test_cpu(cpu, mask);

	cpumask_setall(mask);
	__assign_cpu(cpu, mask, stale_cpu);
	put_cpu();
}
#endif

/**
 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in
 * userspace.
 * @ctx: Set the type of icache flushing instructions permitted/prohibited in
 *	 userspace. Supported values described below.
 *
 * Supported values for ctx:
 *
 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space.
 *
 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in
 *   a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``.
 *   Therefore, caution must be taken; use this flag only when you can guarantee
 *   that no thread in the process will emit fence.i from this point onward.
 *
 * @scope: Set scope of where icache flushing instructions are allowed to be
 *	   emitted. Supported values described below.
 *
 * Supported values for scope:
 *
 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process
 *                               is coherent with instruction storage upon
 *                               migration.
 *
 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is
 *                              coherent with instruction storage upon
 *                              migration.
 *
 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are
 * permitted to emit icache flushing instructions. Whenever any thread in the
 * process is migrated, the corresponding hart's icache will be guaranteed to be
 * consistent with instruction storage. This does not enforce any guarantees
 * outside of migration. If a thread modifies an instruction that another thread
 * may attempt to execute, the other thread must still emit an icache flushing
 * instruction before attempting to execute the potentially modified
 * instruction. This must be performed by the user-space program.
 *
 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the
 * thread calling this function is permitted to emit icache flushing
 * instructions. When the thread is migrated, the corresponding hart's icache
 * will be guaranteed to be consistent with instruction storage.
 *
 * On kernels configured without SMP, this function is a nop as migrations
 * across harts will not occur.
 */
int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope)
{
#ifdef CONFIG_SMP
	switch (ctx) {
	case PR_RISCV_CTX_SW_FENCEI_ON:
		switch (scope) {
		case PR_RISCV_SCOPE_PER_PROCESS:
			current->mm->context.force_icache_flush = true;
			break;
		case PR_RISCV_SCOPE_PER_THREAD:
			current->thread.force_icache_flush = true;
			break;
		default:
			return -EINVAL;
		}
		break;
	case PR_RISCV_CTX_SW_FENCEI_OFF:
		switch (scope) {
		case PR_RISCV_SCOPE_PER_PROCESS:
			set_icache_stale_mask();
			current->mm->context.force_icache_flush = false;
			break;
		case PR_RISCV_SCOPE_PER_THREAD:
			set_icache_stale_mask();
			current->thread.force_icache_flush = false;
			break;
		default:
			return -EINVAL;
		}
		break;
	default:
		return -EINVAL;
	}
	return 0;
#else
	switch (ctx) {
	case PR_RISCV_CTX_SW_FENCEI_ON:
	case PR_RISCV_CTX_SW_FENCEI_OFF:
		return 0;
	default:
		return -EINVAL;
	}
#endif
}