358 lines
12 KiB
C
358 lines
12 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2021 - Google LLC
|
|
* Author: David Brazdil <dbrazdil@google.com>
|
|
*/
|
|
|
|
#ifndef __ARM64_KVM_S2MPU_H__
|
|
#define __ARM64_KVM_S2MPU_H__
|
|
|
|
#include <linux/bitfield.h>
|
|
|
|
#include <asm/kvm_mmu.h>
|
|
|
|
#define S2MPU_MMIO_SIZE SZ_64K
|
|
#define SYSMMU_SYNC_MMIO_SIZE SZ_64K
|
|
#define SYSMMU_SYNC_S2_OFFSET SZ_32K
|
|
#define SYSMMU_SYNC_S2_MMIO_SIZE (SYSMMU_SYNC_MMIO_SIZE - \
|
|
SYSMMU_SYNC_S2_OFFSET)
|
|
|
|
#define NR_VIDS 8
|
|
#define NR_CTX_IDS 8
|
|
|
|
#define ALL_VIDS_BITMAP GENMASK(NR_VIDS - 1, 0)
|
|
|
|
#define REG_NS_CTRL0 0x0
|
|
#define REG_NS_CTRL1 0x4
|
|
#define REG_NS_CFG 0x10
|
|
#define REG_NS_INTERRUPT_ENABLE_PER_VID_SET 0x20
|
|
#define REG_NS_INTERRUPT_CLEAR 0x2c
|
|
#define REG_NS_VERSION 0x60
|
|
#define REG_NS_INFO 0x64
|
|
#define REG_NS_STATUS 0x68
|
|
#define REG_NS_NUM_CONTEXT 0x100
|
|
#define REG_NS_CONTEXT_CFG_VALID_VID 0x104
|
|
#define REG_NS_ALL_INVALIDATION 0x1000
|
|
#define REG_NS_RANGE_INVALIDATION 0x1020
|
|
#define REG_NS_RANGE_INVALIDATION_START_PPN 0x1024
|
|
#define REG_NS_RANGE_INVALIDATION_END_PPN 0x1028
|
|
#define REG_NS_FAULT_STATUS 0x2000
|
|
#define REG_NS_FAULT_PA_LOW(vid) (0x2004 + ((vid) * 0x20))
|
|
#define REG_NS_FAULT_PA_HIGH(vid) (0x2008 + ((vid) * 0x20))
|
|
#define REG_NS_FAULT_INFO(vid) (0x2010 + ((vid) * 0x20))
|
|
#define REG_NS_READ_MPTC 0x3000
|
|
#define REG_NS_READ_MPTC_TAG_PPN 0x3004
|
|
#define REG_NS_READ_MPTC_TAG_OTHERS 0x3008
|
|
#define REG_NS_READ_MPTC_DATA 0x3010
|
|
#define REG_NS_L1ENTRY_L2TABLE_ADDR(vid, gb) (0x4000 + ((vid) * 0x200) + ((gb) * 0x8))
|
|
#define REG_NS_L1ENTRY_ATTR(vid, gb) (0x4004 + ((vid) * 0x200) + ((gb) * 0x8))
|
|
|
|
#define CTRL0_ENABLE BIT(0)
|
|
#define CTRL0_INTERRUPT_ENABLE BIT(1)
|
|
#define CTRL0_FAULT_RESP_TYPE_SLVERR BIT(2) /* for v8 */
|
|
#define CTRL0_FAULT_RESP_TYPE_DECERR BIT(2) /* for v9 */
|
|
#define CTRL0_MASK (CTRL0_ENABLE | \
|
|
CTRL0_INTERRUPT_ENABLE | \
|
|
CTRL0_FAULT_RESP_TYPE_SLVERR | \
|
|
CTRL0_FAULT_RESP_TYPE_DECERR)
|
|
|
|
#define CTRL1_DISABLE_CHK_S1L1PTW BIT(0)
|
|
#define CTRL1_DISABLE_CHK_S1L2PTW BIT(1)
|
|
#define CTRL1_ENABLE_PAGE_SIZE_AWARENESS BIT(2)
|
|
#define CTRL1_DISABLE_CHK_USER_MATCHED_REQ BIT(3)
|
|
#define CTRL1_MASK (CTRL1_DISABLE_CHK_S1L1PTW | \
|
|
CTRL1_DISABLE_CHK_S1L2PTW | \
|
|
CTRL1_ENABLE_PAGE_SIZE_AWARENESS | \
|
|
CTRL1_DISABLE_CHK_USER_MATCHED_REQ)
|
|
|
|
#define CFG_MPTW_CACHE_OVERRIDE BIT(0)
|
|
#define CFG_MPTW_CACHE_VALUE GENMASK(7, 4)
|
|
#define CFG_MPTW_QOS_OVERRIDE BIT(8)
|
|
#define CFG_MPTW_QOS_VALUE GENMASK(15, 12)
|
|
#define CFG_MPTW_SHAREABLE BIT(16)
|
|
#define CFG_MASK (CFG_MPTW_CACHE_OVERRIDE | \
|
|
CFG_MPTW_CACHE_VALUE | \
|
|
CFG_MPTW_QOS_OVERRIDE | \
|
|
CFG_MPTW_QOS_VALUE | \
|
|
CFG_MPTW_SHAREABLE)
|
|
|
|
/* For use with hi_lo_readq_relaxed(). */
|
|
#define REG_NS_FAULT_PA_HIGH_LOW(vid) REG_NS_FAULT_PA_LOW(vid)
|
|
|
|
/* Mask used for extracting VID from FAULT_* register offset. */
|
|
#define REG_NS_FAULT_VID_MASK GENMASK(7, 5)
|
|
|
|
#define VERSION_MAJOR_ARCH_VER_MASK GENMASK(31, 28)
|
|
#define VERSION_MINOR_ARCH_VER_MASK GENMASK(27, 24)
|
|
#define VERSION_REV_ARCH_VER_MASK GENMASK(23, 16)
|
|
#define VERSION_RTL_VER_MASK GENMASK(7, 0)
|
|
|
|
/* Ignore RTL version in driver version check. */
|
|
#define VERSION_CHECK_MASK (VERSION_MAJOR_ARCH_VER_MASK | \
|
|
VERSION_MINOR_ARCH_VER_MASK | \
|
|
VERSION_REV_ARCH_VER_MASK)
|
|
|
|
#define INFO_NUM_SET_MASK GENMASK(15, 0)
|
|
|
|
#define STATUS_BUSY BIT(0)
|
|
#define STATUS_ON_INVALIDATING BIT(1)
|
|
|
|
#define NUM_CONTEXT_MASK GENMASK(3, 0)
|
|
|
|
#define CONTEXT_CFG_VALID_VID_CTX_VALID(ctx) BIT((4 * (ctx)) + 3)
|
|
#define CONTEXT_CFG_VALID_VID_CTX_VID(ctx, vid) \
|
|
FIELD_PREP(GENMASK((4 * (ctx) + 2), 4 * (ctx)), (vid))
|
|
|
|
#define INVALIDATION_INVALIDATE BIT(0)
|
|
#define RANGE_INVALIDATION_PPN_SHIFT 12
|
|
|
|
#define NR_FAULT_INFO_REGS 8
|
|
#define FAULT_INFO_VID_MASK GENMASK(26, 24)
|
|
#define FAULT_INFO_TYPE_MASK GENMASK(23, 21)
|
|
#define FAULT_INFO_TYPE_CONTEXT 0x4 /* v9 only */
|
|
#define FAULT_INFO_TYPE_AP 0x2
|
|
#define FAULT_INFO_TYPE_MPTW 0x1
|
|
#define FAULT_INFO_RW_BIT BIT(20)
|
|
#define FAULT_INFO_LEN_MASK GENMASK(19, 16)
|
|
#define FAULT_INFO_ID_MASK GENMASK(15, 0)
|
|
|
|
#define L1ENTRY_L2TABLE_ADDR_SHIFT 4
|
|
#define L1ENTRY_L2TABLE_ADDR(pa) ((pa) >> L1ENTRY_L2TABLE_ADDR_SHIFT)
|
|
|
|
#define READ_MPTC_WAY_MASK GENMASK(18, 16)
|
|
#define READ_MPTC_SET_MASK GENMASK(15, 0)
|
|
#define READ_MPTC_MASK (READ_MPTC_WAY_MASK | READ_MPTC_SET_MASK)
|
|
#define READ_MPTC_WAY(way) FIELD_PREP(READ_MPTC_WAY_MASK, (way))
|
|
#define READ_MPTC_SET(set) FIELD_PREP(READ_MPTC_SET_MASK, (set))
|
|
#define READ_MPTC(set, way) (READ_MPTC_SET(set) | READ_MPTC_WAY(way))
|
|
#define READ_MPTC_TAG_PPN_MASK GENMASK(23, 0)
|
|
#define READ_MPTC_TAG_OTHERS_VID_MASK GENMASK(10, 8)
|
|
#define READ_MPTC_TAG_OTHERS_GRAN_MASK GENMASK(5, 4)
|
|
#define READ_MPTC_TAG_OTHERS_VALID_BIT BIT(0)
|
|
#define READ_MPTC_TAG_OTHERS_MASK (READ_MPTC_TAG_OTHERS_VID_MASK | \
|
|
READ_MPTC_TAG_OTHERS_GRAN_MASK | \
|
|
READ_MPTC_TAG_OTHERS_VALID_BIT)
|
|
|
|
#define L1ENTRY_ATTR_L2TABLE_EN BIT(0)
|
|
#define L1ENTRY_ATTR_GRAN_4K 0x0
|
|
#define L1ENTRY_ATTR_GRAN_64K 0x1
|
|
#define L1ENTRY_ATTR_GRAN_2M 0x2
|
|
#define L1ENTRY_ATTR_PROT_MASK GENMASK(2, 1)
|
|
#define L1ENTRY_ATTR_GRAN_MASK GENMASK(5, 4)
|
|
#define L1ENTRY_ATTR_PROT(prot) FIELD_PREP(L1ENTRY_ATTR_PROT_MASK, prot)
|
|
#define L1ENTRY_ATTR_GRAN(gran) FIELD_PREP(L1ENTRY_ATTR_GRAN_MASK, gran)
|
|
#define L1ENTRY_ATTR_1G(prot) L1ENTRY_ATTR_PROT(prot)
|
|
#define L1ENTRY_ATTR_L2(gran) (L1ENTRY_ATTR_GRAN(gran) | \
|
|
L1ENTRY_ATTR_L2TABLE_EN)
|
|
|
|
#define NR_GIGABYTES 64
|
|
#define RO_GIGABYTES_FIRST 4
|
|
#define RO_GIGABYTES_LAST 33
|
|
#define NR_RO_GIGABYTES (RO_GIGABYTES_LAST - RO_GIGABYTES_FIRST + 1)
|
|
#define NR_RW_GIGABYTES (NR_GIGABYTES - NR_RO_GIGABYTES)
|
|
|
|
#ifdef CONFIG_ARM64_64K_PAGES
|
|
#define SMPT_GRAN SZ_64K
|
|
#define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_64K
|
|
#else
|
|
#define SMPT_GRAN SZ_4K
|
|
#define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_4K
|
|
#endif
|
|
static_assert(SMPT_GRAN <= PAGE_SIZE);
|
|
|
|
#define MPT_PROT_BITS 2
|
|
#define SMPT_WORD_SIZE sizeof(u32)
|
|
#define SMPT_ELEMS_PER_BYTE (BITS_PER_BYTE / MPT_PROT_BITS)
|
|
#define SMPT_ELEMS_PER_WORD (SMPT_WORD_SIZE * SMPT_ELEMS_PER_BYTE)
|
|
#define SMPT_WORD_BYTE_RANGE (SMPT_GRAN * SMPT_ELEMS_PER_WORD)
|
|
#define SMPT_NUM_ELEMS (SZ_1G / SMPT_GRAN)
|
|
#define SMPT_SIZE (SMPT_NUM_ELEMS / SMPT_ELEMS_PER_BYTE)
|
|
#define SMPT_NUM_WORDS (SMPT_SIZE / SMPT_WORD_SIZE)
|
|
#define SMPT_NUM_PAGES (SMPT_SIZE / PAGE_SIZE)
|
|
#define SMPT_ORDER get_order(SMPT_SIZE)
|
|
|
|
/* SysMMU_SYNC registers, relative to SYSMMU_SYNC_S2_OFFSET. */
|
|
#define REG_NS_SYNC_CMD 0x0
|
|
#define REG_NS_SYNC_COMP 0x4
|
|
|
|
#define SYNC_CMD_SYNC BIT(0)
|
|
#define SYNC_COMP_COMPLETE BIT(0)
|
|
|
|
/*
|
|
* Iterate over S2MPU gigabyte regions. Skip those that cannot be modified
|
|
* (the MMIO registers are read only, with reset value MPT_PROT_NONE).
|
|
*/
|
|
#define for_each_gb_in_range(i, first, last) \
|
|
for ((i) = (first); (i) <= (last) && (i) < NR_GIGABYTES; \
|
|
(i) = (((i) + 1 == RO_GIGABYTES_FIRST) ? RO_GIGABYTES_LAST : (i)) + 1)
|
|
|
|
#define for_each_gb(i) for_each_gb_in_range(i, 0, NR_GIGABYTES - 1)
|
|
#define for_each_vid(i) for ((i) = 0; (i) < NR_VIDS; (i)++)
|
|
#define for_each_gb_and_vid(gb, vid) for_each_vid((vid)) for_each_gb((gb))
|
|
|
|
enum s2mpu_version {
|
|
S2MPU_VERSION_8 = 0x11000000,
|
|
S2MPU_VERSION_9 = 0x20000000,
|
|
};
|
|
|
|
enum mpt_prot {
|
|
MPT_PROT_NONE = 0,
|
|
MPT_PROT_R = BIT(0),
|
|
MPT_PROT_W = BIT(1),
|
|
MPT_PROT_RW = MPT_PROT_R | MPT_PROT_W,
|
|
MPT_PROT_MASK = MPT_PROT_RW,
|
|
};
|
|
|
|
static const u64 mpt_prot_doubleword[] = {
|
|
[MPT_PROT_NONE] = 0x0000000000000000,
|
|
[MPT_PROT_R] = 0x5555555555555555,
|
|
[MPT_PROT_W] = 0xaaaaaaaaaaaaaaaa,
|
|
[MPT_PROT_RW] = 0xffffffffffffffff,
|
|
};
|
|
|
|
enum mpt_update_flags {
|
|
MPT_UPDATE_L1 = BIT(0),
|
|
MPT_UPDATE_L2 = BIT(1),
|
|
};
|
|
|
|
struct fmpt {
|
|
u32 *smpt;
|
|
bool gran_1g;
|
|
enum mpt_prot prot;
|
|
enum mpt_update_flags flags;
|
|
};
|
|
|
|
struct mpt {
|
|
struct fmpt fmpt[NR_GIGABYTES];
|
|
};
|
|
|
|
/* Set protection bits of SMPT in a given range without using memset. */
|
|
static inline void __set_smpt_range_slow(u32 *smpt, size_t start_gb_byte,
|
|
size_t end_gb_byte, enum mpt_prot prot)
|
|
{
|
|
size_t i, start_word_byte, end_word_byte, word_idx, first_elem, last_elem;
|
|
u32 val;
|
|
|
|
/* Iterate over u32 words. */
|
|
start_word_byte = start_gb_byte;
|
|
while (start_word_byte < end_gb_byte) {
|
|
/* Determine the range of bytes covered by this word. */
|
|
word_idx = start_word_byte / SMPT_WORD_BYTE_RANGE;
|
|
end_word_byte = min(
|
|
ALIGN(start_word_byte + 1, SMPT_WORD_BYTE_RANGE),
|
|
end_gb_byte);
|
|
|
|
/* Identify protection bit offsets within the word. */
|
|
first_elem = (start_word_byte / SMPT_GRAN) % SMPT_ELEMS_PER_WORD;
|
|
last_elem = ((end_word_byte - 1) / SMPT_GRAN) % SMPT_ELEMS_PER_WORD;
|
|
|
|
/* Modify the corresponding word. */
|
|
val = READ_ONCE(smpt[word_idx]);
|
|
for (i = first_elem; i <= last_elem; i++) {
|
|
val &= ~(MPT_PROT_MASK << (i * MPT_PROT_BITS));
|
|
val |= prot << (i * MPT_PROT_BITS);
|
|
}
|
|
WRITE_ONCE(smpt[word_idx], val);
|
|
|
|
start_word_byte = end_word_byte;
|
|
}
|
|
}
|
|
|
|
/* Set protection bits of SMPT in a given range. */
|
|
static inline void __set_smpt_range(u32 *smpt, size_t start_gb_byte,
|
|
size_t end_gb_byte, enum mpt_prot prot)
|
|
{
|
|
size_t interlude_start, interlude_end, interlude_bytes, word_idx;
|
|
char prot_byte = (char)mpt_prot_doubleword[prot];
|
|
|
|
if (start_gb_byte >= end_gb_byte)
|
|
return;
|
|
|
|
/* Check if range spans at least one full u32 word. */
|
|
interlude_start = ALIGN(start_gb_byte, SMPT_WORD_BYTE_RANGE);
|
|
interlude_end = ALIGN_DOWN(end_gb_byte, SMPT_WORD_BYTE_RANGE);
|
|
|
|
/* If not, fall back to editing bits in the given range. */
|
|
if (interlude_start >= interlude_end) {
|
|
__set_smpt_range_slow(smpt, start_gb_byte, end_gb_byte, prot);
|
|
return;
|
|
}
|
|
|
|
/* Use bit-editing for prologue/epilogue, memset for interlude. */
|
|
word_idx = interlude_start / SMPT_WORD_BYTE_RANGE;
|
|
interlude_bytes = (interlude_end - interlude_start) / SMPT_GRAN / SMPT_ELEMS_PER_BYTE;
|
|
|
|
__set_smpt_range_slow(smpt, start_gb_byte, interlude_start, prot);
|
|
memset(&smpt[word_idx], prot_byte, interlude_bytes);
|
|
__set_smpt_range_slow(smpt, interlude_end, end_gb_byte, prot);
|
|
}
|
|
|
|
/* Returns true if all SMPT protection bits match 'prot'. */
|
|
static inline bool __is_smpt_uniform(u32 *smpt, enum mpt_prot prot)
|
|
{
|
|
size_t i;
|
|
u64 *doublewords = (u64 *)smpt;
|
|
|
|
for (i = 0; i < SMPT_NUM_WORDS / 2; i++) {
|
|
if (doublewords[i] != mpt_prot_doubleword[prot])
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/*
|
|
* Set protection bits of FMPT/SMPT in a given range.
|
|
* Returns flags specifying whether L1/L2 changes need to be made visible
|
|
* to the device.
|
|
*/
|
|
static inline void __set_fmpt_range(struct fmpt *fmpt, size_t start_gb_byte,
|
|
size_t end_gb_byte, enum mpt_prot prot)
|
|
{
|
|
if (start_gb_byte == 0 && end_gb_byte >= SZ_1G) {
|
|
/* Update covers the entire GB region. */
|
|
if (fmpt->gran_1g && fmpt->prot == prot) {
|
|
fmpt->flags = 0;
|
|
return;
|
|
}
|
|
|
|
fmpt->gran_1g = true;
|
|
fmpt->prot = prot;
|
|
fmpt->flags = MPT_UPDATE_L1;
|
|
return;
|
|
}
|
|
|
|
if (fmpt->gran_1g) {
|
|
/* GB region currently uses 1G mapping. */
|
|
if (fmpt->prot == prot) {
|
|
fmpt->flags = 0;
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Range has different mapping than the rest of the GB.
|
|
* Convert to PAGE_SIZE mapping.
|
|
*/
|
|
fmpt->gran_1g = false;
|
|
__set_smpt_range(fmpt->smpt, 0, start_gb_byte, fmpt->prot);
|
|
__set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot);
|
|
__set_smpt_range(fmpt->smpt, end_gb_byte, SZ_1G, fmpt->prot);
|
|
fmpt->flags = MPT_UPDATE_L1 | MPT_UPDATE_L2;
|
|
return;
|
|
}
|
|
|
|
/* GB region currently uses PAGE_SIZE mapping. */
|
|
__set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot);
|
|
|
|
/* Check if the entire GB region has the same prot bits. */
|
|
if (!__is_smpt_uniform(fmpt->smpt, prot)) {
|
|
fmpt->flags = MPT_UPDATE_L2;
|
|
return;
|
|
}
|
|
|
|
fmpt->gran_1g = true;
|
|
fmpt->prot = prot;
|
|
fmpt->flags = MPT_UPDATE_L1;
|
|
}
|
|
|
|
#endif /* __ARM64_KVM_S2MPU_H__ */
|