cache.h

Go to the documentation of this file.

/*
 * Copyright 2022 Carlo Caione <[email protected]>
 *
 * SPDX-License-Identifier: Apache-2.0
 */
#ifndef ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_
#define ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_
 
#ifndef _ASMLANGUAGE
 
#include <zephyr/types.h>
#include <zephyr/sys/util.h>
#include <zephyr/sys/barrier.h>
#include <zephyr/arch/cpu.h>
#include <errno.h>
 
#ifdef __cplusplus
extern "C" {
#endif
 
#define K_CACHE_WB BIT(0)
#define K_CACHE_INVD BIT(1)
#define K_CACHE_WB_INVD (K_CACHE_WB | K_CACHE_INVD)
 
#if defined(CONFIG_DCACHE)
 
#define CTR_EL0_DMINLINE_SHIFT 16
#define CTR_EL0_DMINLINE_MASK BIT_MASK(4)
#define CTR_EL0_CWG_SHIFT 24
#define CTR_EL0_CWG_MASK BIT_MASK(4)
 
/* clidr_el1 */
#define CLIDR_EL1_LOC_SHIFT 24
#define CLIDR_EL1_LOC_MASK BIT_MASK(3)
#define CLIDR_EL1_CTYPE_SHIFT(level) ((level) * 3)
#define CLIDR_EL1_CTYPE_MASK BIT_MASK(3)
 
/* ccsidr_el1 */
#define CCSIDR_EL1_LN_SZ_SHIFT 0
#define CCSIDR_EL1_LN_SZ_MASK BIT_MASK(3)
#define CCSIDR_EL1_WAYS_SHIFT 3
#define CCSIDR_EL1_WAYS_MASK BIT_MASK(10)
#define CCSIDR_EL1_SETS_SHIFT 13
#define CCSIDR_EL1_SETS_MASK BIT_MASK(15)
 
#define dc_ops(op, val) \
({ \
 __asm__ volatile ("dc " op ", %0" :: "r" (val) : "memory"); \
})
 
static size_t dcache_line_size;
 
static ALWAYS_INLINE size_t arch_dcache_line_size_get(void)
{
 uint64_t ctr_el0;
 uint32_t dminline;
 
 if (dcache_line_size) {
 return dcache_line_size;
 }
 
 ctr_el0 = read_sysreg(CTR_EL0);
 
 dminline = (ctr_el0 >> CTR_EL0_DMINLINE_SHIFT) & CTR_EL0_DMINLINE_MASK;
 
 dcache_line_size = 4 << dminline;
 
 return dcache_line_size;
}
 
/*
 * operation for data cache by virtual address to PoC
 * ops: K_CACHE_INVD: invalidate
 * K_CACHE_WB: clean
 * K_CACHE_WB_INVD: clean and invalidate
 */
static ALWAYS_INLINE int arm64_dcache_range(void *addr, size_t size, int op)
{
 size_t line_size;
 uintptr_t start_addr = (uintptr_t)addr;
 uintptr_t end_addr = start_addr + size;
 
 if (op != K_CACHE_INVD && op != K_CACHE_WB && op != K_CACHE_WB_INVD) {
 return -ENOTSUP;
 }
 
 line_size = arch_dcache_line_size_get();
 
 /*
 * For the data cache invalidate operation, clean and invalidate
 * the partial cache lines at both ends of the given range to
 * prevent data corruption.
 *
 * For example (assume cache line size is 64 bytes):
 * There are 2 consecutive 32-byte buffers, which can be cached in
 * one line like below.
 * +------------------+------------------+
 * Cache line: | buffer 0 (dirty) | buffer 1 |
 * +------------------+------------------+
 * For the start address not aligned case, when invalidate the
 * buffer 1, the full cache line will be invalidated, if the buffer
 * 0 is dirty, its data will be lost.
 * The same logic applies to the not aligned end address.
 */
 if (op == K_CACHE_INVD) {
 if (end_addr & (line_size - 1)) {
 end_addr &= ~(line_size - 1);
 dc_ops("civac", end_addr);
 }
 
 if (start_addr & (line_size - 1)) {
 start_addr &= ~(line_size - 1);
 if (start_addr == end_addr) {
 goto done;
 }
 dc_ops("civac", start_addr);
 start_addr += line_size;
 }
 }
 
 /* Align address to line size */
 start_addr &= ~(line_size - 1);
 
 while (start_addr < end_addr) {
 if (op == K_CACHE_INVD) {
 dc_ops("ivac", start_addr);
 } else if (op == K_CACHE_WB) {
 dc_ops("cvac", start_addr);
 } else if (op == K_CACHE_WB_INVD) {
 dc_ops("civac", start_addr);
 }
 
 start_addr += line_size;
 }
 
done:
 barrier_dsync_fence_full();
 
 return 0;
}
 
static ALWAYS_INLINE int arch_dcache_flush_all(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_dcache_invd_all(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_dcache_flush_and_invd_all(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_dcache_flush_range(void *addr, size_t size)
{
 return arm64_dcache_range(addr, size, K_CACHE_WB);
}
 
static ALWAYS_INLINE int arch_dcache_invd_range(void *addr, size_t size)
{
 return arm64_dcache_range(addr, size, K_CACHE_INVD);
}
 
static ALWAYS_INLINE int arch_dcache_flush_and_invd_range(void *addr, size_t size)
{
 return arm64_dcache_range(addr, size, K_CACHE_WB_INVD);
}
 
static ALWAYS_INLINE void arch_dcache_enable(void)
{
 /* nothing */
}
 
static ALWAYS_INLINE void arch_dcache_disable(void)
{
 /* nothing */
}
 
#endif /* CONFIG_DCACHE */
 
#if defined(CONFIG_ICACHE)
 
static ALWAYS_INLINE size_t arch_icache_line_size_get(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_icache_flush_all(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_icache_invd_all(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_icache_flush_and_invd_all(void)
{
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_icache_flush_range(void *addr, size_t size)
{
 ARG_UNUSED(addr);
 ARG_UNUSED(size);
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_icache_invd_range(void *addr, size_t size)
{
 ARG_UNUSED(addr);
 ARG_UNUSED(size);
 return -ENOTSUP;
}
 
static ALWAYS_INLINE int arch_icache_flush_and_invd_range(void *addr, size_t size)
{
 ARG_UNUSED(addr);
 ARG_UNUSED(size);
 return -ENOTSUP;
}
 
static ALWAYS_INLINE void arch_icache_enable(void)
{
 /* nothing */
}
 
static ALWAYS_INLINE void arch_icache_disable(void)
{
 /* nothing */
}
 
#endif /* CONFIG_ICACHE */
 
#ifdef __cplusplus
}
#endif
 
#endif /* _ASMLANGUAGE */
 
#endif /* ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_ */