a252710fc5
flush_cache_range() wasn't page aligning the end of the range, we can't assume that it will always be page aligned, and we ended up getting unaligned faults in some rare call paths. Additionally, we add a small optimization to just purge the dcache entirely if the range is large enough that the page table walking will take longer. We use an arbitrary value of 64 pages for the large range size, as per sh64. Signed-off-by: Paul Mundt <lethal@linux-sh.org>
382 lines
8.7 KiB
C
382 lines
8.7 KiB
C
/*
|
|
* arch/sh/mm/cache-sh4.c
|
|
*
|
|
* Copyright (C) 1999, 2000, 2002 Niibe Yutaka
|
|
* Copyright (C) 2001, 2002, 2003, 2004, 2005 Paul Mundt
|
|
* Copyright (C) 2003 Richard Curnow
|
|
*
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
* for more details.
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/threads.h>
|
|
#include <asm/addrspace.h>
|
|
#include <asm/page.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/cache.h>
|
|
#include <asm/io.h>
|
|
#include <asm/uaccess.h>
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/cacheflush.h>
|
|
|
|
extern void __flush_cache_4096(unsigned long addr, unsigned long phys,
|
|
unsigned long exec_offset);
|
|
extern void __flush_cache_4096_all(unsigned long start);
|
|
static void __flush_cache_4096_all_ex(unsigned long start);
|
|
extern void __flush_dcache_all(void);
|
|
static void __flush_dcache_all_ex(void);
|
|
|
|
/*
|
|
* SH-4 has virtually indexed and physically tagged cache.
|
|
*/
|
|
|
|
struct semaphore p3map_sem[4];
|
|
|
|
void __init p3_cache_init(void)
|
|
{
|
|
if (remap_area_pages(P3SEG, 0, PAGE_SIZE*4, _PAGE_CACHABLE))
|
|
panic("%s failed.", __FUNCTION__);
|
|
|
|
sema_init (&p3map_sem[0], 1);
|
|
sema_init (&p3map_sem[1], 1);
|
|
sema_init (&p3map_sem[2], 1);
|
|
sema_init (&p3map_sem[3], 1);
|
|
}
|
|
|
|
/*
|
|
* Write back the dirty D-caches, but not invalidate them.
|
|
*
|
|
* START: Virtual Address (U0, P1, or P3)
|
|
* SIZE: Size of the region.
|
|
*/
|
|
void __flush_wback_region(void *start, int size)
|
|
{
|
|
unsigned long v;
|
|
unsigned long begin, end;
|
|
|
|
begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
|
|
end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
|
|
& ~(L1_CACHE_BYTES-1);
|
|
for (v = begin; v < end; v+=L1_CACHE_BYTES) {
|
|
asm volatile("ocbwb %0"
|
|
: /* no output */
|
|
: "m" (__m(v)));
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Write back the dirty D-caches and invalidate them.
|
|
*
|
|
* START: Virtual Address (U0, P1, or P3)
|
|
* SIZE: Size of the region.
|
|
*/
|
|
void __flush_purge_region(void *start, int size)
|
|
{
|
|
unsigned long v;
|
|
unsigned long begin, end;
|
|
|
|
begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
|
|
end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
|
|
& ~(L1_CACHE_BYTES-1);
|
|
for (v = begin; v < end; v+=L1_CACHE_BYTES) {
|
|
asm volatile("ocbp %0"
|
|
: /* no output */
|
|
: "m" (__m(v)));
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
* No write back please
|
|
*/
|
|
void __flush_invalidate_region(void *start, int size)
|
|
{
|
|
unsigned long v;
|
|
unsigned long begin, end;
|
|
|
|
begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
|
|
end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
|
|
& ~(L1_CACHE_BYTES-1);
|
|
for (v = begin; v < end; v+=L1_CACHE_BYTES) {
|
|
asm volatile("ocbi %0"
|
|
: /* no output */
|
|
: "m" (__m(v)));
|
|
}
|
|
}
|
|
|
|
static void __flush_dcache_all_ex(void)
|
|
{
|
|
unsigned long addr, end_addr, entry_offset;
|
|
|
|
end_addr = CACHE_OC_ADDRESS_ARRAY +
|
|
(cpu_data->dcache.sets << cpu_data->dcache.entry_shift) *
|
|
cpu_data->dcache.ways;
|
|
|
|
entry_offset = 1 << cpu_data->dcache.entry_shift;
|
|
for (addr = CACHE_OC_ADDRESS_ARRAY;
|
|
addr < end_addr;
|
|
addr += entry_offset) {
|
|
ctrl_outl(0, addr);
|
|
}
|
|
}
|
|
|
|
static void __flush_cache_4096_all_ex(unsigned long start)
|
|
{
|
|
unsigned long addr, entry_offset;
|
|
int i;
|
|
|
|
entry_offset = 1 << cpu_data->dcache.entry_shift;
|
|
for (i = 0; i < cpu_data->dcache.ways;
|
|
i++, start += cpu_data->dcache.way_incr) {
|
|
for (addr = CACHE_OC_ADDRESS_ARRAY + start;
|
|
addr < CACHE_OC_ADDRESS_ARRAY + 4096 + start;
|
|
addr += entry_offset) {
|
|
ctrl_outl(0, addr);
|
|
}
|
|
}
|
|
}
|
|
|
|
void flush_cache_4096_all(unsigned long start)
|
|
{
|
|
if (cpu_data->dcache.ways == 1)
|
|
__flush_cache_4096_all(start);
|
|
else
|
|
__flush_cache_4096_all_ex(start);
|
|
}
|
|
|
|
/*
|
|
* Write back the range of D-cache, and purge the I-cache.
|
|
*
|
|
* Called from kernel/module.c:sys_init_module and routine for a.out format.
|
|
*/
|
|
void flush_icache_range(unsigned long start, unsigned long end)
|
|
{
|
|
flush_cache_all();
|
|
}
|
|
|
|
/*
|
|
* Write back the D-cache and purge the I-cache for signal trampoline.
|
|
* .. which happens to be the same behavior as flush_icache_range().
|
|
* So, we simply flush out a line.
|
|
*/
|
|
void flush_cache_sigtramp(unsigned long addr)
|
|
{
|
|
unsigned long v, index;
|
|
unsigned long flags;
|
|
int i;
|
|
|
|
v = addr & ~(L1_CACHE_BYTES-1);
|
|
asm volatile("ocbwb %0"
|
|
: /* no output */
|
|
: "m" (__m(v)));
|
|
|
|
index = CACHE_IC_ADDRESS_ARRAY | (v & cpu_data->icache.entry_mask);
|
|
|
|
local_irq_save(flags);
|
|
jump_to_P2();
|
|
for (i = 0; i < cpu_data->icache.ways;
|
|
i++, index += cpu_data->icache.way_incr)
|
|
ctrl_outl(0, index); /* Clear out Valid-bit */
|
|
back_to_P1();
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
static inline void flush_cache_4096(unsigned long start,
|
|
unsigned long phys)
|
|
{
|
|
unsigned long flags;
|
|
|
|
/*
|
|
* SH7751, SH7751R, and ST40 have no restriction to handle cache.
|
|
* (While SH7750 must do that at P2 area.)
|
|
*/
|
|
if ((cpu_data->flags & CPU_HAS_P2_FLUSH_BUG)
|
|
|| start < CACHE_OC_ADDRESS_ARRAY) {
|
|
local_irq_save(flags);
|
|
__flush_cache_4096(start | SH_CACHE_ASSOC,
|
|
P1SEGADDR(phys), 0x20000000);
|
|
local_irq_restore(flags);
|
|
} else {
|
|
__flush_cache_4096(start | SH_CACHE_ASSOC,
|
|
P1SEGADDR(phys), 0);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Write back & invalidate the D-cache of the page.
|
|
* (To avoid "alias" issues)
|
|
*/
|
|
void flush_dcache_page(struct page *page)
|
|
{
|
|
if (test_bit(PG_mapped, &page->flags)) {
|
|
unsigned long phys = PHYSADDR(page_address(page));
|
|
|
|
/* Loop all the D-cache */
|
|
flush_cache_4096(CACHE_OC_ADDRESS_ARRAY, phys);
|
|
flush_cache_4096(CACHE_OC_ADDRESS_ARRAY | 0x1000, phys);
|
|
flush_cache_4096(CACHE_OC_ADDRESS_ARRAY | 0x2000, phys);
|
|
flush_cache_4096(CACHE_OC_ADDRESS_ARRAY | 0x3000, phys);
|
|
}
|
|
}
|
|
|
|
static inline void flush_icache_all(void)
|
|
{
|
|
unsigned long flags, ccr;
|
|
|
|
local_irq_save(flags);
|
|
jump_to_P2();
|
|
|
|
/* Flush I-cache */
|
|
ccr = ctrl_inl(CCR);
|
|
ccr |= CCR_CACHE_ICI;
|
|
ctrl_outl(ccr, CCR);
|
|
|
|
back_to_P1();
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
void flush_dcache_all(void)
|
|
{
|
|
if (cpu_data->dcache.ways == 1)
|
|
__flush_dcache_all();
|
|
else
|
|
__flush_dcache_all_ex();
|
|
}
|
|
|
|
void flush_cache_all(void)
|
|
{
|
|
flush_dcache_all();
|
|
flush_icache_all();
|
|
}
|
|
|
|
void flush_cache_mm(struct mm_struct *mm)
|
|
{
|
|
flush_cache_all();
|
|
}
|
|
|
|
/*
|
|
* Write back and invalidate I/D-caches for the page.
|
|
*
|
|
* ADDR: Virtual Address (U0 address)
|
|
* PFN: Physical page number
|
|
*/
|
|
void flush_cache_page(struct vm_area_struct *vma, unsigned long address, unsigned long pfn)
|
|
{
|
|
unsigned long phys = pfn << PAGE_SHIFT;
|
|
|
|
/* We only need to flush D-cache when we have alias */
|
|
if ((address^phys) & CACHE_ALIAS) {
|
|
/* Loop 4K of the D-cache */
|
|
flush_cache_4096(
|
|
CACHE_OC_ADDRESS_ARRAY | (address & CACHE_ALIAS),
|
|
phys);
|
|
/* Loop another 4K of the D-cache */
|
|
flush_cache_4096(
|
|
CACHE_OC_ADDRESS_ARRAY | (phys & CACHE_ALIAS),
|
|
phys);
|
|
}
|
|
|
|
if (vma->vm_flags & VM_EXEC)
|
|
/* Loop 4K (half) of the I-cache */
|
|
flush_cache_4096(
|
|
CACHE_IC_ADDRESS_ARRAY | (address & 0x1000),
|
|
phys);
|
|
}
|
|
|
|
/*
|
|
* Write back and invalidate D-caches.
|
|
*
|
|
* START, END: Virtual Address (U0 address)
|
|
*
|
|
* NOTE: We need to flush the _physical_ page entry.
|
|
* Flushing the cache lines for U0 only isn't enough.
|
|
* We need to flush for P1 too, which may contain aliases.
|
|
*/
|
|
void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
|
|
unsigned long end)
|
|
{
|
|
unsigned long p = start & PAGE_MASK;
|
|
pgd_t *dir;
|
|
pmd_t *pmd;
|
|
pud_t *pud;
|
|
pte_t *pte;
|
|
pte_t entry;
|
|
unsigned long phys;
|
|
unsigned long d = 0;
|
|
|
|
/*
|
|
* Don't bother with the lookup and alias check if we have a
|
|
* wide range to cover, just blow away the dcache in its
|
|
* entirety instead. -- PFM.
|
|
*/
|
|
if (((end - start) >> PAGE_SHIFT) >= 64) {
|
|
flush_dcache_all();
|
|
|
|
if (vma->vm_flags & VM_EXEC)
|
|
flush_icache_all();
|
|
|
|
return;
|
|
}
|
|
|
|
dir = pgd_offset(vma->vm_mm, p);
|
|
pud = pud_offset(dir, p);
|
|
pmd = pmd_offset(pud, p);
|
|
end = PAGE_ALIGN(end);
|
|
|
|
do {
|
|
if (pmd_none(*pmd) || pmd_bad(*pmd)) {
|
|
p &= ~((1 << PMD_SHIFT) -1);
|
|
p += (1 << PMD_SHIFT);
|
|
pmd++;
|
|
continue;
|
|
}
|
|
pte = pte_offset_kernel(pmd, p);
|
|
do {
|
|
entry = *pte;
|
|
if ((pte_val(entry) & _PAGE_PRESENT)) {
|
|
phys = pte_val(entry)&PTE_PHYS_MASK;
|
|
if ((p^phys) & CACHE_ALIAS) {
|
|
d |= 1 << ((p & CACHE_ALIAS)>>12);
|
|
d |= 1 << ((phys & CACHE_ALIAS)>>12);
|
|
if (d == 0x0f)
|
|
goto loop_exit;
|
|
}
|
|
}
|
|
pte++;
|
|
p += PAGE_SIZE;
|
|
} while (p < end && ((unsigned long)pte & ~PAGE_MASK));
|
|
pmd++;
|
|
} while (p < end);
|
|
loop_exit:
|
|
if (d & 1)
|
|
flush_cache_4096_all(0);
|
|
if (d & 2)
|
|
flush_cache_4096_all(0x1000);
|
|
if (d & 4)
|
|
flush_cache_4096_all(0x2000);
|
|
if (d & 8)
|
|
flush_cache_4096_all(0x3000);
|
|
if (vma->vm_flags & VM_EXEC)
|
|
flush_icache_all();
|
|
}
|
|
|
|
/*
|
|
* flush_icache_user_range
|
|
* @vma: VMA of the process
|
|
* @page: page
|
|
* @addr: U0 address
|
|
* @len: length of the range (< page size)
|
|
*/
|
|
void flush_icache_user_range(struct vm_area_struct *vma,
|
|
struct page *page, unsigned long addr, int len)
|
|
{
|
|
flush_cache_page(vma, addr, page_to_pfn(page));
|
|
}
|
|
|