* [RFC 0/6] IA64 Long Format VHPT support v2
@ 2006-05-10 3:42 Ian Wienand
2006-05-10 3:42 ` [RFC 1/6] LVHPT - Remove static ivt.S Ian Wienand
` (5 more replies)
0 siblings, 6 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
Hi,
Following from this message are patches to enable the Long Format VHPT
on IA64, version 2, which I am posting in the hope of community
review. They are against 2.6.17-rc3
The major changes
* Create ivt.S dynamically from include files
* Re-jig Kconfig text
* Split into more patches
* Split out most functions into lvhpt.c/lvhpt.h
* Split find_largest_page into separate patch
I think overall, this version should be clearer. I still have TODO's
related to mca handlers, but I would appreciate comments on this
updated version.
-i
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RFC 1/6] LVHPT - Remove static ivt.S
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
@ 2006-05-10 3:42 ` Ian Wienand
2006-05-10 3:42 ` [RFC 2/6] LVHPT - Create ivt.S dyanamically Ian Wienand
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
LVHPT remove ivt.S
Remove ivt.S; it is dynamically generated.
Signed-Off-By: Ian Wienand <ianw@gelato.unsw.edu.au>
---
ivt.S | 1726 ------------------------------------------------------------------
1 file changed, 1726 deletions(-)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt.S
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/kernel/ivt.S 2006-05-05 10:06:54.000000000 +1000
+++ /dev/null 1970-01-01 00:00:00.000000000 +0000
@@ -1,1726 +0,0 @@
-/*
- * arch/ia64/kernel/ivt.S
- *
- * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
- * Stephane Eranian <eranian@hpl.hp.com>
- * David Mosberger <davidm@hpl.hp.com>
- * Copyright (C) 2000, 2002-2003 Intel Co
- * Asit Mallick <asit.k.mallick@intel.com>
- * Suresh Siddha <suresh.b.siddha@intel.com>
- * Kenneth Chen <kenneth.w.chen@intel.com>
- * Fenghua Yu <fenghua.yu@intel.com>
- *
- * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
- * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
- */
-/*
- * This file defines the interruption vector table used by the CPU.
- * It does not include one entry per possible cause of interruption.
- *
- * The first 20 entries of the table contain 64 bundles each while the
- * remaining 48 entries contain only 16 bundles each.
- *
- * The 64 bundles are used to allow inlining the whole handler for critical
- * interruptions like TLB misses.
- *
- * For each entry, the comment is as follows:
- *
- * // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
- * entry offset ----/ / / / /
- * entry number ---------/ / / /
- * size of the entry -------------/ / /
- * vector name -------------------------------------/ /
- * interruptions triggering this vector ----------------------/
- *
- * The table is 32KB in size and must be aligned on 32KB boundary.
- * (The CPU ignores the 15 lower bits of the address)
- *
- * Table is based upon EAS2.6 (Oct 1999)
- */
-
-#include <linux/config.h>
-
-#include <asm/asmmacro.h>
-#include <asm/break.h>
-#include <asm/ia32.h>
-#include <asm/kregs.h>
-#include <asm/asm-offsets.h>
-#include <asm/pgtable.h>
-#include <asm/processor.h>
-#include <asm/ptrace.h>
-#include <asm/system.h>
-#include <asm/thread_info.h>
-#include <asm/unistd.h>
-#include <asm/errno.h>
-
-#if 1
-# define PSR_DEFAULT_BITS psr.ac
-#else
-# define PSR_DEFAULT_BITS 0
-#endif
-
-#if 0
- /*
- * This lets you track the last eight faults that occurred on the CPU. Make sure ar.k2 isn't
- * needed for something else before enabling this...
- */
-# define DBG_FAULT(i) mov r16=ar.k2;; shl r16=r16,8;; add r16=(i),r16;;mov ar.k2=r16
-#else
-# define DBG_FAULT(i)
-#endif
-
-#include "minstate.h"
-
-#define FAULT(n) \
- mov r31=pr; \
- mov r19=n;; /* prepare to save predicates */ \
- br.sptk.many dispatch_to_fault_handler
-
- .section .text.ivt,"ax"
-
- .align 32768 // align on 32KB boundary
- .global ia64_ivt
-ia64_ivt:
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
-ENTRY(vhpt_miss)
- DBG_FAULT(0)
- /*
- * The VHPT vector is invoked when the TLB entry for the virtual page table
- * is missing. This happens only as a result of a previous
- * (the "original") TLB miss, which may either be caused by an instruction
- * fetch or a data access (or non-access).
- *
- * What we do here is normal TLB miss handing for the _original_ miss,
- * followed by inserting the TLB entry for the virtual page table page
- * that the VHPT walker was attempting to access. The latter gets
- * inserted as long as page table entry above pte level have valid
- * mappings for the faulting address. The TLB entry for the original
- * miss gets inserted only if the pte entry indicates that the page is
- * present.
- *
- * do_page_fault gets invoked in the following cases:
- * - the faulting virtual address uses unimplemented address bits
- * - the faulting virtual address has no valid page table mapping
- */
- mov r16=cr.ifa // get address that caused the TLB miss
-#ifdef CONFIG_HUGETLB_PAGE
- movl r18=PAGE_SHIFT
- mov r25=cr.itir
-#endif
- ;;
- rsm psr.dt // use physical addressing for data
- mov r31=pr // save the predicate registers
- mov r19=IA64_KR(PT_BASE) // get page table base address
- shl r21=r16,3 // shift bit 60 into sign bit
- shr.u r17=r16,61 // get the region number into r17
- ;;
- shr.u r22=r21,3
-#ifdef CONFIG_HUGETLB_PAGE
- extr.u r26=r25,2,6
- ;;
- cmp.ne p8,p0=r18,r26
- sub r27=r26,r18
- ;;
-(p8) dep r25=r18,r25,2,6
-(p8) shr r22=r22,r27
-#endif
- ;;
- cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5?
- shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit
- ;;
-(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
-
- srlz.d
- LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
-
- .pred.rel "mutex", p6, p7
-(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
-(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
- ;;
-(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
-(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
- cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
-#ifdef CONFIG_PGTABLE_4
- shr.u r28=r22,PUD_SHIFT // shift pud index into position
-#else
- shr.u r18=r22,PMD_SHIFT // shift pmd index into position
-#endif
- ;;
- ld8 r17=[r17] // get *pgd (may be 0)
- ;;
-(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
-#ifdef CONFIG_PGTABLE_4
- dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
- ;;
- shr.u r18=r22,PMD_SHIFT // shift pmd index into position
-(p7) ld8 r29=[r28] // get *pud (may be 0)
- ;;
-(p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL?
- dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
-#else
- dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
-#endif
- ;;
-(p7) ld8 r20=[r17] // get *pmd (may be 0)
- shr.u r19=r22,PAGE_SHIFT // shift pte index into position
- ;;
-(p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL?
- dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
- ;;
-(p7) ld8 r18=[r21] // read *pte
- mov r19=cr.isr // cr.isr bit 32 tells us if this is an insn miss
- ;;
-(p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared?
- mov r22=cr.iha // get the VHPT address that caused the TLB miss
- ;; // avoid RAW on p7
-(p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss?
- dep r23=0,r20,0,PAGE_SHIFT // clear low bits to get page address
- ;;
-(p10) itc.i r18 // insert the instruction TLB entry
-(p11) itc.d r18 // insert the data TLB entry
-(p6) br.cond.spnt.many page_fault // handle bad address/page not present (page fault)
- mov cr.ifa=r22
-
-#ifdef CONFIG_HUGETLB_PAGE
-(p8) mov cr.itir=r25 // change to default page-size for VHPT
-#endif
-
- /*
- * Now compute and insert the TLB entry for the virtual page table. We never
- * execute in a page table page so there is no need to set the exception deferral
- * bit.
- */
- adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23
- ;;
-(p7) itc.d r24
- ;;
-#ifdef CONFIG_SMP
- /*
- * Tell the assemblers dependency-violation checker that the above "itc" instructions
- * cannot possibly affect the following loads:
- */
- dv_serialize_data
-
- /*
- * Re-check pagetable entry. If they changed, we may have received a ptc.g
- * between reading the pagetable and the "itc". If so, flush the entry we
- * inserted and retry. At this point, we have:
- *
- * r28 = equivalent of pud_offset(pgd, ifa)
- * r17 = equivalent of pmd_offset(pud, ifa)
- * r21 = equivalent of pte_offset(pmd, ifa)
- *
- * r29 = *pud
- * r20 = *pmd
- * r18 = *pte
- */
- ld8 r25=[r21] // read *pte again
- ld8 r26=[r17] // read *pmd again
-#ifdef CONFIG_PGTABLE_4
- ld8 r19=[r28] // read *pud again
-#endif
- cmp.ne p6,p7=r0,r0
- ;;
- cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change
-#ifdef CONFIG_PGTABLE_4
- cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change
-#endif
- mov r27=PAGE_SHIFT<<2
- ;;
-(p6) ptc.l r22,r27 // purge PTE page translation
-(p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change
- ;;
-(p6) ptc.l r16,r27 // purge translation
-#endif
-
- mov pr=r31,-1 // restore predicate registers
- rfi
-END(vhpt_miss)
-
- .org ia64_ivt+0x400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x0400 Entry 1 (size 64 bundles) ITLB (21)
-ENTRY(itlb_miss)
- DBG_FAULT(1)
- /*
- * The ITLB handler accesses the PTE via the virtually mapped linear
- * page table. If a nested TLB miss occurs, we switch into physical
- * mode, walk the page table, and then re-execute the PTE read and
- * go on normally after that.
- */
- mov r16=cr.ifa // get virtual address
- mov r29=b0 // save b0
- mov r31=pr // save predicates
-.itlb_fault:
- mov r17=cr.iha // get virtual address of PTE
- movl r30=1f // load nested fault continuation point
- ;;
-1: ld8 r18=[r17] // read *pte
- ;;
- mov b0=r29
- tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
-(p6) br.cond.spnt page_fault
- ;;
- itc.i r18
- ;;
-#ifdef CONFIG_SMP
- /*
- * Tell the assemblers dependency-violation checker that the above "itc" instructions
- * cannot possibly affect the following loads:
- */
- dv_serialize_data
-
- ld8 r19=[r17] // read *pte again and see if same
- mov r20=PAGE_SHIFT<<2 // setup page size for purge
- ;;
- cmp.ne p7,p0=r18,r19
- ;;
-(p7) ptc.l r16,r20
-#endif
- mov pr=r31,-1
- rfi
-END(itlb_miss)
-
- .org ia64_ivt+0x0800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
-ENTRY(dtlb_miss)
- DBG_FAULT(2)
- /*
- * The DTLB handler accesses the PTE via the virtually mapped linear
- * page table. If a nested TLB miss occurs, we switch into physical
- * mode, walk the page table, and then re-execute the PTE read and
- * go on normally after that.
- */
- mov r16=cr.ifa // get virtual address
- mov r29=b0 // save b0
- mov r31=pr // save predicates
-dtlb_fault:
- mov r17=cr.iha // get virtual address of PTE
- movl r30=1f // load nested fault continuation point
- ;;
-1: ld8 r18=[r17] // read *pte
- ;;
- mov b0=r29
- tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
-(p6) br.cond.spnt page_fault
- ;;
- itc.d r18
- ;;
-#ifdef CONFIG_SMP
- /*
- * Tell the assemblers dependency-violation checker that the above "itc" instructions
- * cannot possibly affect the following loads:
- */
- dv_serialize_data
-
- ld8 r19=[r17] // read *pte again and see if same
- mov r20=PAGE_SHIFT<<2 // setup page size for purge
- ;;
- cmp.ne p7,p0=r18,r19
- ;;
-(p7) ptc.l r16,r20
-#endif
- mov pr=r31,-1
- rfi
-END(dtlb_miss)
-
- .org ia64_ivt+0x0c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
-ENTRY(alt_itlb_miss)
- DBG_FAULT(3)
- mov r16=cr.ifa // get address that caused the TLB miss
- movl r17=PAGE_KERNEL
- mov r21=cr.ipsr
- movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
- mov r31=pr
- ;;
-#ifdef CONFIG_DISABLE_VHPT
- shr.u r22=r16,61 // get the region number into r21
- ;;
- cmp.gt p8,p0=6,r22 // user mode
- ;;
-(p8) thash r17=r16
- ;;
-(p8) mov cr.iha=r17
-(p8) mov r29=b0 // save b0
-(p8) br.cond.dptk .itlb_fault
-#endif
- extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
- and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
- shr.u r18=r16,57 // move address bit 61 to bit 4
- ;;
- andcm r18=0x10,r18 // bit 4=~address-bit(61)
- cmp.ne p8,p0=r0,r23 // psr.cpl != 0?
- or r19=r17,r19 // insert PTE control bits into r19
- ;;
- or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
-(p8) br.cond.spnt page_fault
- ;;
- itc.i r19 // insert the TLB entry
- mov pr=r31,-1
- rfi
-END(alt_itlb_miss)
-
- .org ia64_ivt+0x1000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
-ENTRY(alt_dtlb_miss)
- DBG_FAULT(4)
- mov r16=cr.ifa // get address that caused the TLB miss
- movl r17=PAGE_KERNEL
- mov r20=cr.isr
- movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
- mov r21=cr.ipsr
- mov r31=pr
- ;;
-#ifdef CONFIG_DISABLE_VHPT
- shr.u r22=r16,61 // get the region number into r21
- ;;
- cmp.gt p8,p0=6,r22 // access to region 0-5
- ;;
-(p8) thash r17=r16
- ;;
-(p8) mov cr.iha=r17
-(p8) mov r29=b0 // save b0
-(p8) br.cond.dptk dtlb_fault
-#endif
- extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
- and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field
- tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on?
- shr.u r18=r16,57 // move address bit 61 to bit 4
- and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
- tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
- ;;
- andcm r18=0x10,r18 // bit 4=~address-bit(61)
- cmp.ne p8,p0=r0,r23
-(p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field
-(p8) br.cond.spnt page_fault
-
- dep r21=-1,r21,IA64_PSR_ED_BIT,1
- or r19=r19,r17 // insert PTE control bits into r19
- ;;
- or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
-(p6) mov cr.ipsr=r21
- ;;
-(p7) itc.d r19 // insert the TLB entry
- mov pr=r31,-1
- rfi
-END(alt_dtlb_miss)
-
- .org ia64_ivt+0x1400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
-ENTRY(nested_dtlb_miss)
- /*
- * In the absence of kernel bugs, we get here when the virtually mapped linear
- * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
- * Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page
- * table is missing, a nested TLB miss fault is triggered and control is
- * transferred to this point. When this happens, we lookup the pte for the
- * faulting address by walking the page table in physical mode and return to the
- * continuation point passed in register r30 (or call page_fault if the address is
- * not mapped).
- *
- * Input: r16: faulting address
- * r29: saved b0
- * r30: continuation address
- * r31: saved pr
- *
- * Output: r17: physical address of PTE of faulting address
- * r29: saved b0
- * r30: continuation address
- * r31: saved pr
- *
- * Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared)
- */
- rsm psr.dt // switch to using physical data addressing
- mov r19=IA64_KR(PT_BASE) // get the page table base address
- shl r21=r16,3 // shift bit 60 into sign bit
- mov r18=cr.itir
- ;;
- shr.u r17=r16,61 // get the region number into r17
- extr.u r18=r18,2,6 // get the faulting page size
- ;;
- cmp.eq p6,p7=5,r17 // is faulting address in region 5?
- add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
- add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
- ;;
- shr.u r22=r16,r22
- shr.u r18=r16,r18
-(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
-
- srlz.d
- LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
-
- .pred.rel "mutex", p6, p7
-(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
-(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
- ;;
-(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
-(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
- cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
-#ifdef CONFIG_PGTABLE_4
- shr.u r18=r22,PUD_SHIFT // shift pud index into position
-#else
- shr.u r18=r22,PMD_SHIFT // shift pmd index into position
-#endif
- ;;
- ld8 r17=[r17] // get *pgd (may be 0)
- ;;
-(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
- dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
- ;;
-#ifdef CONFIG_PGTABLE_4
-(p7) ld8 r17=[r17] // get *pud (may be 0)
- shr.u r18=r22,PMD_SHIFT // shift pmd index into position
- ;;
-(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
- dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
- ;;
-#endif
-(p7) ld8 r17=[r17] // get *pmd (may be 0)
- shr.u r19=r22,PAGE_SHIFT // shift pte index into position
- ;;
-(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
- dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
-(p6) br.cond.spnt page_fault
- mov b0=r30
- br.sptk.many b0 // return to continuation point
-END(nested_dtlb_miss)
-
- .org ia64_ivt+0x1800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
-ENTRY(ikey_miss)
- DBG_FAULT(6)
- FAULT(6)
-END(ikey_miss)
-
- //-----------------------------------------------------------------------------------
- // call do_page_fault (predicates are in r31, psr.dt may be off, r16 is faulting address)
-ENTRY(page_fault)
- ssm psr.dt
- ;;
- srlz.i
- ;;
- SAVE_MIN_WITH_COVER
- alloc r15=ar.pfs,0,0,3,0
- mov out0=cr.ifa
- mov out1=cr.isr
- adds r3=8,r2 // set up second base pointer
- ;;
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- srlz.i // guarantee that interruption collectin is on
- ;;
-(p15) ssm psr.i // restore psr.i
- movl r14=ia64_leave_kernel
- ;;
- SAVE_REST
- mov rp=r14
- ;;
- adds out2=16,r12 // out2 = pointer to pt_regs
- br.call.sptk.many b6=ia64_do_page_fault // ignore return address
-END(page_fault)
-
- .org ia64_ivt+0x1c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
-ENTRY(dkey_miss)
- DBG_FAULT(7)
- FAULT(7)
-END(dkey_miss)
-
- .org ia64_ivt+0x2000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
-ENTRY(dirty_bit)
- DBG_FAULT(8)
- /*
- * What we do here is to simply turn on the dirty bit in the PTE. We need to
- * update both the page-table and the TLB entry. To efficiently access the PTE,
- * we address it through the virtual page table. Most likely, the TLB entry for
- * the relevant virtual page table page is still present in the TLB so we can
- * normally do this without additional TLB misses. In case the necessary virtual
- * page table TLB entry isn't present, we take a nested TLB miss hit where we look
- * up the physical address of the L3 PTE and then continue at label 1 below.
- */
- mov r16=cr.ifa // get the address that caused the fault
- movl r30=1f // load continuation point in case of nested fault
- ;;
- thash r17=r16 // compute virtual address of L3 PTE
- mov r29=b0 // save b0 in case of nested fault
- mov r31=pr // save pr
-#ifdef CONFIG_SMP
- mov r28=ar.ccv // save ar.ccv
- ;;
-1: ld8 r18=[r17]
- ;; // avoid RAW on r18
- mov ar.ccv=r18 // set compare value for cmpxchg
- or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
- tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
- ;;
-(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
- mov r24=PAGE_SHIFT<<2
- ;;
-(p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
- ;;
-(p6) itc.d r25 // install updated PTE
- ;;
- /*
- * Tell the assemblers dependency-violation checker that the above "itc" instructions
- * cannot possibly affect the following loads:
- */
- dv_serialize_data
-
- ld8 r18=[r17] // read PTE again
- ;;
- cmp.eq p6,p7=r18,r25 // is it same as the newly installed
- ;;
-(p7) ptc.l r16,r24
- mov b0=r29 // restore b0
- mov ar.ccv=r28
-#else
- ;;
-1: ld8 r18=[r17]
- ;; // avoid RAW on r18
- or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
- mov b0=r29 // restore b0
- ;;
- st8 [r17]=r18 // store back updated PTE
- itc.d r18 // install updated PTE
-#endif
- mov pr=r31,-1 // restore pr
- rfi
-END(dirty_bit)
-
- .org ia64_ivt+0x2400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
-ENTRY(iaccess_bit)
- DBG_FAULT(9)
- // Like Entry 8, except for instruction access
- mov r16=cr.ifa // get the address that caused the fault
- movl r30=1f // load continuation point in case of nested fault
- mov r31=pr // save predicates
-#ifdef CONFIG_ITANIUM
- /*
- * Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
- */
- mov r17=cr.ipsr
- ;;
- mov r18=cr.iip
- tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set?
- ;;
-(p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa
-#endif /* CONFIG_ITANIUM */
- ;;
- thash r17=r16 // compute virtual address of L3 PTE
- mov r29=b0 // save b0 in case of nested fault)
-#ifdef CONFIG_SMP
- mov r28=ar.ccv // save ar.ccv
- ;;
-1: ld8 r18=[r17]
- ;;
- mov ar.ccv=r18 // set compare value for cmpxchg
- or r25=_PAGE_A,r18 // set the accessed bit
- tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
- ;;
-(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
- mov r24=PAGE_SHIFT<<2
- ;;
-(p6) cmp.eq p6,p7=r26,r18 // Only if page present
- ;;
-(p6) itc.i r25 // install updated PTE
- ;;
- /*
- * Tell the assemblers dependency-violation checker that the above "itc" instructions
- * cannot possibly affect the following loads:
- */
- dv_serialize_data
-
- ld8 r18=[r17] // read PTE again
- ;;
- cmp.eq p6,p7=r18,r25 // is it same as the newly installed
- ;;
-(p7) ptc.l r16,r24
- mov b0=r29 // restore b0
- mov ar.ccv=r28
-#else /* !CONFIG_SMP */
- ;;
-1: ld8 r18=[r17]
- ;;
- or r18=_PAGE_A,r18 // set the accessed bit
- mov b0=r29 // restore b0
- ;;
- st8 [r17]=r18 // store back updated PTE
- itc.i r18 // install updated PTE
-#endif /* !CONFIG_SMP */
- mov pr=r31,-1
- rfi
-END(iaccess_bit)
-
- .org ia64_ivt+0x2800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
-ENTRY(daccess_bit)
- DBG_FAULT(10)
- // Like Entry 8, except for data access
- mov r16=cr.ifa // get the address that caused the fault
- movl r30=1f // load continuation point in case of nested fault
- ;;
- thash r17=r16 // compute virtual address of L3 PTE
- mov r31=pr
- mov r29=b0 // save b0 in case of nested fault)
-#ifdef CONFIG_SMP
- mov r28=ar.ccv // save ar.ccv
- ;;
-1: ld8 r18=[r17]
- ;; // avoid RAW on r18
- mov ar.ccv=r18 // set compare value for cmpxchg
- or r25=_PAGE_A,r18 // set the dirty bit
- tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
- ;;
-(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present
- mov r24=PAGE_SHIFT<<2
- ;;
-(p6) cmp.eq p6,p7=r26,r18 // Only if page is present
- ;;
-(p6) itc.d r25 // install updated PTE
- /*
- * Tell the assemblers dependency-violation checker that the above "itc" instructions
- * cannot possibly affect the following loads:
- */
- dv_serialize_data
- ;;
- ld8 r18=[r17] // read PTE again
- ;;
- cmp.eq p6,p7=r18,r25 // is it same as the newly installed
- ;;
-(p7) ptc.l r16,r24
- mov ar.ccv=r28
-#else
- ;;
-1: ld8 r18=[r17]
- ;; // avoid RAW on r18
- or r18=_PAGE_A,r18 // set the accessed bit
- ;;
- st8 [r17]=r18 // store back updated PTE
- itc.d r18 // install updated PTE
-#endif
- mov b0=r29 // restore b0
- mov pr=r31,-1
- rfi
-END(daccess_bit)
-
- .org ia64_ivt+0x2c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
-ENTRY(break_fault)
- /*
- * The streamlined system call entry/exit paths only save/restore the initial part
- * of pt_regs. This implies that the callers of system-calls must adhere to the
- * normal procedure calling conventions.
- *
- * Registers to be saved & restored:
- * CR registers: cr.ipsr, cr.iip, cr.ifs
- * AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr
- * others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15
- * Registers to be restored only:
- * r8-r11: output value from the system call.
- *
- * During system call exit, scratch registers (including r15) are modified/cleared
- * to prevent leaking bits from kernel to user level.
- */
- DBG_FAULT(11)
- mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
- mov r29=cr.ipsr // M2 (12 cyc)
- mov r31=pr // I0 (2 cyc)
-
- mov r17=cr.iim // M2 (2 cyc)
- mov.m r27=ar.rsc // M2 (12 cyc)
- mov r18=__IA64_BREAK_SYSCALL // A
-
- mov.m ar.rsc=0 // M2
- mov.m r21=ar.fpsr // M2 (12 cyc)
- mov r19=b6 // I0 (2 cyc)
- ;;
- mov.m r23=ar.bspstore // M2 (12 cyc)
- mov.m r24=ar.rnat // M2 (5 cyc)
- mov.i r26=ar.pfs // I0 (2 cyc)
-
- invala // M0|1
- nop.m 0 // M
- mov r20=r1 // A save r1
-
- nop.m 0
- movl r30=sys_call_table // X
-
- mov r28=cr.iip // M2 (2 cyc)
- cmp.eq p0,p7=r18,r17 // I0 is this a system call?
-(p7) br.cond.spnt non_syscall // B no ->
- //
- // From this point on, we are definitely on the syscall-path
- // and we can use (non-banked) scratch registers.
- //
-///////////////////////////////////////////////////////////////////////
- mov r1=r16 // A move task-pointer to "addl"-addressable reg
- mov r2=r16 // A setup r2 for ia64_syscall_setup
- add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags
-
- adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
- adds r15=-1024,r15 // A subtract 1024 from syscall number
- mov r3=NR_syscalls - 1
- ;;
- ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
- ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
- extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr
-
- shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
- addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS
- cmp.leu p6,p7=r15,r3 // A syscall number in range?
- ;;
-
- lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS
-(p6) ld8 r30=[r30] // M0|1 load address of syscall entry point
- tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT?
-
- mov.m ar.bspstore=r22 // M2 switch to kernel RBS
- cmp.eq p8,p9=2,r8 // A isr.ei==2?
- ;;
-
-(p8) mov r8=0 // A clear ei to 0
-(p7) movl r30=sys_ni_syscall // X
-
-(p8) adds r28=16,r28 // A switch cr.iip to next bundle
-(p9) adds r8=1,r8 // A increment ei to next slot
- nop.i 0
- ;;
-
- mov.m r25=ar.unat // M2 (5 cyc)
- dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr
- adds r15=1024,r15 // A restore original syscall number
- //
- // If any of the above loads miss in L1D, we'll stall here until
- // the data arrives.
- //
-///////////////////////////////////////////////////////////////////////
- st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
- mov b6=r30 // I0 setup syscall handler branch reg early
- cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already?
-
- and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit
- mov r18=ar.bsp // M2 (12 cyc)
-(pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
- ;;
-.back_from_break_fixup:
-(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
- cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
- br.call.sptk.many b7=ia64_syscall_setup // B
-1:
- mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
- nop 0
- bsw.1 // B (6 cyc) regs are saved, switch to bank 1
- ;;
-
- ssm psr.ic | PSR_DEFAULT_BITS // M2 now it's safe to re-enable intr.-collection
- movl r3=ia64_ret_from_syscall // X
- ;;
-
- srlz.i // M0 ensure interruption collection is on
- mov rp=r3 // I0 set the real return addr
-(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
-
-(p15) ssm psr.i // M2 restore psr.i
-(p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
- br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
- // NOT REACHED
-///////////////////////////////////////////////////////////////////////
- // On entry, we optimistically assumed that we're coming from user-space.
- // For the rare cases where a system-call is done from within the kernel,
- // we fix things up at this point:
-.break_fixup:
- add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
- mov ar.rnat=r24 // M2 restore kernel's AR.RNAT
- ;;
- mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE
- br.cond.sptk .back_from_break_fixup
-END(break_fault)
-
- .org ia64_ivt+0x3000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
-ENTRY(interrupt)
- DBG_FAULT(12)
- mov r31=pr // prepare to save predicates
- ;;
- SAVE_MIN_WITH_COVER // uses r31; defines r2 and r3
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- adds r3=8,r2 // set up second base pointer for SAVE_REST
- srlz.i // ensure everybody knows psr.ic is back on
- ;;
- SAVE_REST
- ;;
- MCA_RECOVER_RANGE(interrupt)
- alloc r14=ar.pfs,0,0,2,0 // must be first in an insn group
- mov out0=cr.ivr // pass cr.ivr as first arg
- add out1=16,sp // pass pointer to pt_regs as second arg
- ;;
- srlz.d // make sure we see the effect of cr.ivr
- movl r14=ia64_leave_kernel
- ;;
- mov rp=r14
- br.call.sptk.many b6=ia64_handle_irq
-END(interrupt)
-
- .org ia64_ivt+0x3400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x3400 Entry 13 (size 64 bundles) Reserved
- DBG_FAULT(13)
- FAULT(13)
-
- .org ia64_ivt+0x3800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x3800 Entry 14 (size 64 bundles) Reserved
- DBG_FAULT(14)
- FAULT(14)
-
- /*
- * There is no particular reason for this code to be here, other than that
- * there happens to be space here that would go unused otherwise. If this
- * fault ever gets "unreserved", simply moved the following code to a more
- * suitable spot...
- *
- * ia64_syscall_setup() is a separate subroutine so that it can
- * allocate stacked registers so it can safely demine any
- * potential NaT values from the input registers.
- *
- * On entry:
- * - executing on bank 0 or bank 1 register set (doesn't matter)
- * - r1: stack pointer
- * - r2: current task pointer
- * - r3: preserved
- * - r11: original contents (saved ar.pfs to be saved)
- * - r12: original contents (sp to be saved)
- * - r13: original contents (tp to be saved)
- * - r15: original contents (syscall # to be saved)
- * - r18: saved bsp (after switching to kernel stack)
- * - r19: saved b6
- * - r20: saved r1 (gp)
- * - r21: saved ar.fpsr
- * - r22: kernel's register backing store base (krbs_base)
- * - r23: saved ar.bspstore
- * - r24: saved ar.rnat
- * - r25: saved ar.unat
- * - r26: saved ar.pfs
- * - r27: saved ar.rsc
- * - r28: saved cr.iip
- * - r29: saved cr.ipsr
- * - r31: saved pr
- * - b0: original contents (to be saved)
- * On exit:
- * - p10: TRUE if syscall is invoked with more than 8 out
- * registers or r15's Nat is true
- * - r1: kernel's gp
- * - r3: preserved (same as on entry)
- * - r8: -EINVAL if p10 is true
- * - r12: points to kernel stack
- * - r13: points to current task
- * - r14: preserved (same as on entry)
- * - p13: preserved
- * - p15: TRUE if interrupts need to be re-enabled
- * - ar.fpsr: set to kernel settings
- * - b6: preserved (same as on entry)
- */
-GLOBAL_ENTRY(ia64_syscall_setup)
-#if PT(B6) != 0
-# error This code assumes that b6 is the first field in pt_regs.
-#endif
- st8 [r1]=r19 // save b6
- add r16=PT(CR_IPSR),r1 // initialize first base pointer
- add r17=PT(R11),r1 // initialize second base pointer
- ;;
- alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable
- st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr
- tnat.nz p8,p0=in0
-
- st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11
- tnat.nz p9,p0=in1
-(pKStk) mov r18=r0 // make sure r18 isn't NaT
- ;;
-
- st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs
- st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip
- mov r28=b0 // save b0 (2 cyc)
- ;;
-
- st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat
- dep r19=0,r19,38,26 // clear all bits but 0..37 [I0]
-(p8) mov in0=-1
- ;;
-
- st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs
- extr.u r11=r19,7,7 // I0 // get sol of ar.pfs
- and r8=0x7f,r19 // A // get sof of ar.pfs
-
- st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
- tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0
-(p9) mov in1=-1
- ;;
-
-(pUStk) sub r18=r18,r22 // r18=RSE.ndirty*8
- tnat.nz p10,p0=in2
- add r11=8,r11
- ;;
-(pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field
-(pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field
- tnat.nz p11,p0=in3
- ;;
-(p10) mov in2=-1
- tnat.nz p12,p0=in4 // [I0]
-(p11) mov in3=-1
- ;;
-(pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat
-(pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore
- shl r18=r18,16 // compute ar.rsc to be used for "loadrs"
- ;;
- st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates
- st8 [r17]=r28,PT(R1)-PT(B0) // save b0
- tnat.nz p13,p0=in5 // [I0]
- ;;
- st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs"
- st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1
-(p12) mov in4=-1
- ;;
-
-.mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12) // save r12
-.mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13) // save r13
-(p13) mov in5=-1
- ;;
- st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr
- tnat.nz p13,p0=in6
- cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8
- ;;
- mov r8=1
-(p9) tnat.nz p10,p0=r15
- adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch)
-
- st8.spill [r17]=r15 // save r15
- tnat.nz p8,p0=in7
- nop.i 0
-
- mov r13=r2 // establish `current'
- movl r1=__gp // establish kernel global pointer
- ;;
- st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error)
-(p13) mov in6=-1
-(p8) mov in7=-1
-
- cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
- movl r17=FPSR_DEFAULT
- ;;
- mov.m ar.fpsr=r17 // set ar.fpsr to kernel default value
-(p10) mov r8=-EINVAL
- br.ret.sptk.many b7
-END(ia64_syscall_setup)
-
- .org ia64_ivt+0x3c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x3c00 Entry 15 (size 64 bundles) Reserved
- DBG_FAULT(15)
- FAULT(15)
-
- /*
- * Squatting in this space ...
- *
- * This special case dispatcher for illegal operation faults allows preserved
- * registers to be modified through a callback function (asm only) that is handed
- * back from the fault handler in r8. Up to three arguments can be passed to the
- * callback function by returning an aggregate with the callback as its first
- * element, followed by the arguments.
- */
-ENTRY(dispatch_illegal_op_fault)
- .prologue
- .body
- SAVE_MIN_WITH_COVER
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- srlz.i // guarantee that interruption collection is on
- ;;
-(p15) ssm psr.i // restore psr.i
- adds r3=8,r2 // set up second base pointer for SAVE_REST
- ;;
- alloc r14=ar.pfs,0,0,1,0 // must be first in insn group
- mov out0=ar.ec
- ;;
- SAVE_REST
- PT_REGS_UNWIND_INFO(0)
- ;;
- br.call.sptk.many rp=ia64_illegal_op_fault
-.ret0: ;;
- alloc r14=ar.pfs,0,0,3,0 // must be first in insn group
- mov out0=r9
- mov out1=r10
- mov out2=r11
- movl r15=ia64_leave_kernel
- ;;
- mov rp=r15
- mov b6=r8
- ;;
- cmp.ne p6,p0=0,r8
-(p6) br.call.dpnt.many b6=b6 // call returns to ia64_leave_kernel
- br.sptk.many ia64_leave_kernel
-END(dispatch_illegal_op_fault)
-
- .org ia64_ivt+0x4000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x4000 Entry 16 (size 64 bundles) Reserved
- DBG_FAULT(16)
- FAULT(16)
-
- .org ia64_ivt+0x4400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x4400 Entry 17 (size 64 bundles) Reserved
- DBG_FAULT(17)
- FAULT(17)
-
-ENTRY(non_syscall)
- mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER
- ;;
- SAVE_MIN_WITH_COVER
-
- // There is no particular reason for this code to be here, other than that
- // there happens to be space here that would go unused otherwise. If this
- // fault ever gets "unreserved", simply moved the following code to a more
- // suitable spot...
-
- alloc r14=ar.pfs,0,0,2,0
- mov out0=cr.iim
- add out1=16,sp
- adds r3=8,r2 // set up second base pointer for SAVE_REST
-
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- srlz.i // guarantee that interruption collection is on
- ;;
-(p15) ssm psr.i // restore psr.i
- movl r15=ia64_leave_kernel
- ;;
- SAVE_REST
- mov rp=r15
- ;;
- br.call.sptk.many b6=ia64_bad_break // avoid WAW on CFM and ignore return addr
-END(non_syscall)
-
- .org ia64_ivt+0x4800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x4800 Entry 18 (size 64 bundles) Reserved
- DBG_FAULT(18)
- FAULT(18)
-
- /*
- * There is no particular reason for this code to be here, other than that
- * there happens to be space here that would go unused otherwise. If this
- * fault ever gets "unreserved", simply moved the following code to a more
- * suitable spot...
- */
-
-ENTRY(dispatch_unaligned_handler)
- SAVE_MIN_WITH_COVER
- ;;
- alloc r14=ar.pfs,0,0,2,0 // now it's safe (must be first in insn group!)
- mov out0=cr.ifa
- adds out1=16,sp
-
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- srlz.i // guarantee that interruption collection is on
- ;;
-(p15) ssm psr.i // restore psr.i
- adds r3=8,r2 // set up second base pointer
- ;;
- SAVE_REST
- movl r14=ia64_leave_kernel
- ;;
- mov rp=r14
- br.sptk.many ia64_prepare_handle_unaligned
-END(dispatch_unaligned_handler)
-
- .org ia64_ivt+0x4c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x4c00 Entry 19 (size 64 bundles) Reserved
- DBG_FAULT(19)
- FAULT(19)
-
- /*
- * There is no particular reason for this code to be here, other than that
- * there happens to be space here that would go unused otherwise. If this
- * fault ever gets "unreserved", simply moved the following code to a more
- * suitable spot...
- */
-
-ENTRY(dispatch_to_fault_handler)
- /*
- * Input:
- * psr.ic: off
- * r19: fault vector number (e.g., 24 for General Exception)
- * r31: contains saved predicates (pr)
- */
- SAVE_MIN_WITH_COVER_R19
- alloc r14=ar.pfs,0,0,5,0
- mov out0=r15
- mov out1=cr.isr
- mov out2=cr.ifa
- mov out3=cr.iim
- mov out4=cr.itir
- ;;
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- srlz.i // guarantee that interruption collection is on
- ;;
-(p15) ssm psr.i // restore psr.i
- adds r3=8,r2 // set up second base pointer for SAVE_REST
- ;;
- SAVE_REST
- movl r14=ia64_leave_kernel
- ;;
- mov rp=r14
- br.call.sptk.many b6=ia64_fault
-END(dispatch_to_fault_handler)
-
-//
-// --- End of long entries, Beginning of short entries
-//
-
- .org ia64_ivt+0x5000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
-ENTRY(page_not_present)
- DBG_FAULT(20)
- mov r16=cr.ifa
- rsm psr.dt
- /*
- * The Linux page fault handler doesn't expect non-present pages to be in
- * the TLB. Flush the existing entry now, so we meet that expectation.
- */
- mov r17=PAGE_SHIFT<<2
- ;;
- ptc.l r16,r17
- ;;
- mov r31=pr
- srlz.d
- br.sptk.many page_fault
-END(page_not_present)
-
- .org ia64_ivt+0x5100
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
-ENTRY(key_permission)
- DBG_FAULT(21)
- mov r16=cr.ifa
- rsm psr.dt
- mov r31=pr
- ;;
- srlz.d
- br.sptk.many page_fault
-END(key_permission)
-
- .org ia64_ivt+0x5200
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
-ENTRY(iaccess_rights)
- DBG_FAULT(22)
- mov r16=cr.ifa
- rsm psr.dt
- mov r31=pr
- ;;
- srlz.d
- br.sptk.many page_fault
-END(iaccess_rights)
-
- .org ia64_ivt+0x5300
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
-ENTRY(daccess_rights)
- DBG_FAULT(23)
- mov r16=cr.ifa
- rsm psr.dt
- mov r31=pr
- ;;
- srlz.d
- br.sptk.many page_fault
-END(daccess_rights)
-
- .org ia64_ivt+0x5400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39)
-ENTRY(general_exception)
- DBG_FAULT(24)
- mov r16=cr.isr
- mov r31=pr
- ;;
- cmp4.eq p6,p0=0,r16
-(p6) br.sptk.many dispatch_illegal_op_fault
- ;;
- mov r19=24 // fault number
- br.sptk.many dispatch_to_fault_handler
-END(general_exception)
-
- .org ia64_ivt+0x5500
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
-ENTRY(disabled_fp_reg)
- DBG_FAULT(25)
- rsm psr.dfh // ensure we can access fph
- ;;
- srlz.d
- mov r31=pr
- mov r19=25
- br.sptk.many dispatch_to_fault_handler
-END(disabled_fp_reg)
-
- .org ia64_ivt+0x5600
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
-ENTRY(nat_consumption)
- DBG_FAULT(26)
-
- mov r16=cr.ipsr
- mov r17=cr.isr
- mov r31=pr // save PR
- ;;
- and r18=0xf,r17 // r18 = cr.ipsr.code{3:0}
- tbit.z p6,p0=r17,IA64_ISR_NA_BIT
- ;;
- cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
- dep r16=-1,r16,IA64_PSR_ED_BIT,1
-(p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
- ;;
- mov cr.ipsr=r16 // set cr.ipsr.na
- mov pr=r31,-1
- ;;
- rfi
-
-1: mov pr=r31,-1
- ;;
- FAULT(26)
-END(nat_consumption)
-
- .org ia64_ivt+0x5700
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5700 Entry 27 (size 16 bundles) Speculation (40)
-ENTRY(speculation_vector)
- DBG_FAULT(27)
- /*
- * A [f]chk.[as] instruction needs to take the branch to the recovery code but
- * this part of the architecture is not implemented in hardware on some CPUs, such
- * as Itanium. Thus, in general we need to emulate the behavior. IIM contains
- * the relative target (not yet sign extended). So after sign extending it we
- * simply add it to IIP. We also need to reset the EI field of the IPSR to zero,
- * i.e., the slot to restart into.
- *
- * cr.imm contains zero_ext(imm21)
- */
- mov r18=cr.iim
- ;;
- mov r17=cr.iip
- shl r18=r18,43 // put sign bit in position (43=64-21)
- ;;
-
- mov r16=cr.ipsr
- shr r18=r18,39 // sign extend (39=43-4)
- ;;
-
- add r17=r17,r18 // now add the offset
- ;;
- mov cr.iip=r17
- dep r16=0,r16,41,2 // clear EI
- ;;
-
- mov cr.ipsr=r16
- ;;
-
- rfi // and go back
-END(speculation_vector)
-
- .org ia64_ivt+0x5800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5800 Entry 28 (size 16 bundles) Reserved
- DBG_FAULT(28)
- FAULT(28)
-
- .org ia64_ivt+0x5900
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56)
-ENTRY(debug_vector)
- DBG_FAULT(29)
- FAULT(29)
-END(debug_vector)
-
- .org ia64_ivt+0x5a00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
-ENTRY(unaligned_access)
- DBG_FAULT(30)
- mov r31=pr // prepare to save predicates
- ;;
- br.sptk.many dispatch_unaligned_handler
-END(unaligned_access)
-
- .org ia64_ivt+0x5b00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57)
-ENTRY(unsupported_data_reference)
- DBG_FAULT(31)
- FAULT(31)
-END(unsupported_data_reference)
-
- .org ia64_ivt+0x5c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64)
-ENTRY(floating_point_fault)
- DBG_FAULT(32)
- FAULT(32)
-END(floating_point_fault)
-
- .org ia64_ivt+0x5d00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66)
-ENTRY(floating_point_trap)
- DBG_FAULT(33)
- FAULT(33)
-END(floating_point_trap)
-
- .org ia64_ivt+0x5e00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66)
-ENTRY(lower_privilege_trap)
- DBG_FAULT(34)
- FAULT(34)
-END(lower_privilege_trap)
-
- .org ia64_ivt+0x5f00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68)
-ENTRY(taken_branch_trap)
- DBG_FAULT(35)
- FAULT(35)
-END(taken_branch_trap)
-
- .org ia64_ivt+0x6000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69)
-ENTRY(single_step_trap)
- DBG_FAULT(36)
- FAULT(36)
-END(single_step_trap)
-
- .org ia64_ivt+0x6100
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6100 Entry 37 (size 16 bundles) Reserved
- DBG_FAULT(37)
- FAULT(37)
-
- .org ia64_ivt+0x6200
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6200 Entry 38 (size 16 bundles) Reserved
- DBG_FAULT(38)
- FAULT(38)
-
- .org ia64_ivt+0x6300
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6300 Entry 39 (size 16 bundles) Reserved
- DBG_FAULT(39)
- FAULT(39)
-
- .org ia64_ivt+0x6400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6400 Entry 40 (size 16 bundles) Reserved
- DBG_FAULT(40)
- FAULT(40)
-
- .org ia64_ivt+0x6500
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6500 Entry 41 (size 16 bundles) Reserved
- DBG_FAULT(41)
- FAULT(41)
-
- .org ia64_ivt+0x6600
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6600 Entry 42 (size 16 bundles) Reserved
- DBG_FAULT(42)
- FAULT(42)
-
- .org ia64_ivt+0x6700
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6700 Entry 43 (size 16 bundles) Reserved
- DBG_FAULT(43)
- FAULT(43)
-
- .org ia64_ivt+0x6800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6800 Entry 44 (size 16 bundles) Reserved
- DBG_FAULT(44)
- FAULT(44)
-
- .org ia64_ivt+0x6900
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76,77)
-ENTRY(ia32_exception)
- DBG_FAULT(45)
- FAULT(45)
-END(ia32_exception)
-
- .org ia64_ivt+0x6a00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept (30,31,59,70,71)
-ENTRY(ia32_intercept)
- DBG_FAULT(46)
-#ifdef CONFIG_IA32_SUPPORT
- mov r31=pr
- mov r16=cr.isr
- ;;
- extr.u r17=r16,16,8 // get ISR.code
- mov r18=ar.eflag
- mov r19=cr.iim // old eflag value
- ;;
- cmp.ne p6,p0=2,r17
-(p6) br.cond.spnt 1f // not a system flag fault
- xor r16=r18,r19
- ;;
- extr.u r17=r16,18,1 // get the eflags.ac bit
- ;;
- cmp.eq p6,p0=0,r17
-(p6) br.cond.spnt 1f // eflags.ac bit didn't change
- ;;
- mov pr=r31,-1 // restore predicate registers
- rfi
-
-1:
-#endif // CONFIG_IA32_SUPPORT
- FAULT(46)
-END(ia32_intercept)
-
- .org ia64_ivt+0x6b00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt (74)
-ENTRY(ia32_interrupt)
- DBG_FAULT(47)
-#ifdef CONFIG_IA32_SUPPORT
- mov r31=pr
- br.sptk.many dispatch_to_ia32_handler
-#else
- FAULT(47)
-#endif
-END(ia32_interrupt)
-
- .org ia64_ivt+0x6c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6c00 Entry 48 (size 16 bundles) Reserved
- DBG_FAULT(48)
- FAULT(48)
-
- .org ia64_ivt+0x6d00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6d00 Entry 49 (size 16 bundles) Reserved
- DBG_FAULT(49)
- FAULT(49)
-
- .org ia64_ivt+0x6e00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6e00 Entry 50 (size 16 bundles) Reserved
- DBG_FAULT(50)
- FAULT(50)
-
- .org ia64_ivt+0x6f00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x6f00 Entry 51 (size 16 bundles) Reserved
- DBG_FAULT(51)
- FAULT(51)
-
- .org ia64_ivt+0x7000
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7000 Entry 52 (size 16 bundles) Reserved
- DBG_FAULT(52)
- FAULT(52)
-
- .org ia64_ivt+0x7100
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7100 Entry 53 (size 16 bundles) Reserved
- DBG_FAULT(53)
- FAULT(53)
-
- .org ia64_ivt+0x7200
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7200 Entry 54 (size 16 bundles) Reserved
- DBG_FAULT(54)
- FAULT(54)
-
- .org ia64_ivt+0x7300
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7300 Entry 55 (size 16 bundles) Reserved
- DBG_FAULT(55)
- FAULT(55)
-
- .org ia64_ivt+0x7400
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7400 Entry 56 (size 16 bundles) Reserved
- DBG_FAULT(56)
- FAULT(56)
-
- .org ia64_ivt+0x7500
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7500 Entry 57 (size 16 bundles) Reserved
- DBG_FAULT(57)
- FAULT(57)
-
- .org ia64_ivt+0x7600
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7600 Entry 58 (size 16 bundles) Reserved
- DBG_FAULT(58)
- FAULT(58)
-
- .org ia64_ivt+0x7700
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7700 Entry 59 (size 16 bundles) Reserved
- DBG_FAULT(59)
- FAULT(59)
-
- .org ia64_ivt+0x7800
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7800 Entry 60 (size 16 bundles) Reserved
- DBG_FAULT(60)
- FAULT(60)
-
- .org ia64_ivt+0x7900
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7900 Entry 61 (size 16 bundles) Reserved
- DBG_FAULT(61)
- FAULT(61)
-
- .org ia64_ivt+0x7a00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7a00 Entry 62 (size 16 bundles) Reserved
- DBG_FAULT(62)
- FAULT(62)
-
- .org ia64_ivt+0x7b00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7b00 Entry 63 (size 16 bundles) Reserved
- DBG_FAULT(63)
- FAULT(63)
-
- .org ia64_ivt+0x7c00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7c00 Entry 64 (size 16 bundles) Reserved
- DBG_FAULT(64)
- FAULT(64)
-
- .org ia64_ivt+0x7d00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7d00 Entry 65 (size 16 bundles) Reserved
- DBG_FAULT(65)
- FAULT(65)
-
- .org ia64_ivt+0x7e00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7e00 Entry 66 (size 16 bundles) Reserved
- DBG_FAULT(66)
- FAULT(66)
-
- .org ia64_ivt+0x7f00
-/////////////////////////////////////////////////////////////////////////////////////////
-// 0x7f00 Entry 67 (size 16 bundles) Reserved
- DBG_FAULT(67)
- FAULT(67)
-
-#ifdef CONFIG_IA32_SUPPORT
-
- /*
- * There is no particular reason for this code to be here, other than that
- * there happens to be space here that would go unused otherwise. If this
- * fault ever gets "unreserved", simply moved the following code to a more
- * suitable spot...
- */
-
- // IA32 interrupt entry point
-
-ENTRY(dispatch_to_ia32_handler)
- SAVE_MIN
- ;;
- mov r14=cr.isr
- ssm psr.ic | PSR_DEFAULT_BITS
- ;;
- srlz.i // guarantee that interruption collection is on
- ;;
-(p15) ssm psr.i
- adds r3=8,r2 // Base pointer for SAVE_REST
- ;;
- SAVE_REST
- ;;
- mov r15=0x80
- shr r14=r14,16 // Get interrupt number
- ;;
- cmp.ne p6,p0=r14,r15
-(p6) br.call.dpnt.many b6=non_ia32_syscall
-
- adds r14=IA64_PT_REGS_R8_OFFSET + 16,sp // 16 byte hole per SW conventions
- adds r15=IA64_PT_REGS_R1_OFFSET + 16,sp
- ;;
- cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
- ld8 r8=[r14] // get r8
- ;;
- st8 [r15]=r8 // save original EAX in r1 (IA32 procs don't use the GP)
- ;;
- alloc r15=ar.pfs,0,0,6,0 // must first in an insn group
- ;;
- ld4 r8=[r14],8 // r8 == eax (syscall number)
- mov r15=IA32_NR_syscalls
- ;;
- cmp.ltu.unc p6,p7=r8,r15
- ld4 out1=[r14],8 // r9 == ecx
- ;;
- ld4 out2=[r14],8 // r10 == edx
- ;;
- ld4 out0=[r14] // r11 == ebx
- adds r14=(IA64_PT_REGS_R13_OFFSET) + 16,sp
- ;;
- ld4 out5=[r14],PT(R14)-PT(R13) // r13 == ebp
- ;;
- ld4 out3=[r14],PT(R15)-PT(R14) // r14 == esi
- adds r2=TI_FLAGS+IA64_TASK_SIZE,r13
- ;;
- ld4 out4=[r14] // r15 == edi
- movl r16=ia32_syscall_table
- ;;
-(p6) shladd r16=r8,3,r16 // force ni_syscall if not valid syscall number
- ld4 r2=[r2] // r2 = current_thread_info()->flags
- ;;
- ld8 r16=[r16]
- and r2=_TIF_SYSCALL_TRACEAUDIT,r2 // mask trace or audit
- ;;
- mov b6=r16
- movl r15=ia32_ret_from_syscall
- cmp.eq p8,p0=r2,r0
- ;;
- mov rp=r15
-(p8) br.call.sptk.many b6=b6
- br.cond.sptk ia32_trace_syscall
-
-non_ia32_syscall:
- alloc r15=ar.pfs,0,0,2,0
- mov out0=r14 // interrupt #
- add out1=16,sp // pointer to pt_regs
- ;; // avoid WAW on CFM
- br.call.sptk.many rp=ia32_bad_interrupt
-.ret1: movl r15=ia64_leave_kernel
- ;;
- mov rp=r15
- br.ret.sptk.many rp
-END(dispatch_to_ia32_handler)
-
-#endif /* CONFIG_IA32_SUPPORT */
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RFC 2/6] LVHPT - Create ivt.S dyanamically
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
2006-05-10 3:42 ` [RFC 1/6] LVHPT - Remove static ivt.S Ian Wienand
@ 2006-05-10 3:42 ` Ian Wienand
2006-05-10 3:42 ` [RFC 3/6] LVHPT - ivt.S long format support Ian Wienand
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
LVHPT ivt break apart
Create a script to dynamically create ivt.S. Pull out code from ivt.S
into a template file. The template(s) are then inserted back into
ivt.S at build time.
This approach avoids
- replicating the entire file
- #including a multitude of smaller files
- using CPP defines (which makes editing assembly hard)
- using assembler macros, which can be confusing as you do not get
context
Signed-Off-By: Ian Wienand <ianw@gelato.unsw.edu.au>
---
kernel/Makefile | 5
kernel/ivt-sfvhpt.in | 444 +++++++++++++++++
kernel/ivt.S.in | 1324 +++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/merge.py | 58 ++
4 files changed, 1831 insertions(+)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/scripts/merge.py
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/scripts/merge.py 2006-05-05 10:09:38.000000000 +1000
@@ -0,0 +1,58 @@
+#!/usr/bin/env python2.4
+# nb. requires python2.4 for string templating
+#
+# Usage: merge.py output template code
+#
+# merge the code in 'code' into 'template', writing out to 'output'
+#
+# Anything between lines __begin_name__ and __end_name__ in 'code'
+# gets inserted into $name in 'template'.
+#
+# Ian Wienand <ianw@gelato.unsw.edu.au>
+#
+import os
+import sys
+from string import Template
+
+if len(sys.argv) != 4:
+ print "Usage: %s output template code" % sys.argv[0]
+ sys.exit(2)
+
+print "Merging %s and template %s to %s" % (sys.argv[2], sys.argv[3], sys.argv[1])
+
+# bring the code file into a dictionary
+# anything between lines __begin_name__ and __end_name__ goes into
+# a dictionary entry of name
+template_dictionary = {}
+am_processing = False
+current_template = ""
+current_template_name = ""
+for line in open(sys.argv[3], 'r').readlines():
+
+ if am_processing:
+ # if this line is the end, stop
+ # XXX check this end is actually the name we are processing
+ if line[:6] == "__end_":
+ template_dictionary[current_template_name] = current_template
+ am_processing = False
+ print "... done"
+ continue
+ # otherwise, add this line to the current template
+ current_template += line
+ continue
+ # if we got here, we are not processing
+ if line[:8] == "__begin_":
+ am_processing = True
+ current_template_name = line[8:-3] #newline
+ print "Processing %s" % (current_template_name),
+ current_template = ""
+ continue
+ # this is some random line
+ continue
+
+#now open the file where we put these templates
+template = Template(open(sys.argv[2],'r').read())
+
+#finally, substitute them all in
+output = open(sys.argv[1],'w')
+output.write(template.substitute(template_dictionary))
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt.S.in
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt.S.in 2006-05-05 10:09:38.000000000 +1000
@@ -0,0 +1,1324 @@
+/*
+ * arch/ia64/kernel/ivt.S
+ *
+ * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
+ * Stephane Eranian <eranian@hpl.hp.com>
+ * David Mosberger <davidm@hpl.hp.com>
+ * Copyright (C) 2000, 2002-2003 Intel Co
+ * Asit Mallick <asit.k.mallick@intel.com>
+ * Suresh Siddha <suresh.b.siddha@intel.com>
+ * Kenneth Chen <kenneth.w.chen@intel.com>
+ * Fenghua Yu <fenghua.yu@intel.com>
+ *
+ * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
+ * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
+ */
+/*
+ * This file defines the interruption vector table used by the CPU.
+ * It does not include one entry per possible cause of interruption.
+ *
+ * The first 20 entries of the table contain 64 bundles each while the
+ * remaining 48 entries contain only 16 bundles each.
+ *
+ * The 64 bundles are used to allow inlining the whole handler for critical
+ * interruptions like TLB misses.
+ *
+ * For each entry, the comment is as follows:
+ *
+ * // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
+ * entry offset ----/ / / / /
+ * entry number ---------/ / / /
+ * size of the entry -------------/ / /
+ * vector name -------------------------------------/ /
+ * interruptions triggering this vector ----------------------/
+ *
+ * The table is 32KB in size and must be aligned on 32KB boundary.
+ * (The CPU ignores the 15 lower bits of the address)
+ *
+ * Table is based upon EAS2.6 (Oct 1999)
+ */
+
+#include <linux/config.h>
+
+#include <asm/asmmacro.h>
+#include <asm/break.h>
+#include <asm/ia32.h>
+#include <asm/kregs.h>
+#include <asm/asm-offsets.h>
+#include <asm/pgtable.h>
+#include <asm/processor.h>
+#include <asm/ptrace.h>
+#include <asm/system.h>
+#include <asm/thread_info.h>
+#include <asm/unistd.h>
+#include <asm/errno.h>
+
+#if 1
+# define PSR_DEFAULT_BITS psr.ac
+#else
+# define PSR_DEFAULT_BITS 0
+#endif
+
+#if 0
+ /*
+ * This lets you track the last eight faults that occurred on the CPU. Make sure ar.k2 isn't
+ * needed for something else before enabling this...
+ */
+# define DBG_FAULT(i) mov r16=ar.k2;; shl r16=r16,8;; add r16=(i),r16;;mov ar.k2=r16
+#else
+# define DBG_FAULT(i)
+#endif
+
+#include "minstate.h"
+
+#define FAULT(n) \
+ mov r31=pr; \
+ mov r19=n;; /* prepare to save predicates */ \
+ br.sptk.many dispatch_to_fault_handler
+
+ .section .text.ivt,"ax"
+
+ .align 32768 // align on 32KB boundary
+ .global ia64_ivt
+ia64_ivt:
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
+ENTRY(vhpt_miss)
+ DBG_FAULT(0)
+ /*
+ * The VHPT vector is invoked when the TLB entry for the virtual page table
+ * is missing. This happens only as a result of a previous
+ * (the "original") TLB miss, which may either be caused by an instruction
+ * fetch or a data access (or non-access).
+ *
+ * What we do here is normal TLB miss handing for the _original_ miss,
+ * followed by inserting the TLB entry for the virtual page table page
+ * that the VHPT walker was attempting to access. The latter gets
+ * inserted as long as page table entry above pte level have valid
+ * mappings for the faulting address. The TLB entry for the original
+ * miss gets inserted only if the pte entry indicates that the page is
+ * present.
+ *
+ * do_page_fault gets invoked in the following cases:
+ * - the faulting virtual address uses unimplemented address bits
+ * - the faulting virtual address has no valid page table mapping
+ */
+$vhpt_miss_handler
+END(vhpt_miss)
+
+ .org ia64_ivt+0x400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x0400 Entry 1 (size 64 bundles) ITLB (21)
+ENTRY(itlb_miss)
+ DBG_FAULT(1)
+ /*
+ * The ITLB handler accesses the PTE via the virtually mapped linear
+ * page table. If a nested TLB miss occurs, we switch into physical
+ * mode, walk the page table, and then re-execute the PTE read and
+ * go on normally after that.
+ */
+$itlb_miss_handler
+END(itlb_miss)
+
+ .org ia64_ivt+0x0800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
+ENTRY(dtlb_miss)
+ DBG_FAULT(2)
+ /*
+ * The DTLB handler accesses the PTE via the virtually mapped linear
+ * page table. If a nested TLB miss occurs, we switch into physical
+ * mode, walk the page table, and then re-execute the PTE read and
+ * go on normally after that.
+ */
+$dtlb_miss_handler
+END(dtlb_miss)
+
+ .org ia64_ivt+0x0c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
+ENTRY(alt_itlb_miss)
+ DBG_FAULT(3)
+ mov r16=cr.ifa // get address that caused the TLB miss
+ movl r17=PAGE_KERNEL
+ mov r21=cr.ipsr
+ movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
+ mov r31=pr
+ ;;
+#ifdef CONFIG_DISABLE_VHPT
+ shr.u r22=r16,61 // get the region number into r21
+ ;;
+ cmp.gt p8,p0=6,r22 // user mode
+ ;;
+(p8) thash r17=r16
+ ;;
+(p8) mov cr.iha=r17
+(p8) mov r29=b0 // save b0
+(p8) br.cond.dptk .itlb_fault
+#endif
+ extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
+ and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
+ shr.u r18=r16,57 // move address bit 61 to bit 4
+ ;;
+ andcm r18=0x10,r18 // bit 4=~address-bit(61)
+ cmp.ne p8,p0=r0,r23 // psr.cpl != 0?
+ or r19=r17,r19 // insert PTE control bits into r19
+ ;;
+ or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
+(p8) br.cond.spnt page_fault
+ ;;
+ itc.i r19 // insert the TLB entry
+ mov pr=r31,-1
+ rfi
+END(alt_itlb_miss)
+
+ .org ia64_ivt+0x1000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
+ENTRY(alt_dtlb_miss)
+ DBG_FAULT(4)
+ mov r16=cr.ifa // get address that caused the TLB miss
+ movl r17=PAGE_KERNEL
+ mov r20=cr.isr
+ movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
+ mov r21=cr.ipsr
+ mov r31=pr
+ ;;
+#ifdef CONFIG_DISABLE_VHPT
+ shr.u r22=r16,61 // get the region number into r21
+ ;;
+ cmp.gt p8,p0=6,r22 // access to region 0-5
+ ;;
+(p8) thash r17=r16
+ ;;
+(p8) mov cr.iha=r17
+(p8) mov r29=b0 // save b0
+(p8) br.cond.dptk dtlb_fault
+#endif
+ extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
+ and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field
+ tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on?
+ shr.u r18=r16,57 // move address bit 61 to bit 4
+ and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
+ tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
+ ;;
+ andcm r18=0x10,r18 // bit 4=~address-bit(61)
+ cmp.ne p8,p0=r0,r23
+(p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field
+(p8) br.cond.spnt page_fault
+
+ dep r21=-1,r21,IA64_PSR_ED_BIT,1
+ or r19=r19,r17 // insert PTE control bits into r19
+ ;;
+ or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
+(p6) mov cr.ipsr=r21
+ ;;
+(p7) itc.d r19 // insert the TLB entry
+ mov pr=r31,-1
+ rfi
+END(alt_dtlb_miss)
+
+ .org ia64_ivt+0x1400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
+ENTRY(nested_dtlb_miss)
+ /*
+ * In the absence of kernel bugs, we get here when the virtually mapped linear
+ * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
+ * Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page
+ * table is missing, a nested TLB miss fault is triggered and control is
+ * transferred to this point. When this happens, we lookup the pte for the
+ * faulting address by walking the page table in physical mode and return to the
+ * continuation point passed in register r30 (or call page_fault if the address is
+ * not mapped).
+ *
+ * Input: r16: faulting address
+ * r29: saved b0
+ * r30: continuation address
+ * r31: saved pr
+ *
+ * Output: r17: physical address of PTE of faulting address
+ * r29: saved b0
+ * r30: continuation address
+ * r31: saved pr
+ *
+ * Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared)
+ */
+$nested_dtlb_miss_handler
+END(nested_dtlb_miss)
+
+ .org ia64_ivt+0x1800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
+ENTRY(ikey_miss)
+ DBG_FAULT(6)
+ FAULT(6)
+END(ikey_miss)
+
+ //-----------------------------------------------------------------------------------
+ // call do_page_fault (predicates are in r31, psr.dt may be off, r16 is faulting address)
+ENTRY(page_fault)
+ ssm psr.dt
+ ;;
+ srlz.i
+ ;;
+ SAVE_MIN_WITH_COVER
+ alloc r15=ar.pfs,0,0,3,0
+ mov out0=cr.ifa
+ mov out1=cr.isr
+ adds r3=8,r2 // set up second base pointer
+ ;;
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ srlz.i // guarantee that interruption collectin is on
+ ;;
+(p15) ssm psr.i // restore psr.i
+ movl r14=ia64_leave_kernel
+ ;;
+ SAVE_REST
+ mov rp=r14
+ ;;
+ adds out2=16,r12 // out2 = pointer to pt_regs
+ br.call.sptk.many b6=ia64_do_page_fault // ignore return address
+END(page_fault)
+
+ .org ia64_ivt+0x1c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
+ENTRY(dkey_miss)
+ DBG_FAULT(7)
+ FAULT(7)
+END(dkey_miss)
+
+ .org ia64_ivt+0x2000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
+ENTRY(dirty_bit)
+ DBG_FAULT(8)
+$dirty_bit_handler
+END(dirty_bit)
+
+ .org ia64_ivt+0x2400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
+ENTRY(iaccess_bit)
+ DBG_FAULT(9)
+$iaccess_bit_handler
+END(iaccess_bit)
+
+ .org ia64_ivt+0x2800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
+ENTRY(daccess_bit)
+ DBG_FAULT(10)
+$daccess_bit_handler
+END(daccess_bit)
+
+ .org ia64_ivt+0x2c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
+ENTRY(break_fault)
+ /*
+ * The streamlined system call entry/exit paths only save/restore the initial part
+ * of pt_regs. This implies that the callers of system-calls must adhere to the
+ * normal procedure calling conventions.
+ *
+ * Registers to be saved & restored:
+ * CR registers: cr.ipsr, cr.iip, cr.ifs
+ * AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr
+ * others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15
+ * Registers to be restored only:
+ * r8-r11: output value from the system call.
+ *
+ * During system call exit, scratch registers (including r15) are modified/cleared
+ * to prevent leaking bits from kernel to user level.
+ */
+ DBG_FAULT(11)
+ mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
+ mov r29=cr.ipsr // M2 (12 cyc)
+ mov r31=pr // I0 (2 cyc)
+
+ mov r17=cr.iim // M2 (2 cyc)
+ mov.m r27=ar.rsc // M2 (12 cyc)
+ mov r18=__IA64_BREAK_SYSCALL // A
+
+ mov.m ar.rsc=0 // M2
+ mov.m r21=ar.fpsr // M2 (12 cyc)
+ mov r19=b6 // I0 (2 cyc)
+ ;;
+ mov.m r23=ar.bspstore // M2 (12 cyc)
+ mov.m r24=ar.rnat // M2 (5 cyc)
+ mov.i r26=ar.pfs // I0 (2 cyc)
+
+ invala // M0|1
+ nop.m 0 // M
+ mov r20=r1 // A save r1
+
+ nop.m 0
+ movl r30=sys_call_table // X
+
+ mov r28=cr.iip // M2 (2 cyc)
+ cmp.eq p0,p7=r18,r17 // I0 is this a system call?
+(p7) br.cond.spnt non_syscall // B no ->
+ //
+ // From this point on, we are definitely on the syscall-path
+ // and we can use (non-banked) scratch registers.
+ //
+///////////////////////////////////////////////////////////////////////
+ mov r1=r16 // A move task-pointer to "addl"-addressable reg
+ mov r2=r16 // A setup r2 for ia64_syscall_setup
+ add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags
+
+ adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
+ adds r15=-1024,r15 // A subtract 1024 from syscall number
+ mov r3=NR_syscalls - 1
+ ;;
+ ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
+ ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
+ extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr
+
+ shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
+ addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS
+ cmp.leu p6,p7=r15,r3 // A syscall number in range?
+ ;;
+
+ lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS
+(p6) ld8 r30=[r30] // M0|1 load address of syscall entry point
+ tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT?
+
+ mov.m ar.bspstore=r22 // M2 switch to kernel RBS
+ cmp.eq p8,p9=2,r8 // A isr.ei==2?
+ ;;
+
+(p8) mov r8=0 // A clear ei to 0
+(p7) movl r30=sys_ni_syscall // X
+
+(p8) adds r28=16,r28 // A switch cr.iip to next bundle
+(p9) adds r8=1,r8 // A increment ei to next slot
+ nop.i 0
+ ;;
+
+ mov.m r25=ar.unat // M2 (5 cyc)
+ dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr
+ adds r15=1024,r15 // A restore original syscall number
+ //
+ // If any of the above loads miss in L1D, we'll stall here until
+ // the data arrives.
+ //
+///////////////////////////////////////////////////////////////////////
+ st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
+ mov b6=r30 // I0 setup syscall handler branch reg early
+ cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already?
+
+ and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit
+ mov r18=ar.bsp // M2 (12 cyc)
+(pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
+ ;;
+.back_from_break_fixup:
+(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
+ cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
+ br.call.sptk.many b7=ia64_syscall_setup // B
+1:
+ mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
+ nop 0
+ bsw.1 // B (6 cyc) regs are saved, switch to bank 1
+ ;;
+
+ ssm psr.ic | PSR_DEFAULT_BITS // M2 now it's safe to re-enable intr.-collection
+ movl r3=ia64_ret_from_syscall // X
+ ;;
+
+ srlz.i // M0 ensure interruption collection is on
+ mov rp=r3 // I0 set the real return addr
+(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
+
+(p15) ssm psr.i // M2 restore psr.i
+(p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
+ br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
+ // NOT REACHED
+///////////////////////////////////////////////////////////////////////
+ // On entry, we optimistically assumed that we're coming from user-space.
+ // For the rare cases where a system-call is done from within the kernel,
+ // we fix things up at this point:
+.break_fixup:
+ add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
+ mov ar.rnat=r24 // M2 restore kernel's AR.RNAT
+ ;;
+ mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE
+ br.cond.sptk .back_from_break_fixup
+END(break_fault)
+
+ .org ia64_ivt+0x3000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
+ENTRY(interrupt)
+ DBG_FAULT(12)
+ mov r31=pr // prepare to save predicates
+ ;;
+ SAVE_MIN_WITH_COVER // uses r31; defines r2 and r3
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ adds r3=8,r2 // set up second base pointer for SAVE_REST
+ srlz.i // ensure everybody knows psr.ic is back on
+ ;;
+ SAVE_REST
+ ;;
+ MCA_RECOVER_RANGE(interrupt)
+ alloc r14=ar.pfs,0,0,2,0 // must be first in an insn group
+ mov out0=cr.ivr // pass cr.ivr as first arg
+ add out1=16,sp // pass pointer to pt_regs as second arg
+ ;;
+ srlz.d // make sure we see the effect of cr.ivr
+ movl r14=ia64_leave_kernel
+ ;;
+ mov rp=r14
+ br.call.sptk.many b6=ia64_handle_irq
+END(interrupt)
+
+ .org ia64_ivt+0x3400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x3400 Entry 13 (size 64 bundles) Reserved
+ DBG_FAULT(13)
+ FAULT(13)
+
+ .org ia64_ivt+0x3800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x3800 Entry 14 (size 64 bundles) Reserved
+ DBG_FAULT(14)
+ FAULT(14)
+
+ /*
+ * There is no particular reason for this code to be here, other than that
+ * there happens to be space here that would go unused otherwise. If this
+ * fault ever gets "unreserved", simply moved the following code to a more
+ * suitable spot...
+ *
+ * ia64_syscall_setup() is a separate subroutine so that it can
+ * allocate stacked registers so it can safely demine any
+ * potential NaT values from the input registers.
+ *
+ * On entry:
+ * - executing on bank 0 or bank 1 register set (doesn't matter)
+ * - r1: stack pointer
+ * - r2: current task pointer
+ * - r3: preserved
+ * - r11: original contents (saved ar.pfs to be saved)
+ * - r12: original contents (sp to be saved)
+ * - r13: original contents (tp to be saved)
+ * - r15: original contents (syscall # to be saved)
+ * - r18: saved bsp (after switching to kernel stack)
+ * - r19: saved b6
+ * - r20: saved r1 (gp)
+ * - r21: saved ar.fpsr
+ * - r22: kernel's register backing store base (krbs_base)
+ * - r23: saved ar.bspstore
+ * - r24: saved ar.rnat
+ * - r25: saved ar.unat
+ * - r26: saved ar.pfs
+ * - r27: saved ar.rsc
+ * - r28: saved cr.iip
+ * - r29: saved cr.ipsr
+ * - r31: saved pr
+ * - b0: original contents (to be saved)
+ * On exit:
+ * - p10: TRUE if syscall is invoked with more than 8 out
+ * registers or r15's Nat is true
+ * - r1: kernel's gp
+ * - r3: preserved (same as on entry)
+ * - r8: -EINVAL if p10 is true
+ * - r12: points to kernel stack
+ * - r13: points to current task
+ * - r14: preserved (same as on entry)
+ * - p13: preserved
+ * - p15: TRUE if interrupts need to be re-enabled
+ * - ar.fpsr: set to kernel settings
+ * - b6: preserved (same as on entry)
+ */
+GLOBAL_ENTRY(ia64_syscall_setup)
+#if PT(B6) != 0
+# error This code assumes that b6 is the first field in pt_regs.
+#endif
+ st8 [r1]=r19 // save b6
+ add r16=PT(CR_IPSR),r1 // initialize first base pointer
+ add r17=PT(R11),r1 // initialize second base pointer
+ ;;
+ alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable
+ st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr
+ tnat.nz p8,p0=in0
+
+ st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11
+ tnat.nz p9,p0=in1
+(pKStk) mov r18=r0 // make sure r18 isn't NaT
+ ;;
+
+ st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs
+ st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip
+ mov r28=b0 // save b0 (2 cyc)
+ ;;
+
+ st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat
+ dep r19=0,r19,38,26 // clear all bits but 0..37 [I0]
+(p8) mov in0=-1
+ ;;
+
+ st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs
+ extr.u r11=r19,7,7 // I0 // get sol of ar.pfs
+ and r8=0x7f,r19 // A // get sof of ar.pfs
+
+ st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
+ tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0
+(p9) mov in1=-1
+ ;;
+
+(pUStk) sub r18=r18,r22 // r18=RSE.ndirty*8
+ tnat.nz p10,p0=in2
+ add r11=8,r11
+ ;;
+(pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field
+(pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field
+ tnat.nz p11,p0=in3
+ ;;
+(p10) mov in2=-1
+ tnat.nz p12,p0=in4 // [I0]
+(p11) mov in3=-1
+ ;;
+(pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat
+(pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore
+ shl r18=r18,16 // compute ar.rsc to be used for "loadrs"
+ ;;
+ st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates
+ st8 [r17]=r28,PT(R1)-PT(B0) // save b0
+ tnat.nz p13,p0=in5 // [I0]
+ ;;
+ st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs"
+ st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1
+(p12) mov in4=-1
+ ;;
+
+.mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12) // save r12
+.mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13) // save r13
+(p13) mov in5=-1
+ ;;
+ st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr
+ tnat.nz p13,p0=in6
+ cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8
+ ;;
+ mov r8=1
+(p9) tnat.nz p10,p0=r15
+ adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch)
+
+ st8.spill [r17]=r15 // save r15
+ tnat.nz p8,p0=in7
+ nop.i 0
+
+ mov r13=r2 // establish `current'
+ movl r1=__gp // establish kernel global pointer
+ ;;
+ st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error)
+(p13) mov in6=-1
+(p8) mov in7=-1
+
+ cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
+ movl r17=FPSR_DEFAULT
+ ;;
+ mov.m ar.fpsr=r17 // set ar.fpsr to kernel default value
+(p10) mov r8=-EINVAL
+ br.ret.sptk.many b7
+END(ia64_syscall_setup)
+
+ .org ia64_ivt+0x3c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x3c00 Entry 15 (size 64 bundles) Reserved
+ DBG_FAULT(15)
+ FAULT(15)
+
+ /*
+ * Squatting in this space ...
+ *
+ * This special case dispatcher for illegal operation faults allows preserved
+ * registers to be modified through a callback function (asm only) that is handed
+ * back from the fault handler in r8. Up to three arguments can be passed to the
+ * callback function by returning an aggregate with the callback as its first
+ * element, followed by the arguments.
+ */
+ENTRY(dispatch_illegal_op_fault)
+ .prologue
+ .body
+ SAVE_MIN_WITH_COVER
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ srlz.i // guarantee that interruption collection is on
+ ;;
+(p15) ssm psr.i // restore psr.i
+ adds r3=8,r2 // set up second base pointer for SAVE_REST
+ ;;
+ alloc r14=ar.pfs,0,0,1,0 // must be first in insn group
+ mov out0=ar.ec
+ ;;
+ SAVE_REST
+ PT_REGS_UNWIND_INFO(0)
+ ;;
+ br.call.sptk.many rp=ia64_illegal_op_fault
+.ret0: ;;
+ alloc r14=ar.pfs,0,0,3,0 // must be first in insn group
+ mov out0=r9
+ mov out1=r10
+ mov out2=r11
+ movl r15=ia64_leave_kernel
+ ;;
+ mov rp=r15
+ mov b6=r8
+ ;;
+ cmp.ne p6,p0=0,r8
+(p6) br.call.dpnt.many b6=b6 // call returns to ia64_leave_kernel
+ br.sptk.many ia64_leave_kernel
+END(dispatch_illegal_op_fault)
+
+ .org ia64_ivt+0x4000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x4000 Entry 16 (size 64 bundles) Reserved
+ DBG_FAULT(16)
+ FAULT(16)
+
+ .org ia64_ivt+0x4400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x4400 Entry 17 (size 64 bundles) Reserved
+ DBG_FAULT(17)
+ FAULT(17)
+
+ENTRY(non_syscall)
+ mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER
+ ;;
+ SAVE_MIN_WITH_COVER
+
+ // There is no particular reason for this code to be here, other than that
+ // there happens to be space here that would go unused otherwise. If this
+ // fault ever gets "unreserved", simply moved the following code to a more
+ // suitable spot...
+
+ alloc r14=ar.pfs,0,0,2,0
+ mov out0=cr.iim
+ add out1=16,sp
+ adds r3=8,r2 // set up second base pointer for SAVE_REST
+
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ srlz.i // guarantee that interruption collection is on
+ ;;
+(p15) ssm psr.i // restore psr.i
+ movl r15=ia64_leave_kernel
+ ;;
+ SAVE_REST
+ mov rp=r15
+ ;;
+ br.call.sptk.many b6=ia64_bad_break // avoid WAW on CFM and ignore return addr
+END(non_syscall)
+
+ .org ia64_ivt+0x4800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x4800 Entry 18 (size 64 bundles) Reserved
+ DBG_FAULT(18)
+ FAULT(18)
+
+ /*
+ * There is no particular reason for this code to be here, other than that
+ * there happens to be space here that would go unused otherwise. If this
+ * fault ever gets "unreserved", simply moved the following code to a more
+ * suitable spot...
+ */
+
+ENTRY(dispatch_unaligned_handler)
+ SAVE_MIN_WITH_COVER
+ ;;
+ alloc r14=ar.pfs,0,0,2,0 // now it's safe (must be first in insn group!)
+ mov out0=cr.ifa
+ adds out1=16,sp
+
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ srlz.i // guarantee that interruption collection is on
+ ;;
+(p15) ssm psr.i // restore psr.i
+ adds r3=8,r2 // set up second base pointer
+ ;;
+ SAVE_REST
+ movl r14=ia64_leave_kernel
+ ;;
+ mov rp=r14
+ br.sptk.many ia64_prepare_handle_unaligned
+END(dispatch_unaligned_handler)
+
+ .org ia64_ivt+0x4c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x4c00 Entry 19 (size 64 bundles) Reserved
+ DBG_FAULT(19)
+ FAULT(19)
+
+ /*
+ * There is no particular reason for this code to be here, other than that
+ * there happens to be space here that would go unused otherwise. If this
+ * fault ever gets "unreserved", simply moved the following code to a more
+ * suitable spot...
+ */
+
+ENTRY(dispatch_to_fault_handler)
+ /*
+ * Input:
+ * psr.ic: off
+ * r19: fault vector number (e.g., 24 for General Exception)
+ * r31: contains saved predicates (pr)
+ */
+ SAVE_MIN_WITH_COVER_R19
+ alloc r14=ar.pfs,0,0,5,0
+ mov out0=r15
+ mov out1=cr.isr
+ mov out2=cr.ifa
+ mov out3=cr.iim
+ mov out4=cr.itir
+ ;;
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ srlz.i // guarantee that interruption collection is on
+ ;;
+(p15) ssm psr.i // restore psr.i
+ adds r3=8,r2 // set up second base pointer for SAVE_REST
+ ;;
+ SAVE_REST
+ movl r14=ia64_leave_kernel
+ ;;
+ mov rp=r14
+ br.call.sptk.many b6=ia64_fault
+END(dispatch_to_fault_handler)
+
+//
+// --- End of long entries, Beginning of short entries
+//
+
+ .org ia64_ivt+0x5000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
+ENTRY(page_not_present)
+ DBG_FAULT(20)
+ mov r16=cr.ifa
+ rsm psr.dt
+ /*
+ * The Linux page fault handler doesn't expect non-present pages to be in
+ * the TLB. Flush the existing entry now, so we meet that expectation.
+ */
+ mov r17=PAGE_SHIFT<<2
+ ;;
+ ptc.l r16,r17
+ ;;
+ mov r31=pr
+ srlz.d
+ br.sptk.many page_fault
+END(page_not_present)
+
+ .org ia64_ivt+0x5100
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
+ENTRY(key_permission)
+ DBG_FAULT(21)
+ mov r16=cr.ifa
+ rsm psr.dt
+ mov r31=pr
+ ;;
+ srlz.d
+ br.sptk.many page_fault
+END(key_permission)
+
+ .org ia64_ivt+0x5200
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
+ENTRY(iaccess_rights)
+ DBG_FAULT(22)
+ mov r16=cr.ifa
+ rsm psr.dt
+ mov r31=pr
+ ;;
+ srlz.d
+ br.sptk.many page_fault
+END(iaccess_rights)
+
+ .org ia64_ivt+0x5300
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
+ENTRY(daccess_rights)
+ DBG_FAULT(23)
+ mov r16=cr.ifa
+ rsm psr.dt
+ mov r31=pr
+ ;;
+ srlz.d
+ br.sptk.many page_fault
+END(daccess_rights)
+
+ .org ia64_ivt+0x5400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39)
+ENTRY(general_exception)
+ DBG_FAULT(24)
+ mov r16=cr.isr
+ mov r31=pr
+ ;;
+ cmp4.eq p6,p0=0,r16
+(p6) br.sptk.many dispatch_illegal_op_fault
+ ;;
+ mov r19=24 // fault number
+ br.sptk.many dispatch_to_fault_handler
+END(general_exception)
+
+ .org ia64_ivt+0x5500
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
+ENTRY(disabled_fp_reg)
+ DBG_FAULT(25)
+ rsm psr.dfh // ensure we can access fph
+ ;;
+ srlz.d
+ mov r31=pr
+ mov r19=25
+ br.sptk.many dispatch_to_fault_handler
+END(disabled_fp_reg)
+
+ .org ia64_ivt+0x5600
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
+ENTRY(nat_consumption)
+ DBG_FAULT(26)
+
+ mov r16=cr.ipsr
+ mov r17=cr.isr
+ mov r31=pr // save PR
+ ;;
+ and r18=0xf,r17 // r18 = cr.ipsr.code{3:0}
+ tbit.z p6,p0=r17,IA64_ISR_NA_BIT
+ ;;
+ cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
+ dep r16=-1,r16,IA64_PSR_ED_BIT,1
+(p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
+ ;;
+ mov cr.ipsr=r16 // set cr.ipsr.na
+ mov pr=r31,-1
+ ;;
+ rfi
+
+1: mov pr=r31,-1
+ ;;
+ FAULT(26)
+END(nat_consumption)
+
+ .org ia64_ivt+0x5700
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5700 Entry 27 (size 16 bundles) Speculation (40)
+ENTRY(speculation_vector)
+ DBG_FAULT(27)
+ /*
+ * A [f]chk.[as] instruction needs to take the branch to the recovery code but
+ * this part of the architecture is not implemented in hardware on some CPUs, such
+ * as Itanium. Thus, in general we need to emulate the behavior. IIM contains
+ * the relative target (not yet sign extended). So after sign extending it we
+ * simply add it to IIP. We also need to reset the EI field of the IPSR to zero,
+ * i.e., the slot to restart into.
+ *
+ * cr.imm contains zero_ext(imm21)
+ */
+ mov r18=cr.iim
+ ;;
+ mov r17=cr.iip
+ shl r18=r18,43 // put sign bit in position (43=64-21)
+ ;;
+
+ mov r16=cr.ipsr
+ shr r18=r18,39 // sign extend (39=43-4)
+ ;;
+
+ add r17=r17,r18 // now add the offset
+ ;;
+ mov cr.iip=r17
+ dep r16=0,r16,41,2 // clear EI
+ ;;
+
+ mov cr.ipsr=r16
+ ;;
+
+ rfi // and go back
+END(speculation_vector)
+
+ .org ia64_ivt+0x5800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5800 Entry 28 (size 16 bundles) Reserved
+ DBG_FAULT(28)
+ FAULT(28)
+
+ .org ia64_ivt+0x5900
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56)
+ENTRY(debug_vector)
+ DBG_FAULT(29)
+ FAULT(29)
+END(debug_vector)
+
+ .org ia64_ivt+0x5a00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
+ENTRY(unaligned_access)
+ DBG_FAULT(30)
+ mov r31=pr // prepare to save predicates
+ ;;
+ br.sptk.many dispatch_unaligned_handler
+END(unaligned_access)
+
+ .org ia64_ivt+0x5b00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57)
+ENTRY(unsupported_data_reference)
+ DBG_FAULT(31)
+ FAULT(31)
+END(unsupported_data_reference)
+
+ .org ia64_ivt+0x5c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64)
+ENTRY(floating_point_fault)
+ DBG_FAULT(32)
+ FAULT(32)
+END(floating_point_fault)
+
+ .org ia64_ivt+0x5d00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66)
+ENTRY(floating_point_trap)
+ DBG_FAULT(33)
+ FAULT(33)
+END(floating_point_trap)
+
+ .org ia64_ivt+0x5e00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66)
+ENTRY(lower_privilege_trap)
+ DBG_FAULT(34)
+ FAULT(34)
+END(lower_privilege_trap)
+
+ .org ia64_ivt+0x5f00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68)
+ENTRY(taken_branch_trap)
+ DBG_FAULT(35)
+ FAULT(35)
+END(taken_branch_trap)
+
+ .org ia64_ivt+0x6000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69)
+ENTRY(single_step_trap)
+ DBG_FAULT(36)
+ FAULT(36)
+END(single_step_trap)
+
+ .org ia64_ivt+0x6100
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6100 Entry 37 (size 16 bundles) Reserved
+ DBG_FAULT(37)
+ FAULT(37)
+
+ .org ia64_ivt+0x6200
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6200 Entry 38 (size 16 bundles) Reserved
+ DBG_FAULT(38)
+ FAULT(38)
+
+ .org ia64_ivt+0x6300
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6300 Entry 39 (size 16 bundles) Reserved
+ DBG_FAULT(39)
+ FAULT(39)
+
+ .org ia64_ivt+0x6400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6400 Entry 40 (size 16 bundles) Reserved
+ DBG_FAULT(40)
+ FAULT(40)
+
+ .org ia64_ivt+0x6500
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6500 Entry 41 (size 16 bundles) Reserved
+ DBG_FAULT(41)
+ FAULT(41)
+
+ .org ia64_ivt+0x6600
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6600 Entry 42 (size 16 bundles) Reserved
+ DBG_FAULT(42)
+ FAULT(42)
+
+ .org ia64_ivt+0x6700
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6700 Entry 43 (size 16 bundles) Reserved
+ DBG_FAULT(43)
+ FAULT(43)
+
+ .org ia64_ivt+0x6800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6800 Entry 44 (size 16 bundles) Reserved
+ DBG_FAULT(44)
+ FAULT(44)
+
+ .org ia64_ivt+0x6900
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76,77)
+ENTRY(ia32_exception)
+ DBG_FAULT(45)
+ FAULT(45)
+END(ia32_exception)
+
+ .org ia64_ivt+0x6a00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept (30,31,59,70,71)
+ENTRY(ia32_intercept)
+ DBG_FAULT(46)
+#ifdef CONFIG_IA32_SUPPORT
+ mov r31=pr
+ mov r16=cr.isr
+ ;;
+ extr.u r17=r16,16,8 // get ISR.code
+ mov r18=ar.eflag
+ mov r19=cr.iim // old eflag value
+ ;;
+ cmp.ne p6,p0=2,r17
+(p6) br.cond.spnt 1f // not a system flag fault
+ xor r16=r18,r19
+ ;;
+ extr.u r17=r16,18,1 // get the eflags.ac bit
+ ;;
+ cmp.eq p6,p0=0,r17
+(p6) br.cond.spnt 1f // eflags.ac bit didn't change
+ ;;
+ mov pr=r31,-1 // restore predicate registers
+ rfi
+
+1:
+#endif // CONFIG_IA32_SUPPORT
+ FAULT(46)
+END(ia32_intercept)
+
+ .org ia64_ivt+0x6b00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt (74)
+ENTRY(ia32_interrupt)
+ DBG_FAULT(47)
+#ifdef CONFIG_IA32_SUPPORT
+ mov r31=pr
+ br.sptk.many dispatch_to_ia32_handler
+#else
+ FAULT(47)
+#endif
+END(ia32_interrupt)
+
+ .org ia64_ivt+0x6c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6c00 Entry 48 (size 16 bundles) Reserved
+ DBG_FAULT(48)
+ FAULT(48)
+
+ .org ia64_ivt+0x6d00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6d00 Entry 49 (size 16 bundles) Reserved
+ DBG_FAULT(49)
+ FAULT(49)
+
+ .org ia64_ivt+0x6e00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6e00 Entry 50 (size 16 bundles) Reserved
+ DBG_FAULT(50)
+ FAULT(50)
+
+ .org ia64_ivt+0x6f00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x6f00 Entry 51 (size 16 bundles) Reserved
+ DBG_FAULT(51)
+ FAULT(51)
+
+ .org ia64_ivt+0x7000
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7000 Entry 52 (size 16 bundles) Reserved
+ DBG_FAULT(52)
+ FAULT(52)
+
+ .org ia64_ivt+0x7100
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7100 Entry 53 (size 16 bundles) Reserved
+ DBG_FAULT(53)
+ FAULT(53)
+
+ .org ia64_ivt+0x7200
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7200 Entry 54 (size 16 bundles) Reserved
+ DBG_FAULT(54)
+ FAULT(54)
+
+ .org ia64_ivt+0x7300
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7300 Entry 55 (size 16 bundles) Reserved
+ DBG_FAULT(55)
+ FAULT(55)
+
+ .org ia64_ivt+0x7400
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7400 Entry 56 (size 16 bundles) Reserved
+ DBG_FAULT(56)
+ FAULT(56)
+
+ .org ia64_ivt+0x7500
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7500 Entry 57 (size 16 bundles) Reserved
+ DBG_FAULT(57)
+ FAULT(57)
+
+ .org ia64_ivt+0x7600
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7600 Entry 58 (size 16 bundles) Reserved
+ DBG_FAULT(58)
+ FAULT(58)
+
+ .org ia64_ivt+0x7700
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7700 Entry 59 (size 16 bundles) Reserved
+ DBG_FAULT(59)
+ FAULT(59)
+
+ .org ia64_ivt+0x7800
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7800 Entry 60 (size 16 bundles) Reserved
+ DBG_FAULT(60)
+ FAULT(60)
+
+ .org ia64_ivt+0x7900
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7900 Entry 61 (size 16 bundles) Reserved
+ DBG_FAULT(61)
+ FAULT(61)
+
+ .org ia64_ivt+0x7a00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7a00 Entry 62 (size 16 bundles) Reserved
+ DBG_FAULT(62)
+ FAULT(62)
+
+ .org ia64_ivt+0x7b00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7b00 Entry 63 (size 16 bundles) Reserved
+ DBG_FAULT(63)
+ FAULT(63)
+
+ .org ia64_ivt+0x7c00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7c00 Entry 64 (size 16 bundles) Reserved
+ DBG_FAULT(64)
+ FAULT(64)
+
+ .org ia64_ivt+0x7d00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7d00 Entry 65 (size 16 bundles) Reserved
+ DBG_FAULT(65)
+ FAULT(65)
+
+ .org ia64_ivt+0x7e00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7e00 Entry 66 (size 16 bundles) Reserved
+ DBG_FAULT(66)
+ FAULT(66)
+
+ .org ia64_ivt+0x7f00
+/////////////////////////////////////////////////////////////////////////////////////////
+// 0x7f00 Entry 67 (size 16 bundles) Reserved
+ DBG_FAULT(67)
+ FAULT(67)
+
+#ifdef CONFIG_IA32_SUPPORT
+
+ /*
+ * There is no particular reason for this code to be here, other than that
+ * there happens to be space here that would go unused otherwise. If this
+ * fault ever gets "unreserved", simply moved the following code to a more
+ * suitable spot...
+ */
+
+ // IA32 interrupt entry point
+
+ENTRY(dispatch_to_ia32_handler)
+ SAVE_MIN
+ ;;
+ mov r14=cr.isr
+ ssm psr.ic | PSR_DEFAULT_BITS
+ ;;
+ srlz.i // guarantee that interruption collection is on
+ ;;
+(p15) ssm psr.i
+ adds r3=8,r2 // Base pointer for SAVE_REST
+ ;;
+ SAVE_REST
+ ;;
+ mov r15=0x80
+ shr r14=r14,16 // Get interrupt number
+ ;;
+ cmp.ne p6,p0=r14,r15
+(p6) br.call.dpnt.many b6=non_ia32_syscall
+
+ adds r14=IA64_PT_REGS_R8_OFFSET + 16,sp // 16 byte hole per SW conventions
+ adds r15=IA64_PT_REGS_R1_OFFSET + 16,sp
+ ;;
+ cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
+ ld8 r8=[r14] // get r8
+ ;;
+ st8 [r15]=r8 // save original EAX in r1 (IA32 procs don't use the GP)
+ ;;
+ alloc r15=ar.pfs,0,0,6,0 // must first in an insn group
+ ;;
+ ld4 r8=[r14],8 // r8 == eax (syscall number)
+ mov r15=IA32_NR_syscalls
+ ;;
+ cmp.ltu.unc p6,p7=r8,r15
+ ld4 out1=[r14],8 // r9 == ecx
+ ;;
+ ld4 out2=[r14],8 // r10 == edx
+ ;;
+ ld4 out0=[r14] // r11 == ebx
+ adds r14=(IA64_PT_REGS_R13_OFFSET) + 16,sp
+ ;;
+ ld4 out5=[r14],PT(R14)-PT(R13) // r13 == ebp
+ ;;
+ ld4 out3=[r14],PT(R15)-PT(R14) // r14 == esi
+ adds r2=TI_FLAGS+IA64_TASK_SIZE,r13
+ ;;
+ ld4 out4=[r14] // r15 == edi
+ movl r16=ia32_syscall_table
+ ;;
+(p6) shladd r16=r8,3,r16 // force ni_syscall if not valid syscall number
+ ld4 r2=[r2] // r2 = current_thread_info()->flags
+ ;;
+ ld8 r16=[r16]
+ and r2=_TIF_SYSCALL_TRACEAUDIT,r2 // mask trace or audit
+ ;;
+ mov b6=r16
+ movl r15=ia32_ret_from_syscall
+ cmp.eq p8,p0=r2,r0
+ ;;
+ mov rp=r15
+(p8) br.call.sptk.many b6=b6
+ br.cond.sptk ia32_trace_syscall
+
+non_ia32_syscall:
+ alloc r15=ar.pfs,0,0,2,0
+ mov out0=r14 // interrupt #
+ add out1=16,sp // pointer to pt_regs
+ ;; // avoid WAW on CFM
+ br.call.sptk.many rp=ia32_bad_interrupt
+.ret1: movl r15=ia64_leave_kernel
+ ;;
+ mov rp=r15
+ br.ret.sptk.many rp
+END(dispatch_to_ia32_handler)
+
+#endif /* CONFIG_IA32_SUPPORT */
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt-sfvhpt.in
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt-sfvhpt.in 2006-05-05 10:09:38.000000000 +1000
@@ -0,0 +1,444 @@
+/*
+ * This file is to be processed and inserted into the actual ivt.S
+ *
+ * Any variable $name in ivt.S will be replaced with what is between
+ *__begin_name__ and __end_name__ in this file.
+ *
+ */
+
+//vhpt_miss
+__begin_vhpt_miss_handler__
+ mov r16=cr.ifa // get address that caused the TLB miss
+#ifdef CONFIG_HUGETLB_PAGE
+ movl r18=PAGE_SHIFT
+ mov r25=cr.itir
+#endif
+ ;;
+ rsm psr.dt // use physical addressing for data
+ mov r31=pr // save the predicate registers
+ mov r19=IA64_KR(PT_BASE) // get page table base address
+ shl r21=r16,3 // shift bit 60 into sign bit
+ shr.u r17=r16,61 // get the region number into r17
+ ;;
+ shr.u r22=r21,3
+#ifdef CONFIG_HUGETLB_PAGE
+ extr.u r26=r25,2,6
+ ;;
+ cmp.ne p8,p0=r18,r26
+ sub r27=r26,r18
+ ;;
+(p8) dep r25=r18,r25,2,6
+(p8) shr r22=r22,r27
+#endif
+ ;;
+ cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5?
+ shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit
+ ;;
+(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
+
+ srlz.d
+ LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
+
+ .pred.rel "mutex", p6, p7
+(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
+(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
+ ;;
+(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
+(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
+ cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
+#ifdef CONFIG_PGTABLE_4
+ shr.u r28=r22,PUD_SHIFT // shift pud index into position
+#else
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+#endif
+ ;;
+ ld8 r17=[r17] // get *pgd (may be 0)
+ ;;
+(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
+#ifdef CONFIG_PGTABLE_4
+ dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
+ ;;
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+(p7) ld8 r29=[r28] // get *pud (may be 0)
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL?
+ dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
+#else
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
+#endif
+ ;;
+(p7) ld8 r20=[r17] // get *pmd (may be 0)
+ shr.u r19=r22,PAGE_SHIFT // shift pte index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL?
+ dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
+ ;;
+(p7) ld8 r18=[r21] // read *pte
+ mov r19=cr.isr // cr.isr bit 32 tells us if this is an insn miss
+ ;;
+(p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared?
+ mov r22=cr.iha // get the VHPT address that caused the TLB miss
+ ;; // avoid RAW on p7
+(p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss?
+ dep r23=0,r20,0,PAGE_SHIFT // clear low bits to get page address
+ ;;
+(p10) itc.i r18 // insert the instruction TLB entry
+(p11) itc.d r18 // insert the data TLB entry
+(p6) br.cond.spnt.many page_fault // handle bad address/page not present (page fault)
+ mov cr.ifa=r22
+
+#ifdef CONFIG_HUGETLB_PAGE
+(p8) mov cr.itir=r25 // change to default page-size for VHPT
+#endif
+
+ /*
+ * Now compute and insert the TLB entry for the virtual page table. We never
+ * execute in a page table page so there is no need to set the exception deferral
+ * bit.
+ */
+ adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23
+ ;;
+(p7) itc.d r24
+ ;;
+#ifdef CONFIG_SMP
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ /*
+ * Re-check pagetable entry. If they changed, we may have received a ptc.g
+ * between reading the pagetable and the "itc". If so, flush the entry we
+ * inserted and retry. At this point, we have:
+ *
+ * r28 = equivalent of pud_offset(pgd, ifa)
+ * r17 = equivalent of pmd_offset(pud, ifa)
+ * r21 = equivalent of pte_offset(pmd, ifa)
+ *
+ * r29 = *pud
+ * r20 = *pmd
+ * r18 = *pte
+ */
+ ld8 r25=[r21] // read *pte again
+ ld8 r26=[r17] // read *pmd again
+#ifdef CONFIG_PGTABLE_4
+ ld8 r19=[r28] // read *pud again
+#endif
+ cmp.ne p6,p7=r0,r0
+ ;;
+ cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change
+#ifdef CONFIG_PGTABLE_4
+ cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change
+#endif
+ mov r27=PAGE_SHIFT<<2
+ ;;
+(p6) ptc.l r22,r27 // purge PTE page translation
+(p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change
+ ;;
+(p6) ptc.l r16,r27 // purge translation
+#endif
+
+ mov pr=r31,-1 // restore predicate registers
+ rfi
+__end_vhpt_miss_handler__
+
+// itlb_miss
+__begin_itlb_miss_handler__
+ mov r16=cr.ifa // get virtual address
+ mov r29=b0 // save b0
+ mov r31=pr // save predicates
+.itlb_fault:
+ mov r17=cr.iha // get virtual address of PTE
+ movl r30=1f // load nested fault continuation point
+ ;;
+1: ld8 r18=[r17] // read *pte
+ ;;
+ mov b0=r29
+ tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
+(p6) br.cond.spnt page_fault
+ ;;
+ itc.i r18
+ ;;
+#ifdef CONFIG_SMP
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r19=[r17] // read *pte again and see if same
+ mov r20=PAGE_SHIFT<<2 // setup page size for purge
+ ;;
+ cmp.ne p7,p0=r18,r19
+ ;;
+(p7) ptc.l r16,r20
+#endif
+ mov pr=r31,-1
+ rfi
+__end_itlb_miss_handler__
+
+// dtlb_miss
+__begin_dtlb_miss_handler__
+ mov r16=cr.ifa // get virtual address
+ mov r29=b0 // save b0
+ mov r31=pr // save predicates
+dtlb_fault:
+ mov r17=cr.iha // get virtual address of PTE
+ movl r30=1f // load nested fault continuation point
+ ;;
+1: ld8 r18=[r17] // read *pte
+ ;;
+ mov b0=r29
+ tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
+(p6) br.cond.spnt page_fault
+ ;;
+ itc.d r18
+ ;;
+#ifdef CONFIG_SMP
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r19=[r17] // read *pte again and see if same
+ mov r20=PAGE_SHIFT<<2 // setup page size for purge
+ ;;
+ cmp.ne p7,p0=r18,r19
+ ;;
+(p7) ptc.l r16,r20
+#endif
+ mov pr=r31,-1
+ rfi
+__end_dtlb_miss_handler__
+
+// nested_dtlb_miss
+__begin_nested_dtlb_miss_handler__
+ rsm psr.dt // switch to using physical data addressing
+ mov r19=IA64_KR(PT_BASE) // get the page table base address
+ shl r21=r16,3 // shift bit 60 into sign bit
+ mov r18=cr.itir
+ ;;
+ shr.u r17=r16,61 // get the region number into r17
+ extr.u r18=r18,2,6 // get the faulting page size
+ ;;
+ cmp.eq p6,p7=5,r17 // is faulting address in region 5?
+ add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
+ add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
+ ;;
+ shr.u r22=r16,r22
+ shr.u r18=r16,r18
+(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
+
+ srlz.d
+ LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
+
+ .pred.rel "mutex", p6, p7
+(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
+(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
+ ;;
+(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
+(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
+ cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
+#ifdef CONFIG_PGTABLE_4
+ shr.u r18=r22,PUD_SHIFT // shift pud index into position
+#else
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+#endif
+ ;;
+ ld8 r17=[r17] // get *pgd (may be 0)
+ ;;
+(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
+ ;;
+#ifdef CONFIG_PGTABLE_4
+(p7) ld8 r17=[r17] // get *pud (may be 0)
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
+ ;;
+#endif
+(p7) ld8 r17=[r17] // get *pmd (may be 0)
+ shr.u r19=r22,PAGE_SHIFT // shift pte index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
+ dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
+(p6) br.cond.spnt page_fault
+ mov b0=r30
+ br.sptk.many b0 // return to continuation point
+__end_nested_dtlb_miss_handler__
+
+// dirty bit
+__begin_dirty_bit_handler__
+ /*
+ * What we do here is to simply turn on the dirty bit in the PTE. We need to
+ * update both the page-table and the TLB entry. To efficiently access the PTE,
+ * we address it through the virtual page table. Most likely, the TLB entry for
+ * the relevant virtual page table page is still present in the TLB so we can
+ * normally do this without additional TLB misses. In case the necessary virtual
+ * page table TLB entry isn't present, we take a nested TLB miss hit where we look
+ * up the physical address of the L3 PTE and then continue at label 1 below.
+ */
+ mov r16=cr.ifa // get the address that caused the fault
+ movl r30=1f // load continuation point in case of nested fault
+ ;;
+ thash r17=r16 // compute virtual address of L3 PTE
+ mov r29=b0 // save b0 in case of nested fault
+ mov r31=pr // save pr
+#ifdef CONFIG_SMP
+ mov r28=ar.ccv // save ar.ccv
+ ;;
+1: ld8 r18=[r17]
+ ;; // avoid RAW on r18
+ mov ar.ccv=r18 // set compare value for cmpxchg
+ or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
+ ;;
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
+ mov r24=PAGE_SHIFT<<2
+ ;;
+(p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
+ ;;
+(p6) itc.d r25 // install updated PTE
+ ;;
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r18=[r17] // read PTE again
+ ;;
+ cmp.eq p6,p7=r18,r25 // is it same as the newly installed
+ ;;
+(p7) ptc.l r16,r24
+ mov b0=r29 // restore b0
+ mov ar.ccv=r28
+#else
+ ;;
+1: ld8 r18=[r17]
+ ;; // avoid RAW on r18
+ or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
+ mov b0=r29 // restore b0
+ ;;
+ st8 [r17]=r18 // store back updated PTE
+ itc.d r18 // install updated PTE
+#endif
+ mov pr=r31,-1 // restore pr
+ rfi
+__end_dirty_bit_handler__
+
+// iaccess bit
+__begin_iaccess_bit_handler__
+ // Like dirty bit handler, except for instruction access
+ mov r16=cr.ifa // get the address that caused the fault
+ movl r30=1f // load continuation point in case of nested fault
+ mov r31=pr // save predicates
+#ifdef CONFIG_ITANIUM
+ /*
+ * Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
+ */
+ mov r17=cr.ipsr
+ ;;
+ mov r18=cr.iip
+ tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set?
+ ;;
+(p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa
+#endif /* CONFIG_ITANIUM */
+ ;;
+ thash r17=r16 // compute virtual address of L3 PTE
+ mov r29=b0 // save b0 in case of nested fault)
+#ifdef CONFIG_SMP
+ mov r28=ar.ccv // save ar.ccv
+ ;;
+1: ld8 r18=[r17]
+ ;;
+ mov ar.ccv=r18 // set compare value for cmpxchg
+ or r25=_PAGE_A,r18 // set the accessed bit
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
+ ;;
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
+ mov r24=PAGE_SHIFT<<2
+ ;;
+(p6) cmp.eq p6,p7=r26,r18 // Only if page present
+ ;;
+(p6) itc.i r25 // install updated PTE
+ ;;
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r18=[r17] // read PTE again
+ ;;
+ cmp.eq p6,p7=r18,r25 // is it same as the newly installed
+ ;;
+(p7) ptc.l r16,r24
+ mov b0=r29 // restore b0
+ mov ar.ccv=r28
+#else /* !CONFIG_SMP */
+ ;;
+1: ld8 r18=[r17]
+ ;;
+ or r18=_PAGE_A,r18 // set the accessed bit
+ mov b0=r29 // restore b0
+ ;;
+ st8 [r17]=r18 // store back updated PTE
+ itc.i r18 // install updated PTE
+#endif /* !CONFIG_SMP */
+ mov pr=r31,-1
+ rfi
+__end_iaccess_bit_handler__
+
+// daccess bit
+__begin_daccess_bit_handler__
+ // Like dirty bit handler, except for data access
+ mov r16=cr.ifa // get the address that caused the fault
+ movl r30=1f // load continuation point in case of nested fault
+ ;;
+ thash r17=r16 // compute virtual address of L3 PTE
+ mov r31=pr
+ mov r29=b0 // save b0 in case of nested fault)
+#ifdef CONFIG_SMP
+ mov r28=ar.ccv // save ar.ccv
+ ;;
+1: ld8 r18=[r17]
+ ;; // avoid RAW on r18
+ mov ar.ccv=r18 // set compare value for cmpxchg
+ or r25=_PAGE_A,r18 // set the dirty bit
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
+ ;;
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present
+ mov r24=PAGE_SHIFT<<2
+ ;;
+(p6) cmp.eq p6,p7=r26,r18 // Only if page is present
+ ;;
+(p6) itc.d r25 // install updated PTE
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+ ;;
+ ld8 r18=[r17] // read PTE again
+ ;;
+ cmp.eq p6,p7=r18,r25 // is it same as the newly installed
+ ;;
+(p7) ptc.l r16,r24
+ mov ar.ccv=r28
+#else
+ ;;
+1: ld8 r18=[r17]
+ ;; // avoid RAW on r18
+ or r18=_PAGE_A,r18 // set the accessed bit
+ ;;
+ st8 [r17]=r18 // store back updated PTE
+ itc.d r18 // install updated PTE
+#endif
+ mov b0=r29 // restore b0
+ mov pr=r31,-1
+ rfi
+__end_daccess_bit_handler__
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/Makefile
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/kernel/Makefile 2006-05-05 10:06:54.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/Makefile 2006-05-05 10:09:38.000000000 +1000
@@ -59,3 +59,8 @@
# We must build gate.so before we can assemble it.
# Note: kbuild does not track this dependency due to usage of .incbin
$(obj)/gate-data.o: $(obj)/gate.so
+
+# The real ivt.S needs to be built
+AFLAGS_ivt.o += -I$(srctree)/arch/ia64/kernel
+$(obj)/ivt.S: $(src)/ivt.S.in $(src)/ivt-sfvhpt.in
+ $(srctree)/arch/ia64/scripts/merge.py $@ $^
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RFC 3/6] LVHPT - ivt.S long format support
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
2006-05-10 3:42 ` [RFC 1/6] LVHPT - Remove static ivt.S Ian Wienand
2006-05-10 3:42 ` [RFC 2/6] LVHPT - Create ivt.S dyanamically Ian Wienand
@ 2006-05-10 3:42 ` Ian Wienand
2006-05-10 3:42 ` [RFC 4/6] LVHPT - find architectured page size help Ian Wienand
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
LVHPT ivt long format support
Add templates for long format VHPT support, to be merged into ivt.S
Signed-Off-By: Ian Wienand <ianw@gelato.unsw.edu.au>
---
Kconfig | 10 +
kernel/Makefile | 9
kernel/ivt-lfvhpt.in | 466 +++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 484 insertions(+), 1 deletion(-)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/Makefile
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/kernel/Makefile 2006-05-05 10:09:38.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/Makefile 2006-05-05 10:12:36.000000000 +1000
@@ -62,5 +62,12 @@
# The real ivt.S needs to be built
AFLAGS_ivt.o += -I$(srctree)/arch/ia64/kernel
-$(obj)/ivt.S: $(src)/ivt.S.in $(src)/ivt-sfvhpt.in
+
+ifeq ($(CONFIG_IA64_LONG_FORMAT_VHPT),)
+fault_handler_in = ivt-sfvhpt.in
+else
+fault_handler_in = ivt-lfvhpt.in
+endif
+
+$(obj)/ivt.S: $(src)/ivt.S.in $(src)/$(fault_handler_in)
$(srctree)/arch/ia64/scripts/merge.py $@ $^
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt-lfvhpt.in
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/ivt-lfvhpt.in 2006-05-05 10:12:36.000000000 +1000
@@ -0,0 +1,466 @@
+/*
+ * Long Format VHPT fault handlers
+ *
+ * Copyright (C) 2006; see ivt.S for original authors
+ * These bits by
+ * Matthew Chapman <matthewc@cse.unsw.edu.au>
+ * Darren Williams <darren.williams@nicta.com.au>
+ * Ian Wienand <ianw@gelato.unsw.edu.au>
+ *
+ * This file is to be processed and inserted into the actual ivt.S
+ *
+ * Any variable $name in ivt.S will be replaced with what is between
+ *__begin_name__ and __end_name__ in this file.
+ *
+ */
+
+//vhpt_miss
+__begin_vhpt_miss_handler__
+// This fault can not happen with long format VHPT
+ FAULT(0)
+__end_vhpt_miss_handler__
+
+// itlb_miss
+__begin_itlb_miss_handler__
+ mov r16=cr.ifa // get virtual address
+ mov r29=b0 // save b0
+ mov r31=pr // save predicates
+ ;;
+.itlb_fault:
+ // walk the page table to satisfy this miss
+ rsm psr.dt // switch to using physical data addressing
+ mov r19=IA64_KR(PT_BASE) // get the page table base address
+ shl r21=r16,3 // shift bit 60 into sign bit
+ mov r18=cr.itir
+ ;;
+ shr.u r17=r16,61 // get the region number into ppte
+ extr.u r18=r18,2,6 // get the faulting page size
+ ;;
+ cmp.eq p6,p7=5,r17 // is faulting address in region 5?
+ add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
+ add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
+ ;;
+ shr.u r22=r16,r22
+ shr.u r18=r16,r18
+(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
+
+ srlz.d
+ LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
+
+ .pred.rel "mutex", p6, p7
+(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
+(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
+ ;;
+(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // ppte=pgd_offset for region 5
+(p7) dep r17=r18,r17,3,(PAGE_SHIFT-3)-3 // ppte=pgd_offset for region[0-4]
+ cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
+#ifdef CONFIG_PGTABLE_4
+ shr.u r18=r22,PUD_SHIFT // shift pud index into position
+#else
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+#endif
+ ;;
+ ld8 r17=[r17] // get *pgd (may be 0)
+ ;;
+(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // ppte=p[u|m]d_offset(pgd,addr)
+ ;;
+#ifdef CONFIG_PGTABLE_4
+(p7) ld8 r17=[r17] // get *pud (may be 0)
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // ppte=pmd_offset(pud,addr)
+ ;;
+#endif
+(p7) ld8 r17=[r17] // get *pmd (may be 0)
+ shr.u r19=r22,PAGE_SHIFT // shift pte index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
+ dep r17=r19,r17,3,(PAGE_SHIFT-3) // ppte=pte_offset(pmd,addr)
+ ;;
+(p7) ld8 r18=[r17]
+ ;;
+(p7) tbit.z p6,p0=r18,_PAGE_P_BIT /* page present bit cleared? */
+(p6) br.cond.spnt page_fault
+ // insert vhpt mapping
+ mov r22=cr.iha
+ mov r26=cr.itir
+ ;;
+ tpa r22=r22 /* make hash address physical */
+ ttag r25=r16
+ ;;
+ add r24=16,r22
+ ;;
+ st8 [r22]=r18,8 /* fill out VHPT entry */
+ st8 [r24]=r25,8
+ ;;
+ st8 [r22]=r26,8
+ st8 [r24]=r17
+ //insert actual translation
+ ;;
+ itc.i r18
+ ;;
+#ifdef CONFIG_SMP
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r19=[r17] // read *pte again and see if same
+ mov r20=PAGE_SHIFT<<2 // setup page size for purge
+ ;;
+ cmp.ne p7,p0=r18,r19
+ ;;
+(p7) dep r25=-1,r0,63,1 /* set tag-invalid bit */
+ ;;
+(p7) st8 [r22]=r25 /* hpte already points to tag (see above) */
+(p7) ptc.l r16,r20
+#endif
+ mov pr=r31,-1
+ rfi
+__end_itlb_miss_handler__
+
+// dtlb_miss
+__begin_dtlb_miss_handler__
+ mov r16=cr.ifa // get virtual address
+ mov r29=b0 // save b0
+ mov r31=pr // save predicates
+ ;;
+.dtlb_fault:
+ // walk the page table to satisfy this miss
+ rsm psr.dt // switch to using physical data addressing
+ mov r19=IA64_KR(PT_BASE) // get the page table base address
+ shl r21=r16,3 // shift bit 60 into sign bit
+ mov r18=cr.itir
+ ;;
+ shr.u r17=r16,61 // get the region number into ppte
+ extr.u r18=r18,2,6 // get the faulting page size
+ ;;
+ cmp.eq p6,p7=5,r17 // is faulting address in region 5?
+ add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
+ add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
+ ;;
+ shr.u r22=r16,r22
+ shr.u r18=r16,r18
+(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
+
+ srlz.d
+ LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
+
+ .pred.rel "mutex", p6, p7
+(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
+(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
+ ;;
+(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // ppte=pgd_offset for region 5
+(p7) dep r17=r18,r17,3,(PAGE_SHIFT-3)-3 // ppte=pgd_offset for region[0-4]
+ cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
+#ifdef CONFIG_PGTABLE_4
+ shr.u r18=r22,PUD_SHIFT // shift pud index into position
+#else
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+#endif
+ ;;
+ ld8 r17=[r17] // get *pgd (may be 0)
+ ;;
+(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // ppte=p[u|m]d_offset(pgd,addr)
+ ;;
+#ifdef CONFIG_PGTABLE_4
+(p7) ld8 r17=[r17] // get *pud (may be 0)
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // ppte=pmd_offset(pud,addr)
+ ;;
+#endif
+(p7) ld8 r17=[r17] // get *pmd (may be 0)
+ shr.u r19=r22,PAGE_SHIFT // shift pte index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
+ dep r17=r19,r17,3,(PAGE_SHIFT-3) // ppte=pte_offset(pmd,addr)
+ ;;
+(p7) ld8 r18=[r17]
+ ;;
+(p7) tbit.z p6,p0=r18,_PAGE_P_BIT /* page present bit cleared? */
+(p6) br.cond.spnt page_fault
+// insert vhpt mapping
+ mov r22=cr.iha
+ mov r26=cr.itir
+ ;;
+ tpa r22=r22 /* make hash address physical */
+ ttag r25=r16
+ ;;
+ add r24=16,r22
+ ;;
+ st8 [r22]=r18,8 /* fill out VHPT entry */
+ st8 [r24]=r25,8
+ ;;
+ st8 [r22]=r26,8
+ st8 [r24]=r17
+//insert actual translation
+ ;;
+ itc.d r18
+ ;;
+#ifdef CONFIG_SMP
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r19=[r17] // read *pte again and see if same
+ mov r20=PAGE_SHIFT<<2 // setup page size for purge
+ ;;
+ cmp.ne p7,p0=r18,r19
+ ;;
+(p7) dep r25=-1,r0,63,1 /* set tag-invalid bit */
+ ;;
+(p7) st8 [r22]=r25 /* hpte already points to tag (see above) */
+(p7) ptc.l r16,r20
+#endif
+ mov pr=r31,-1
+ rfi
+__end_dtlb_miss_handler__
+
+// nested_dtlb_miss
+__begin_nested_dtlb_miss_handler__
+// This fault can not happen with long format VHPT
+ FAULT(5)
+__end_nested_dtlb_miss_handler__
+
+// dirty bit
+__begin_dirty_bit_handler__
+ mov r16=cr.ifa // get the address that caused the fault
+ mov r29=b0 // save b0 for nested fault (XXX)
+ mov r31=pr // save pr
+ /*
+ * we try to find the address via the VHPT, if not fall back to
+ * dtlb_fault.
+ */
+ ;;
+ thash r28=r16
+ rsm psr.dt
+ ;;
+ tpa r22=r28 // make hash address physical
+ ttag r25=r16
+ ;;
+ srlz.d
+ add r24=16,r22
+ add r17=24,r22
+ ;;
+ ld8 r26=[r24] // load tag
+ ld8 r17=[r17]
+ ;;
+ cmp.ne p6,p7=r26, r25 // verify tag
+ ;;
+(p7) ld8 r18=[r17]
+ ;;
+(p6) mov cr.iha=r28 // set cr.iha only if we are going to take
+(p6) br.cond.spnt .dtlb_fault // the failfn fault - it depends on it.
+#ifdef CONFIG_SMP
+ mov r28=ar.ccv // save ar.ccv
+ ;;
+ mov ar.ccv=r18 // set compare value for cmpxchg
+ or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
+ ;;
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
+ mov r24=PAGE_SHIFT<<2
+ ;;
+(p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
+ ;;
+(p6) st8 [r22]=r18,16 // update VHPT
+(p6) itc.d r25 // install updated PTE
+ ;;
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r18=[r17] // read PTE again
+ ;;
+ cmp.eq p6,p7=r18,r25 // is it same as the newly installed
+ ;;
+(p7) dep r25=-1,r0,63,1 // set tag-invalid bit
+ ;;
+(p7) st8 [r22]=r25 // hpte already points to tag (see above)
+(p7) ptc.l r16,r24
+ mov b0=r29 // restore b0
+ mov ar.ccv=r28
+#else
+ ;;
+ or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
+ mov b0=r29 // restore b0
+ ;;
+ st8 [r17]=r18 // store back updated PTE
+(p0) st8 [r22]=r18,16 // update VHPT (p0 set from dtlb_fault)
+ itc.d r18 // install updated PTE
+#endif
+ mov pr=r31,-1 // restore pr
+ rfi
+__end_dirty_bit_handler__
+
+// iaccess bit
+__begin_iaccess_bit_handler__
+ // Like dirty bit handler, except for instruction access
+ mov r16=cr.ifa // get the address that caused the fault
+ mov r29=b0
+ mov r31=pr // save predicates
+ ;;
+ /*
+ * we try to find the address via the VHPT, if not fall back to
+ * dtlb_fault.
+ */
+ thash r28=r16
+ rsm psr.dt
+ ;;
+ tpa r22=r28 // make hash address physical
+ ttag r25=r16
+ ;;
+ srlz.d
+ add r24=16,r22
+ add r17=24,r22
+ ;;
+ ld8 r26=[r24] // load tag
+ ld8 r17=[r17]
+ ;;
+ cmp.ne p6,p7=r26, r25 // verify tag
+ ;;
+(p7) ld8 r18=[r17]
+ ;;
+(p6) mov cr.iha=r28 // set cr.iha only if we are going to take
+(p6) br.cond.spnt .itlb_fault // the failfn fault - it depends on it.
+#ifdef CONFIG_ITANIUM
+ /*
+ * Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
+ */
+ mov r17=cr.ipsr
+ ;;
+ mov r18=cr.iip
+ tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set?
+ ;;
+(p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa
+#endif /* CONFIG_ITANIUM */
+ ;;
+ mov b0=r29 // restore b0
+#ifdef CONFIG_SMP
+ mov r28=ar.ccv // save ar.ccv
+ ;;
+ mov ar.ccv=r18 // set compare value for cmpxchg
+ or r25=_PAGE_A,r18 // set the accessed bit
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
+ ;;
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
+ mov r24=PAGE_SHIFT<<2
+ ;;
+(p6) cmp.eq p6,p7=r26,r18 // Only if page present
+ ;;
+(p6) st8 [r22]=r18,16 // update VHPT
+(p6) itc.i r25 // install updated PTE
+ ;;
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r18=[r17] // read PTE again
+ ;;
+ cmp.eq p6,p7=r18,r25 // is it same as the newly installed
+ ;;
+(p7) dep r25=-1,r0,63,1 // set tag-invalid bit
+ ;;
+(p7) st8 [r22]=r25 // hpte already points to tag (see above)
+(p7) ptc.l r16,r24
+ mov b0=r29 // restore b0
+ mov ar.ccv=r28
+#else /* !CONFIG_SMP */
+ ;;
+ or r18=_PAGE_A,r18 // set the accessed bit
+ mov b0=r29 // restore b0
+ ;;
+ st8 [r17]=r18 // store back updated PTE
+(p0) st8 [r22]=r18,16 // update VHPT (p0 from itlb_miss)
+ itc.i r18 // install updated PTE
+#endif /* !CONFIG_SMP */
+ mov pr=r31,-1
+ rfi
+__end_iaccess_bit_handler__
+
+// daccess bit
+__begin_daccess_bit_handler__
+ // Like dirty bit handler, except for data access
+ mov r16=cr.ifa // get the address that caused the fault
+ mov r29=b0
+ mov r31=pr // save predicates
+ ;;
+ /*
+ * we try to find the address via the VHPT, if not fall back to
+ * dtlb_fault.
+ */
+ ;;
+ thash r28=r16
+ rsm psr.dt
+ ;;
+ tpa r22=r28 // make hash address physical
+ ttag r25=r16
+ ;;
+ srlz.d
+ add r24=16,r22
+ add r17=24,r22
+ ;;
+ ld8 r26=[r24] // load tag
+ ld8 r17=[r17]
+ ;;
+ cmp.ne p6,p7=r26, r25 // verify tag
+ ;;
+(p7) ld8 r18=[r17]
+ ;;
+(p6) mov cr.iha=r28 // set cr.iha only if we are going to take
+(p6) br.cond.spnt .dtlb_fault // the failfn fault - it depends on it.
+#ifdef CONFIG_SMP
+ mov r28=ar.ccv // save ar.ccv
+ ;;
+ mov ar.ccv=r18 // set compare value for cmpxchg
+ or r25=_PAGE_A,r18 // set the accessed bit
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
+ ;;
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
+ mov r24=PAGE_SHIFT<<2
+ ;;
+(p6) cmp.eq p6,p7=r26,r18 // Only if page present
+ ;;
+(p6) st8 [r22]=r18,16 // update VHPT
+(p6) itc.d r25 // install updated PTE
+ ;;
+ /*
+ * Tell the assemblers dependency-violation checker that the above "itc" instructions
+ * cannot possibly affect the following loads:
+ */
+ dv_serialize_data
+
+ ld8 r18=[r17] // read PTE again
+ ;;
+ cmp.eq p6,p7=r18,r25 // is it same as the newly installed
+ ;;
+(p7) dep r25=-1,r0,63,1 // set tag-invalid bit
+ ;;
+(p7) st8 [r22]=r25 // hpte already points to tag (see above)
+(p7) ptc.l r16,r24
+ mov b0=r29 // restore b0
+ mov ar.ccv=r28
+#else /* !CONFIG_SMP */
+ ;;
+ or r18=_PAGE_A,r18 // set the accessed bit
+ mov b0=r29 // restore b0
+ ;;
+ st8 [r17]=r18 // store back updated PTE
+(p0) st8 [r22]=r18,16 // update VHPT (p0 from itlb_miss)
+ itc.i r18 // install updated PTE
+#endif /* !CONFIG_SMP */
+ mov pr=r31,-1
+ rfi
+__end_daccess_bit_handler__
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/Kconfig
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/Kconfig 2006-05-05 10:02:39.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/Kconfig 2006-05-05 10:12:36.000000000 +1000
@@ -374,6 +374,16 @@
def_bool y
depends on NEED_MULTIPLE_NODES
+config IA64_LONG_FORMAT_VHPT
+ bool "Long format VHPT"
+ depends on !DISABLE_VHPT
+ help
+ The long format VHPT is an alternative hashed page table.
+ It is more TLB friendly, but less cache friendly. By its
+ self this may have a negative, neutral, or positive impact
+ on performance, depending on your workloads. If you're
+ unsure, answer N and keep the default short-format VHPT.
+
config IA32_SUPPORT
bool "Support for Linux/x86 binaries"
help
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RFC 4/6] LVHPT - find architectured page size help
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
` (2 preceding siblings ...)
2006-05-10 3:42 ` [RFC 3/6] LVHPT - ivt.S long format support Ian Wienand
@ 2006-05-10 3:42 ` Ian Wienand
2006-05-10 3:42 ` [RFC 5/6] LVHPT - setup infrastructure Ian Wienand
2006-05-10 3:42 ` [RFC 6/6] LVHPT - long format support functions Ian Wienand
5 siblings, 0 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
Find biggest page
Split out a handy function for finding the largest page the
architecture can map over a region.
Signed-Off-By: Ian Wienand <ianw@gelato.unsw.edu.au>
---
arch/ia64/mm/tlb.c | 43 +++++++++++++++++++++++--------------------
include/asm-ia64/tlb.h | 25 +++++++++++++++++++++++++
2 files changed, 48 insertions(+), 20 deletions(-)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/tlb.c
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/mm/tlb.c 2006-05-03 17:11:43.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/tlb.c 2006-05-10 09:13:08.000000000 +1000
@@ -26,11 +26,10 @@
#include <asm/pal.h>
#include <asm/tlbflush.h>
#include <asm/dma.h>
+#include <asm/pgtable.h>
+#include <asm/tlb.h>
-static struct {
- unsigned long mask; /* mask of supported purge page-sizes */
- unsigned long max_bits; /* log2 of largest supported purge page-size */
-} purge;
+struct ia64_page_sizes_t ia64_page_sizes;
struct ia64_ctx ia64_ctx = {
.lock = SPIN_LOCK_UNLOCKED,
@@ -138,7 +137,6 @@
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
- unsigned long size = end - start;
unsigned long nbits;
#ifndef CONFIG_SMP
@@ -148,12 +146,7 @@
}
#endif
- nbits = ia64_fls(size + 0xfff);
- while (unlikely (((1UL << nbits) & purge.mask) == 0) &&
- (nbits < purge.max_bits))
- ++nbits;
- if (nbits > purge.max_bits)
- nbits = purge.max_bits;
+ nbits = find_largest_page_size(end-start);
start &= ~((1UL << nbits) - 1);
preempt_disable();
@@ -173,19 +166,29 @@
}
EXPORT_SYMBOL(flush_tlb_range);
+/*
+ * We need this data early in the boot, so it gets called from
+ * setup_arch()
+ */
+void __devinit
+ia64_tlb_early_init (void)
+{
+ long status;
+ unsigned long tr_pgbits;
+
+ /* Setup valid page sizes for find_largest_page() */
+ if ((status = ia64_pal_vm_page_size(&tr_pgbits, &ia64_page_sizes.mask)) != 0) {
+ printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld;"
+ "defaulting to architected purge page-sizes.\n", status);
+ ia64_page_sizes.mask = 0x115557000UL;
+ }
+ ia64_page_sizes.max_bits = ia64_fls(ia64_page_sizes.mask);
+}
+
void __devinit
ia64_tlb_init (void)
{
ia64_ptce_info_t ptce_info;
- unsigned long tr_pgbits;
- long status;
-
- if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) {
- printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld;"
- "defaulting to architected purge page-sizes.\n", status);
- purge.mask = 0x115557000UL;
- }
- purge.max_bits = ia64_fls(purge.mask);
ia64_get_ptce(&ptce_info);
local_cpu_data->ptce_base = ptce_info.base;
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/tlb.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/tlb.h 2006-05-03 17:11:43.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/tlb.h 2006-05-10 09:04:39.000000000 +1000
@@ -200,6 +200,31 @@
tlb->end_addr = address + PAGE_SIZE;
}
+/*
+ * Find an architecture suitable page size based big enough to map
+ * input size. Return the number of bits; i.e. (1 << nbits) is the
+ * page size in bytes.
+ */
+struct ia64_page_sizes_t {
+ unsigned long mask; /* mask of supported page-sizes */
+ unsigned long max_bits; /* log2 of largest supported page-size */
+};
+
+ /* initalised in ia64_tlb_init() from EFI */
+extern struct ia64_page_sizes_t ia64_page_sizes;
+
+static inline unsigned long
+find_largest_page_size(unsigned long size)
+{
+ int nbits = ia64_fls(size + 0xfff);
+ while (unlikely (((1UL << nbits) & ia64_page_sizes.mask) == 0) &&
+ (nbits < ia64_page_sizes.max_bits))
+ ++nbits;
+ if (nbits > ia64_page_sizes.max_bits)
+ nbits = ia64_page_sizes.max_bits;
+ return nbits;
+}
+
#define tlb_migrate_finish(mm) platform_tlb_migrate_finish(mm)
#define tlb_start_vma(tlb, vma) do { } while (0)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RFC 5/6] LVHPT - setup infrastructure
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
` (3 preceding siblings ...)
2006-05-10 3:42 ` [RFC 4/6] LVHPT - find architectured page size help Ian Wienand
@ 2006-05-10 3:42 ` Ian Wienand
2006-05-10 3:42 ` [RFC 6/6] LVHPT - long format support functions Ian Wienand
5 siblings, 0 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
LVHPT setup
The following patch sets up the LVHPT on boot.
For the initial boot processor, we allocate the VHPT in ia64_mmu_init.
Other CPUs get the LVHPT allocated from do_boot_cpu before they are
woken up.
The logic is per CPU, but it attempts to choose a reasonable size that
can be pinned in the TLB. There are facilities to clamp it to a
specific size.
Signed-Off-By: Ian Wienand <ianw@gelato.unsw.edu.au>
---
Documentation/kernel-parameters.txt | 14 +++
arch/ia64/kernel/setup.c | 31 +++++++
arch/ia64/kernel/smpboot.c | 12 ++
arch/ia64/mm/Makefile | 1
arch/ia64/mm/init.c | 55 ++++++++++---
arch/ia64/mm/lvhpt.c | 150 ++++++++++++++++++++++++++++++++++++
include/asm-ia64/kregs.h | 1
include/asm-ia64/lvhpt.h | 24 +++++
include/asm-ia64/pgtable.h | 10 ++
9 files changed, 286 insertions(+), 12 deletions(-)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/setup.c
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/kernel/setup.c 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/setup.c 2006-05-10 10:01:49.000000000 +1000
@@ -62,6 +62,10 @@
#include <asm/unistd.h>
#include <asm/system.h>
+#if defined(CONFIG_IA64_LONG_FORMAT_VHPT)
+#include <asm/lvhpt.h>
+#endif
+
#if defined(CONFIG_SMP) && (IA64_CPU_SIZE > PAGE_SIZE)
# error "struct cpuinfo_ia64 too big!"
#endif
@@ -284,6 +288,18 @@
#endif
}
+static void __init parse_cmdline_early (char ** cmdline_p)
+{
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ char *p;
+ extern int lvhpt_bits_clamp_setup(char *s);
+
+ strlcpy(saved_command_line, *cmdline_p, COMMAND_LINE_SIZE);
+ if ((p = strstr(*cmdline_p, "lvhpt_bits_clamp=")))
+ lvhpt_bits_clamp_setup(p + 17);
+#endif
+}
+
static void __init
io_port_init (void)
{
@@ -400,12 +416,14 @@
void __init
setup_arch (char **cmdline_p)
{
+ extern void __devinit ia64_tlb_early_init(void);
+
unw_init();
ia64_patch_vtop((u64) __start___vtop_patchlist, (u64) __end___vtop_patchlist);
*cmdline_p = __va(ia64_boot_param->command_line);
- strlcpy(saved_command_line, *cmdline_p, COMMAND_LINE_SIZE);
+ parse_cmdline_early(cmdline_p);
efi_init();
io_port_init();
@@ -438,6 +456,17 @@
ia64_setup_printk_clock();
+ /* Setup some information about the TLBS */
+ ia64_tlb_early_init();
+
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ /*
+ * put this after all the ACPI walking so we can get the size
+ * of memory on nodes
+ */
+ compute_vhpt_size();
+#endif
+
#ifdef CONFIG_SMP
cpu_physical_id(0) = hard_smp_processor_id();
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/smpboot.c
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/kernel/smpboot.c 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/kernel/smpboot.c 2006-05-10 10:01:49.000000000 +1000
@@ -61,6 +61,11 @@
#include <asm/tlbflush.h>
#include <asm/unistd.h>
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+#include <asm/lvhpt.h>
+#endif
+
+
#define SMP_DEBUG 0
#if SMP_DEBUG
@@ -512,6 +517,13 @@
do_rest:
task_for_booting_cpu = c_idle.idle;
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ if (alloc_vhpt(cpu)) {
+ panic("Couldn't allocate VHPT on CPU %d\n", cpu);
+ }
+ Dprintk("Allocated long format VHPT for CPU %d at: 0x%lx, size: 0x%lx\n", cpu, vhpt_base[cpu], long_vhpt_size(cpu));
+#endif /* CONFIG_IA64_LONG_FORMAT_VHPT */
+
Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid);
set_brendez_area(cpu);
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/init.c
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/mm/init.c 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/init.c 2006-05-10 10:01:49.000000000 +1000
@@ -37,6 +37,10 @@
#include <asm/unistd.h>
#include <asm/mca.h>
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+#include <asm/lvhpt.h>
+#endif
+
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
DEFINE_PER_CPU(unsigned long *, __pgtable_quicklist);
@@ -338,7 +342,7 @@
void __devinit
ia64_mmu_init (void *my_cpu_data)
{
- unsigned long psr, pta, impl_va_bits;
+ unsigned long psr, pta;
extern void __devinit tlb_init (void);
#ifdef CONFIG_DISABLE_VHPT
@@ -347,16 +351,47 @@
# define VHPT_ENABLE_BIT 1
#endif
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ int cpu = smp_processor_id();
+
+ /* Allocate the VHPT for the boot processor. VHPT for other
+ * processors is allocated in smpboot.c:do_boot_cpu() as they
+ * are bought online.
+ */
+ if (cpu == 0)
+ {
+ unsigned long size = lvhpt_size(cpu);
+ lvhpt_per_cpu_info[0].base = (unsigned long)__alloc_bootmem(size, size, __pa(MAX_DMA_ADDRESS));
+ if (lvhpt_per_cpu_info[0].base == 0)
+ panic("Couldn't allocate VHPT on CPU %d, size: 0x%lx!\n",
+ cpu, size);
+ printk(KERN_INFO "Allocated long format VHPT for boot processor (CPU %d) at: 0x%lx, size: 0x%lx\n",
+ cpu, lvhpt_per_cpu_info[0].base, size);
+ }
+#else /* !CONFIG_IA64_LONG_FORMAT_VHPT */
+ unsigned long impl_va_bits;
+#endif
/* Pin mapping for percpu area into TLB */
psr = ia64_clear_ic();
ia64_itr(0x2, IA64_TR_PERCPU_DATA, PERCPU_ADDR,
pte_val(pfn_pte(__pa(my_cpu_data) >> PAGE_SHIFT, PAGE_KERNEL)),
PERCPU_PAGE_SHIFT);
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ /* Insert the permanent translation for the VHPT */
+ ia64_itr(0x2, IA64_TR_LONG_VHPT, LONG_VHPT_BASE,
+ pte_val(pfn_pte(__pa(lvhpt_per_cpu_info[cpu].base) >> PAGE_SHIFT, PAGE_KERNEL)), lvhpt_per_cpu_info[cpu].bits);
+#endif
ia64_set_psr(psr);
ia64_srlz_i();
-
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+# define VHPT_FORMAT_BIT 1
+# define vhpt_bits lvhpt_per_cpu_info[cpu].bits
+ pta = LONG_VHPT_BASE;
+#else
/*
+ * SHORT FORMAT VHPT (virtually mapped linear pagetable)
+ *
* Check if the virtually mapped linear page table (VMLPT) overlaps with a mapped
* address space. The IA-64 architecture guarantees that at least 50 bits of
* virtual address space are implemented but if we pick a large enough page size
@@ -367,6 +402,7 @@
* address space to not permit mappings that would overlap with the VMLPT.
* --davidm 00/12/06
*/
+# define VHPT_FORMAT_BIT 0
# define pte_bits 3
# define mapped_space_bits (3*(PAGE_SHIFT - pte_bits) + PAGE_SHIFT)
/*
@@ -376,28 +412,27 @@
* non-speculative accesses to the virtual page table, so the address range of the
* virtual page table itself needs to be covered by virtual page table.
*/
-# define vmlpt_bits (impl_va_bits - PAGE_SHIFT + pte_bits)
+# define vhpt_bits (impl_va_bits - PAGE_SHIFT + pte_bits)
# define POW2(n) (1ULL << (n))
-
impl_va_bits = ffz(~(local_cpu_data->unimpl_va_mask | (7UL << 61)));
if (impl_va_bits < 51 || impl_va_bits > 61)
panic("CPU has bogus IMPL_VA_MSB value of %lu!\n", impl_va_bits - 1);
/*
* mapped_space_bits - PAGE_SHIFT is the total number of ptes we need,
- * which must fit into "vmlpt_bits - pte_bits" slots. Second half of
+ * which must fit into "vhpt_bits - pte_bits" slots. Second half of
* the test makes sure that our mapped space doesn't overlap the
* unimplemented hole in the middle of the region.
*/
- if ((mapped_space_bits - PAGE_SHIFT > vmlpt_bits - pte_bits) ||
+ if ((mapped_space_bits - PAGE_SHIFT > vhpt_bits - pte_bits) ||
(mapped_space_bits > impl_va_bits - 1))
panic("Cannot build a big enough virtual-linear page table"
" to cover mapped address space.\n"
" Try using a smaller page size.\n");
-
/* place the VMLPT at the end of each page-table mapped region: */
- pta = POW2(61) - POW2(vmlpt_bits);
+ pta = POW2(61) - POW2(vhpt_bits);
+#endif
/*
* Set the (virtually mapped linear) page table address. Bit
@@ -405,10 +440,8 @@
* size of the table, and bit 0 whether the VHPT walker is
* enabled.
*/
- ia64_set_pta(pta | (0 << 8) | (vmlpt_bits << 2) | VHPT_ENABLE_BIT);
-
+ ia64_set_pta(pta | (VHPT_FORMAT_BIT << 8) | (vhpt_bits << 2) | VHPT_ENABLE_BIT);
ia64_tlb_init();
-
#ifdef CONFIG_HUGETLB_PAGE
ia64_set_rr(HPAGE_REGION_BASE, HPAGE_SHIFT << 2);
ia64_srlz_d();
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/pgtable.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/pgtable.h 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/pgtable.h 2006-05-10 10:08:46.000000000 +1000
@@ -156,6 +156,10 @@
#include <asm/mmu_context.h>
#include <asm/processor.h>
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+#include <asm/lvhpt.h>
+#endif
+
/*
* Next come the mappings that determine how mmap() protection bits
* (PROT_EXEC, PROT_READ, PROT_WRITE, PROT_NONE) get implemented. The
@@ -556,6 +560,7 @@
extern void memmap_init (unsigned long size, int nid, unsigned long zone,
unsigned long start_pfn);
# endif /* CONFIG_VIRTUAL_MEM_MAP */
+
# endif /* !__ASSEMBLY__ */
/*
@@ -576,6 +581,11 @@
#define KERNEL_TR_PAGE_SIZE (1 << KERNEL_TR_PAGE_SHIFT)
/*
+ * Long format VHPT
+ */
+#define LONG_VHPT_BASE (0xc000000000000000 - (1UL << lvhpt_per_cpu_info[smp_processor_id()].bits))
+
+/*
* No page table caches to initialise
*/
#define pgtable_cache_init() do { } while (0)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/Documentation/kernel-parameters.txt
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/Documentation/kernel-parameters.txt 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/Documentation/kernel-parameters.txt 2006-05-10 10:01:49.000000000 +1000
@@ -50,6 +50,7 @@
ISDN Appropriate ISDN support is enabled.
JOY Appropriate joystick support is enabled.
LP Printer support is enabled.
+ LONG_FORMAT_VHPT Long Format VHPT is enabled
LOOP Loopback device support is enabled.
M68k M68k architecture is enabled.
These options have more detailed description inside of
@@ -805,6 +806,19 @@
ltpc= [NET]
Format: <io>,<irq>,<dma>
+ lvhpt_bits_clamp= [IA64,LONG_FORMAT_VHPT]
+ Format: <1-39>
+
+ Clamp the size of the LVHPT (on all
+ nodes for a NUMA system) to 2^n bits.
+ E.g. 2^22 gives a LVHPT of 4MB. We
+ pin a TLB entry of this size, so the
+ size must be valid for the
+ architecture, otherwise your kernel
+ will not boot. By default we take a
+ good guess at sizing this for optimal
+ operation.
+
mac5380= [HW,SCSI] Format:
<can_queue>,<cmd_per_lun>,<sg_tablesize>,<hostid>,<use_tags>
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/Makefile
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/mm/Makefile 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/Makefile 2006-05-10 10:01:49.000000000 +1000
@@ -9,3 +9,4 @@
obj-$(CONFIG_DISCONTIGMEM) += discontig.o
obj-$(CONFIG_SPARSEMEM) += discontig.o
obj-$(CONFIG_FLATMEM) += contig.o
+obj-$(CONFIG_IA64_LONG_FORMAT_VHPT) += lvhpt.o
\ No newline at end of file
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/lvhpt.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/lvhpt.c 2006-05-10 13:05:53.000000000 +1000
@@ -0,0 +1,150 @@
+/* Long Format VHPT support functions */
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+
+#include <linux/acpi.h>
+#include <linux/bootmem.h>
+#include <linux/efi.h>
+
+#include <asm/tlb.h>
+#include <asm/lvhpt.h>
+
+/*
+ * This allows you to clamp the number of bits used for the long
+ * format vhpt. TODO check for invalid values here.
+ */
+static int lvhpt_bits_clamp;
+
+int __init
+lvhpt_bits_clamp_setup(char *s)
+{
+ if (sscanf(s, "%d", &lvhpt_bits_clamp) <= 0)
+ lvhpt_bits_clamp = 0;
+ return 1;
+}
+
+__setup("lvhpt_bits_clamp=", lvhpt_bits_clamp_setup);
+
+
+/* We try to size the LVHPT to cover the node local memory for the
+ * CPU. On initalisation we stash the location and size of each CPU's
+ * LVHPT table here. */
+struct lvhpt_per_cpu_info_struct lvhpt_per_cpu_info[NR_CPUS];
+
+/*
+ * This code must be called on a CPU which has it's MMU
+ * initialized. The page allocator seems to depend on it.
+ *
+ * Returns 0 on success.
+ */
+unsigned int
+alloc_vhpt(int cpu)
+{
+ int lvhpt_bits = lvhpt_per_cpu_info[cpu].bits;
+
+#ifdef CONFIG_NUMA
+ struct page *page;
+
+ page = alloc_pages_node(cpu_to_node(cpu), __GFP_HIGHMEM|GFP_ATOMIC, lvhpt_bits - PAGE_SHIFT);
+ if (!page)
+ return -1;
+ lvhpt_per_cpu_info[cpu].base = (unsigned long) page_address(page);
+#else
+ lvhpt_per_cpu_info[cpu].base = (unsigned long)__get_free_pages(__GFP_HIGHMEM|GFP_ATOMIC,
+ lvhpt_bits - PAGE_SHIFT);
+#endif
+ return (lvhpt_per_cpu_info[cpu].base == 0UL);
+}
+
+/*
+ * Passed to efi_memmap_walk to simply add up how much memory we have.
+ * This is used to size the LVHPT
+ */
+static int
+get_total_ram(unsigned long start, unsigned long end, void *arg)
+{
+ unsigned long *s = arg;
+ *s += (end - start);
+ return 0;
+}
+
+/* We use this for sizing the lvhpt */
+static unsigned long lvhpt_node_addressable_memory[MAX_NUMNODES];
+
+#ifdef CONFIG_NUMA
+static void compute_vhpt_size_numa(void)
+{
+ int i;
+
+ if (lvhpt_bits_clamp)
+ {
+ printk(KERN_INFO "Clamping LVHPT to %d bits\n", lvhpt_bits_clamp);
+ for (i = 0; i < NR_CPUS; i++)
+ lvhpt_per_cpu_info[i].bits = lvhpt_bits_clamp;
+ return;
+ }
+
+ /* In the NUMA case, we evaluate how much memory each node has
+ * and then try to size it to three times the physical memory
+ * of the node (as this gives us the best coverage. As we pin
+ * this with a TLB entry, we need to make sure the size we
+ * choose is however suitable for the architecture.
+ */
+ for (i = 0; i < num_node_memblks; i++) {
+ printk(KERN_ERR "vhpt_addr_mem[%d] = %lx\n", node_memblk[i].nid, node_memblk[i].size);
+ lvhpt_node_addressable_memory[node_memblk[i].nid] +=
+ node_memblk[i].size;
+ }
+
+ for (i = 0; i < NR_CPUS; i++)
+ {
+ unsigned long size =
+ 3 * (lvhpt_node_addressable_memory[cpu_to_node(i)] >> PAGE_SHIFT);
+ lvhpt_per_cpu_info[i].bits = find_largest_page_size(size);
+ }
+}
+#endif
+
+/* This version in both NUMA and non-NUMA, since we can use it in either. */
+static void compute_vhpt_size_non_numa(void)
+{
+ int i;
+ int bits;
+
+ if (lvhpt_bits_clamp)
+ {
+ printk(KERN_INFO "Clamping LVHPT to %d bits\n", lvhpt_bits_clamp);
+ for (i=0; i < NR_CPUS; i++)
+ lvhpt_per_cpu_info[i].bits = lvhpt_bits_clamp;
+ return;
+ }
+
+ /* If this doesn't work, try to find the total memory in the
+ * system, which we will then size the lvhpt table to cover
+ * three times over.
+ */
+ efi_memmap_walk(get_total_ram, &lvhpt_node_addressable_memory);
+
+ /* Put a lower clamp on this of a fairly reasonable 4 megabytes */
+ bits = max(22UL, find_largest_page_size(lvhpt_node_addressable_memory[0] >> PAGE_SHIFT));
+
+ for (i=0; i < NR_CPUS; i++)
+ lvhpt_per_cpu_info[i].bits = bits;
+}
+
+void __init
+compute_vhpt_size(void)
+{
+#ifdef CONFIG_NUMA
+ /* Machines like the ZX1 don't setup all the node info we
+ * require, but someone might still try to boot a NUMA kernel
+ * on it. In this case, fall back to our non-numa case.*/
+ if (num_node_memblks == 0)
+ compute_vhpt_size_non_numa();
+ else
+ compute_vhpt_size_numa();
+#else
+ compute_vhpt_size_non_numa();
+#endif
+}
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/lvhpt.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/lvhpt.h 2006-05-10 10:01:49.000000000 +1000
@@ -0,0 +1,24 @@
+/* Long Format VHPT includes */
+#ifndef _ASM_IA64_LVHPT_H
+#define _ASM_IA64_LVHPT_H
+
+/* We keep a per-cpu record of the VHPT base and size for each
+ * processor */
+struct lvhpt_per_cpu_info_struct {
+ unsigned long base;
+ int bits;
+};
+
+extern struct lvhpt_per_cpu_info_struct lvhpt_per_cpu_info[NR_CPUS];
+
+/* Initalise and compute the VHPT size for each CPU */
+void __init compute_vhpt_size(void);
+/* Allocate the VHPT for a CPU */
+unsigned int alloc_vhpt(int cpu);
+
+static inline unsigned long lvhpt_size(int cpu)
+{
+ return (1UL << lvhpt_per_cpu_info[cpu].bits);
+}
+
+#endif /* _ASM_IA64_LVHPT_H */
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/kregs.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/kregs.h 2006-05-10 10:00:50.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/kregs.h 2006-05-10 10:01:49.000000000 +1000
@@ -31,6 +31,7 @@
#define IA64_TR_PALCODE 1 /* itr1: maps PALcode as required by EFI */
#define IA64_TR_PERCPU_DATA 1 /* dtr1: percpu data */
#define IA64_TR_CURRENT_STACK 2 /* dtr2: maps kernel's memory- & register-stacks */
+#define IA64_TR_LONG_VHPT 3 /* dtr3: maps long format VHPT */
/* Processor status register bits: */
#define IA64_PSR_BE_BIT 1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RFC 6/6] LVHPT - long format support functions
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
` (4 preceding siblings ...)
2006-05-10 3:42 ` [RFC 5/6] LVHPT - setup infrastructure Ian Wienand
@ 2006-05-10 3:42 ` Ian Wienand
5 siblings, 0 replies; 7+ messages in thread
From: Ian Wienand @ 2006-05-10 3:42 UTC (permalink / raw)
To: linux-ia64; +Cc: linux-mm, Ian Wienand
LVHPT support
Functions for dealing with LVHPT flushing and miscellaneous other
control functions.
Signed-Off-By: Ian Wienand <ianw@gelato.unsw.edu.au>
---
arch/ia64/mm/tlb.c | 46 +++++++++++++++++++++++++++++++++++++++++
include/asm-ia64/mmu_context.h | 21 +++++++++++++++++-
include/asm-ia64/page.h | 5 ++++
include/asm-ia64/tlb.h | 2 +
include/asm-ia64/tlbflush.h | 24 +++++++++++++++++++++
5 files changed, 97 insertions(+), 1 deletion(-)
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/tlb.c
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/arch/ia64/mm/tlb.c 2006-05-10 10:31:53.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/arch/ia64/mm/tlb.c 2006-05-10 10:38:33.000000000 +1000
@@ -114,6 +114,17 @@
{
unsigned long i, j, flags, count0, count1, stride0, stride1, addr;
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ unsigned long page;
+
+ /* Admittedly 0 is a valid tag, but in that rare case the
+ * present bit will save us */
+ for (page = LONG_VHPT_BASE;
+ page < LONG_VHPT_BASE + lvhpt_size(smp_processor_id());
+ page += PAGE_SIZE)
+ clear_page((void *)page);
+#endif
+
addr = local_cpu_data->ptce_base;
count0 = local_cpu_data->ptce_count[0];
count1 = local_cpu_data->ptce_count[1];
@@ -132,6 +143,37 @@
ia64_srlz_i(); /* srlz.i implies srlz.d */
}
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+static void
+flush_vhpt_range (struct mm_struct *mm, unsigned long from, unsigned long to)
+{
+ unsigned long addr;
+
+ for (addr = from; addr < to; addr += PAGE_SIZE)
+ flush_vhpt_page(addr);
+
+#ifdef CONFIG_SMP
+ {
+ /* Urgh... flush VHPTs of any other CPUs that have run this mm */
+ unsigned long offset;
+ long_pte_t *hpte;
+ int cpu;
+
+ for_each_cpu_mask(cpu, mm->cpu_vm_mask)
+ {
+ for (addr = from; addr < to; addr += PAGE_SIZE)
+ {
+ offset = ia64_thash(addr) & (lvhpt_size(cpu)-1);
+ hpte = (long_pte_t *)(lvhpt_per_cpu_info[cpu].base + offset);
+ hpte->tag = INVALID_TAG;
+ }
+ }
+ }
+#endif
+ }
+#endif /* CONFIG_IA64_LONG_FORMAT_VHPT */
+
+
void
flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
unsigned long end)
@@ -146,6 +188,10 @@
}
#endif
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ flush_vhpt_range(mm, start, end);
+#endif
+
nbits = find_largest_page_size(end-start);
start &= ~((1UL << nbits) - 1);
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/mmu_context.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/mmu_context.h 2006-05-10 10:31:53.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/mmu_context.h 2006-05-10 10:38:33.000000000 +1000
@@ -18,7 +18,21 @@
#define IA64_REGION_ID_KERNEL 0 /* the kernel's region id (tlb.c depends on this being 0) */
-#define ia64_rid(ctx,addr) (((ctx) << 3) | (addr >> 61))
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+/*
+ * Due to a high number of collisions in the long format VHPT walker hash function
+ * when RIDs and similar address space layout occur "eg. fork()". The following is
+ * used to space out the RIDs we present to the hardware without messing with Linux's
+ * sequential allocation scheme.
+ * Refer to 'Intel Itanium Processor Reference Manual for Software Development'
+ * http://www.intel.com/design/itanium/manuals.htm
+ */
+#define redistribute_rid(rid) (((rid) & ~0xffff) | (((rid) << 8) & 0xff00) | (((rid) >> 8) & 0xff))
+#else
+#define redistribute_rid(rid) (rid)
+#endif
+
+#define ia64_rid(ctx,addr) redistribute_rid(((ctx) << 3) | (addr >> 61))
# include <asm/page.h>
# ifndef __ASSEMBLY__
@@ -135,7 +149,12 @@
old_rr4 = ia64_get_rr(RGN_BASE(RGN_HPAGE));
rid = context << 3; /* make space for encoding the region number */
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+ rid = redistribute_rid(rid);
+ rid_incr = 1 << 16;
+#else
rid_incr = 1 << 8;
+#endif
/* encode the region id, preferred page size, and VHPT enable bit: */
rr0 = (rid << 8) | (PAGE_SHIFT << 2) | 1;
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/page.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/page.h 2006-05-10 10:31:53.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/page.h 2006-05-10 10:38:33.000000000 +1000
@@ -175,6 +175,11 @@
return order;
}
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+/* Long format VHPT entry */
+typedef struct { unsigned long pte, itir, tag, ig; } long_pte_t;
+#endif
+
# endif /* __KERNEL__ */
#endif /* !__ASSEMBLY__ */
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/tlbflush.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/tlbflush.h 2006-05-10 10:31:53.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/tlbflush.h 2006-05-10 10:38:33.000000000 +1000
@@ -19,6 +19,21 @@
* can be very expensive, so try to avoid them whenever possible.
*/
+/* Flushing a translation from the long format VHPT */
+#ifdef CONFIG_IA64_LONG_FORMAT_VHPT
+# define INVALID_TAG (1UL << 63)
+
+static inline void
+flush_vhpt_page(unsigned long addr)
+{
+ long_pte_t *hpte;
+ hpte = (long_pte_t *)ia64_thash(addr);
+ hpte->tag = INVALID_TAG;
+}
+#else
+# define flush_vhpt_page(addr) do { } while (0)
+#endif
+
/*
* Flush everything (kernel mapping may also have changed due to
* vmalloc/vfree).
@@ -54,6 +69,12 @@
set_bit(mm->context, ia64_ctx.flushmap);
mm->context = 0;
+ /* XXX smp_flush_tlb_mm actually enables and disables preempt
+ * ... maybe we should refactor all this
+ */
+ cpu_clear(get_cpu(), mm->cpu_vm_mask);
+ put_cpu();
+
if (atomic_read(&mm->mm_users) == 0)
return; /* happens as a result of exit_mmap() */
@@ -76,7 +97,10 @@
flush_tlb_range(vma, (addr & PAGE_MASK), (addr & PAGE_MASK) + PAGE_SIZE);
#else
if (vma->vm_mm == current->active_mm)
+ {
+ flush_vhpt_page(addr);
ia64_ptcl(addr, (PAGE_SHIFT << 2));
+ }
else
vma->vm_mm->context = 0;
#endif
Index: linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/tlb.h
===================================================================
--- linux-2.6.17-rc3-lvhpt-v2-fresh.orig/include/asm-ia64/tlb.h 2006-05-10 10:31:53.000000000 +1000
+++ linux-2.6.17-rc3-lvhpt-v2-fresh/include/asm-ia64/tlb.h 2006-05-10 10:38:33.000000000 +1000
@@ -107,8 +107,10 @@
vma.vm_mm = tlb->mm;
/* flush the address range from the tlb: */
flush_tlb_range(&vma, start, end);
+#ifndef CONFIG_IA64_LONG_FORMAT_VHPT
/* now flush the virt. page-table area mapping the address range: */
flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
+#endif
}
/* lastly, release the freed pages */
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2006-05-10 3:42 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2006-05-10 3:42 [RFC 0/6] IA64 Long Format VHPT support v2 Ian Wienand
2006-05-10 3:42 ` [RFC 1/6] LVHPT - Remove static ivt.S Ian Wienand
2006-05-10 3:42 ` [RFC 2/6] LVHPT - Create ivt.S dyanamically Ian Wienand
2006-05-10 3:42 ` [RFC 3/6] LVHPT - ivt.S long format support Ian Wienand
2006-05-10 3:42 ` [RFC 4/6] LVHPT - find architectured page size help Ian Wienand
2006-05-10 3:42 ` [RFC 5/6] LVHPT - setup infrastructure Ian Wienand
2006-05-10 3:42 ` [RFC 6/6] LVHPT - long format support functions Ian Wienand
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox