LPAE

arch/arm/include/asm/pgtable-3level.h

/*
 * With LPAE, there are 3 levels of page tables. Each level has 512 entries of
 * 8 bytes each, occupying a 4K page. The first level table covers a range of
 * 512GB, each entry representing 1GB. Since we are limited to 4GB input
 * address range, only 4 entries in the PGD are used.
 *
 * There are enough spare bits in a page table entry for the kernel specific
 * state.
 */
 #define PTRS_PER_PTE            512
 #define PTRS_PER_PMD            512
 #define PTRS_PER_PGD            4
 
 #define PTE_HWTABLE_PTRS        (0)
 #define PTE_HWTABLE_OFF         (0)
 #define PTE_HWTABLE_SIZE        (PTRS_PER_PTE * sizeof(u64))
 /*
  * PGDIR_SHIFT determines the size a top-level page table entry can map.
  */
 #define PGDIR_SHIFT             30
 
 /*
  * PMD_SHIFT determines the size a middle-level page table entry can map.
  */
 #define PMD_SHIFT               21
 
 #define PMD_SIZE                (1UL << PMD_SHIFT)
 #define PMD_MASK                (~((1 << PMD_SHIFT) - 1))
 #define PGDIR_SIZE              (1UL << PGDIR_SHIFT)
 #define PGDIR_MASK              (~((1 << PGDIR_SHIFT) - 1))
 
 /*
  * section address mask and size definitions.
  */
 #define SECTION_SHIFT           21
 #define SECTION_SIZE            (1UL << SECTION_SHIFT)
 #define SECTION_MASK            (~((1 << SECTION_SHIFT) - 1))
 
 #define USER_PTRS_PER_PGD       (PAGE_OFFSET / PGDIR_SIZE)
 
 /*
  * Hugetlb definitions.
  */
 #define HPAGE_SHIFT             PMD_SHIFT
 #define HPAGE_SIZE              (_AC(1, UL) << HPAGE_SHIFT)
 #define HPAGE_MASK              (~(HPAGE_SIZE - 1))
 #define HUGETLB_PAGE_ORDER      (HPAGE_SHIFT - PAGE_SHIFT)

arch/arm/include/asm/arch_gicv3.h

 /*
  * Even in 32bit systems that use LPAE, there is no guarantee that the I/O
  * interface provides true 64bit atomic accesses, so using strd/ldrd doesn't
  * make much sense.
  * Moreover, 64bit I/O emulation is extremely difficult to implement on
  * AArch32, since the syndrome register doesn't provide any information for
  * them.
  * Consequently, the following IO helpers use 32bit accesses.
  */
 static inline void __gic_writeq_nonatomic(u64 val, volatile void __iomem *addr)
 {
         writel_relaxed((u32)val, addr);
         writel_relaxed((u32)(val >> 32), addr + 4);
 }
 
 static inline u64 __gic_readq_nonatomic(const volatile void __iomem *addr)
 {
         u64 val;
 
         val = readl_relaxed(addr);
         val |= (u64)readl_relaxed(addr + 4) << 32;
         return val;
 }

arch/arm/kernel/setup.c

 static void __init cpuid_init_hwcaps(void)
 {
         int block;
         u32 isar5;
 
         if (cpu_architecture() < CPU_ARCH_ARMv7)
                 return;
 
         block = cpuid_feature_extract(CPUID_EXT_ISAR0, 24);
         if (block >= 2)
                 elf_hwcap |= HWCAP_IDIVA;
         if (block >= 1)
                 elf_hwcap |= HWCAP_IDIVT;
 
         /* LPAE implies atomic ldrd/strd instructions */
         block = cpuid_feature_extract(CPUID_EXT_MMFR0, 0);
         if (block >= 5)
                 elf_hwcap |= HWCAP_LPAE;
 
         /* check for supported v8 Crypto instructions */
         isar5 = read_cpuid_ext(CPUID_EXT_ISAR5);
 
         block = cpuid_feature_extract_field(isar5, 4);
         if (block >= 2)
                 elf_hwcap2 |= HWCAP2_PMULL;
         if (block >= 1)
                 elf_hwcap2 |= HWCAP2_AES;
 
         block = cpuid_feature_extract_field(isar5, 8);
         if (block >= 1)
                 elf_hwcap2 |= HWCAP2_SHA1;
 
         block = cpuid_feature_extract_field(isar5, 12);
         if (block >= 1)
                 elf_hwcap2 |= HWCAP2_SHA2;
 
         block = cpuid_feature_extract_field(isar5, 16);
         if (block >= 1)
                 elf_hwcap2 |= HWCAP2_CRC32;
 }

arch/arm/kernel/head.S

 /*
  * swapper_pg_dir is the virtual address of the initial page table.
  * We place the page tables 16K below KERNEL_RAM_VADDR.  Therefore, we must
  * make sure that KERNEL_RAM_VADDR is correctly set.  Currently, we expect
  * the least significant 16 bits to be 0x8000, but we could probably
  * relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000.
  */
 #define KERNEL_RAM_VADDR        (PAGE_OFFSET + TEXT_OFFSET)
 #if (KERNEL_RAM_VADDR & 0xffff) != 0x8000
 #error KERNEL_RAM_VADDR must start at 0xXXXX8000
 #endif
 
 #ifdef CONFIG_ARM_LPAE
         /* LPAE requires an additional page for the PGD */
 #define PG_DIR_SIZE     0x5000
 #define PMD_ORDER       3
 #else
 #define PG_DIR_SIZE     0x4000
 #define PMD_ORDER       2
 #endif
    
         .globl  swapper_pg_dir
         .equ    swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE
 
         .macro  pgtbl, rd, phys
         add     \rd, \phys, #TEXT_OFFSET
         sub     \rd, \rd, #PG_DIR_SIZE
         .endm
 
 #ifdef CONFIG_ARM_LPAE
         mrc     p15, 0, r3, c0, c1, 4           @ read ID_MMFR0
         and     r3, r3, #0xf                    @ extract VMSA support
         cmp     r3, #5                          @ long-descriptor translation table format?
  THUMB( it      lo )                            @ force fixup-able long branch encoding
         blo     __error_lpae                    @ only classic page table format
 #endif
 
         bl      __create_page_tables
 
         /*
          * The following calls CPU specific code in a position independent
          * manner.  See arch/arm/mm/proc-*.S for details.  r10 = base of
          * xxx_proc_info structure selected by __lookup_processor_type
          * above.
          *
          * The processor init function will be called with:
          *  r1 - machine type
          *  r2 - boot data (atags/dt) pointer
          *  r4 - translation table base (low word)
          *  r5 - translation table base (high word, if LPAE)
          *  r8 - translation table base 1 (pfn if LPAE)
          *  r9 - cpuid
          *  r13 - virtual address for __enable_mmu -> __turn_mmu_on
          *
          * On return, the CPU will be ready for the MMU to be turned on,
          * r0 will hold the CPU control register value, r1, r2, r4, and
          * r9 will be preserved.  r5 will also be preserved if LPAE.
          */
         ldr     r13, =__mmap_switched           @ address to jump to after
                                                 @ mmu has been enabled
         badr    lr, 1f                          @ return (PIC) address
 #ifdef CONFIG_ARM_LPAE
         mov     r5, #0                          @ high TTBR0
         mov     r8, r4, lsr #12                 @ TTBR1 is swapper_pg_dir pfn
 #else
         mov     r8, r4                          @ set TTBR1 to swapper_pg_dir
 #endif
         ldr     r12, [r10, #PROCINFO_INITFUNC]
         add     r12, r12, r10
         ret     r12
 1:      b       __enable_mmu
 ENDPROC(stext)
         .ltorg
 #ifndef CONFIG_XIP_KERNEL
 2:      .long   .
         .long   PAGE_OFFSET
 #endif
 
 /*
  * Setup the initial page tables.  We only setup the barest
  * amount which are required to get the kernel running, which
  * generally means mapping in the kernel code.
  *
  * r8 = phys_offset, r9 = cpuid, r10 = procinfo
  *
  * Returns:
  *  r0, r3, r5-r7 corrupted
  *  r4 = physical page table address
  */
 __create_page_tables:
         pgtbl   r4, r8                          @ page table address
 
         /*
          * Clear the swapper page table
          */
         mov     r0, r4
         mov     r3, #0
         add     r6, r0, #PG_DIR_SIZE
 1:      str     r3, [r0], #4
         str     r3, [r0], #4
         str     r3, [r0], #4
         str     r3, [r0], #4
         teq     r0, r6
         bne     1b
 
 #ifdef CONFIG_ARM_LPAE
         /*
          * Build the PGD table (first level) to point to the PMD table. A PGD
          * entry is 64-bit wide.
          */
         mov     r0, r4
         add     r3, r4, #0x1000                 @ first PMD table address
         orr     r3, r3, #3                      @ PGD block type
         mov     r6, #4                          @ PTRS_PER_PGD
         mov     r7, #1 << (55 - 32)             @ L_PGD_SWAPPER
 1:
 #ifdef CONFIG_CPU_ENDIAN_BE8
         str     r7, [r0], #4                    @ set top PGD entry bits
         str     r3, [r0], #4                    @ set bottom PGD entry bits
 #else
         str     r3, [r0], #4                    @ set bottom PGD entry bits
         str     r7, [r0], #4                    @ set top PGD entry bits
 #endif
         add     r3, r3, #0x1000                 @ next PMD table
         subs    r6, r6, #1
         bne     1b
 
         add     r4, r4, #0x1000                 @ point to the PMD tables
 #ifdef CONFIG_CPU_ENDIAN_BE8
         add     r4, r4, #4                      @ we only write the bottom word
 #endif
 #endif
 /*
  * Setup common bits before finally enabling the MMU.  Essentially
  * this is just loading the page table pointer and domain access
  * registers.  All these registers need to be preserved by the
  * processor setup function (or set in the case of r0)
  *
  *  r0  = cp#15 control register
  *  r1  = machine ID
  *  r2  = atags or dtb pointer
  *  r4  = TTBR pointer (low word)
  *  r5  = TTBR pointer (high word if LPAE)
  *  r9  = processor ID
  *  r13 = *virtual* address to jump to upon completion
  */
 __enable_mmu:
 #if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6
         orr     r0, r0, #CR_A
 #else
         bic     r0, r0, #CR_A
 #endif
 #ifdef CONFIG_CPU_DCACHE_DISABLE
         bic     r0, r0, #CR_C
 #endif
 #ifdef CONFIG_CPU_BPREDICT_DISABLE
         bic     r0, r0, #CR_Z
 #endif
 #ifdef CONFIG_CPU_ICACHE_DISABLE
         bic     r0, r0, #CR_I
 #endif
 #ifdef CONFIG_ARM_LPAE
         mcrr    p15, 0, r4, r5, c2              @ load TTBR0
 #else
         mov     r5, #DACR_INIT
         mcr     p15, 0, r5, c3, c0, 0           @ load domain access register
         mcr     p15, 0, r4, c2, c0, 0           @ load page table pointer
 #endif
         b       __turn_mmu_on
 ENDPROC(__enable_mmu)

arch/arm/mm/proc-v7.S

/*
 *      __v7_setup
 *
 *      Initialise TLB, Caches, and MMU state ready to switch the MMU
 *      on.  Return in r0 the new CP15 C1 control register setting.
 *
 *      r1, r2, r4, r5, r9, r13 must be preserved - r13 is not a stack
 *      r4: TTBR0 (low word)
 *      r5: TTBR0 (high word if LPAE)
 *      r8: TTBR1
 *      r9: Main ID register
 *
 *      This should be able to cover all ARMv7 cores.
 *
 *      It is assumed that:
 *      - cache type register is implemented
 */

arch/arm/mm/mmu.c

#ifdef CONFIG_ARM_LPAE
/* the first page is reserved for pgd */
#define SWAPPER_PG_DIR_SIZE     (PAGE_SIZE + \
                                 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
#else
#define SWAPPER_PG_DIR_SIZE     (PTRS_PER_PGD * sizeof(pgd_t))
#endif

/*
 * early_paging_init() recreates boot time page table setup, allowing machines
 * to switch over to a high (>4G) address space on LPAE systems
 */
static void __init early_paging_init(const struct machine_desc *mdesc)
{
	...
｝