Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _ASM_X86_PAGE_64_H
       3             : #define _ASM_X86_PAGE_64_H
       4             : 
       5             : #include <asm/page_64_types.h>
       6             : 
       7             : #ifndef __ASSEMBLY__
       8             : #include <asm/cpufeatures.h>
       9             : #include <asm/alternative.h>
      10             : 
      11             : #include <linux/kmsan-checks.h>
      12             : 
      13             : /* duplicated to the one in bootmem.h */
      14             : extern unsigned long max_pfn;
      15             : extern unsigned long phys_base;
      16             : 
      17             : extern unsigned long page_offset_base;
      18             : extern unsigned long vmalloc_base;
      19             : extern unsigned long vmemmap_base;
      20             : 
      21             : static __always_inline unsigned long __phys_addr_nodebug(unsigned long x)
      22             : {
      23   275859712 :         unsigned long y = x - __START_KERNEL_map;
      24             : 
      25             :         /* use the carry flag to determine if x was < __START_KERNEL_map */
      26   275859712 :         x = y + ((x > y) ? phys_base : (__START_KERNEL_map - PAGE_OFFSET));
      27             : 
      28   275859712 :         return x;
      29             : }
      30             : 
      31             : #ifdef CONFIG_DEBUG_VIRTUAL
      32             : extern unsigned long __phys_addr(unsigned long);
      33             : extern unsigned long __phys_addr_symbol(unsigned long);
      34             : #else
      35             : #define __phys_addr(x)          __phys_addr_nodebug(x)
      36             : #define __phys_addr_symbol(x) \
      37             :         ((unsigned long)(x) - __START_KERNEL_map + phys_base)
      38             : #endif
      39             : 
      40             : #define __phys_reloc_hide(x)    (x)
      41             : 
      42             : void clear_page_orig(void *page);
      43             : void clear_page_rep(void *page);
      44             : void clear_page_erms(void *page);
      45             : 
      46             : static inline void clear_page(void *page)
      47             : {
      48             :         /*
      49             :          * Clean up KMSAN metadata for the page being cleared. The assembly call
      50             :          * below clobbers @page, so we perform unpoisoning before it.
      51             :          */
      52         704 :         kmsan_unpoison_memory(page, PAGE_SIZE);
      53         704 :         alternative_call_2(clear_page_orig,
      54             :                            clear_page_rep, X86_FEATURE_REP_GOOD,
      55             :                            clear_page_erms, X86_FEATURE_ERMS,
      56             :                            "=D" (page),
      57             :                            "0" (page)
      58             :                            : "cc", "memory", "rax", "rcx");
      59         704 : }
      60             : 
      61             : void copy_page(void *to, void *from);
      62             : 
      63             : #ifdef CONFIG_X86_5LEVEL
      64             : /*
      65             :  * User space process size.  This is the first address outside the user range.
      66             :  * There are a few constraints that determine this:
      67             :  *
      68             :  * On Intel CPUs, if a SYSCALL instruction is at the highest canonical
      69             :  * address, then that syscall will enter the kernel with a
      70             :  * non-canonical return address, and SYSRET will explode dangerously.
      71             :  * We avoid this particular problem by preventing anything
      72             :  * from being mapped at the maximum canonical address.
      73             :  *
      74             :  * On AMD CPUs in the Ryzen family, there's a nasty bug in which the
      75             :  * CPUs malfunction if they execute code from the highest canonical page.
      76             :  * They'll speculate right off the end of the canonical space, and
      77             :  * bad things happen.  This is worked around in the same way as the
      78             :  * Intel problem.
      79             :  *
      80             :  * With page table isolation enabled, we map the LDT in ... [stay tuned]
      81             :  */
      82             : static __always_inline unsigned long task_size_max(void)
      83             : {
      84             :         unsigned long ret;
      85             : 
      86             :         alternative_io("movq %[small],%0","movq %[large],%0",
      87             :                         X86_FEATURE_LA57,
      88             :                         "=r" (ret),
      89             :                         [small] "i" ((1ul << 47)-PAGE_SIZE),
      90             :                         [large] "i" ((1ul << 56)-PAGE_SIZE));
      91             : 
      92             :         return ret;
      93             : }
      94             : #endif  /* CONFIG_X86_5LEVEL */
      95             : 
      96             : #endif  /* !__ASSEMBLY__ */
      97             : 
      98             : #ifdef CONFIG_X86_VSYSCALL_EMULATION
      99             : # define __HAVE_ARCH_GATE_AREA 1
     100             : #endif
     101             : 
     102             : #endif /* _ASM_X86_PAGE_64_H */