arch/powerpc/lib/copypage_64.S

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * Copyright (C) 2008 Mark Nelson, IBM Corp.
   4  */
   5 #include <linux/export.h>
   6 #include <asm/page.h>
   7 #include <asm/processor.h>
   8 #include <asm/ppc_asm.h>
   9 #include <asm/asm-offsets.h>
  10 #include <asm/feature-fixups.h>
  11
  12 _GLOBAL_TOC(copy_page)
  13 BEGIN_FTR_SECTION
  14         lis     r5,PAGE_SIZE@h
  15 FTR_SECTION_ELSE
  16 #ifdef CONFIG_PPC_BOOK3S_64
  17         b       copypage_power7
  18 #endif
  19 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
  20         ori     r5,r5,PAGE_SIZE@l
  21 #ifdef CONFIG_PPC_KERNEL_PCREL
  22         /*
  23          * Hack for toolchain - prefixed instructions cause label difference to
  24          * be non-constant even if 8 byte alignment is known, so they can not
  25          * be put in FTR sections.
  26          */
  27         LOAD_REG_ADDR(r10, ppc64_caches)
  28 BEGIN_FTR_SECTION
  29 #else
  30 BEGIN_FTR_SECTION
  31         LOAD_REG_ADDR(r10, ppc64_caches)
  32 #endif
  33         lwz     r11,DCACHEL1LOGBLOCKSIZE(r10)   /* log2 of cache block size */
  34         lwz     r12,DCACHEL1BLOCKSIZE(r10)      /* get cache block size */
  35         li      r9,0
  36         srd     r8,r5,r11
  37
  38         mtctr   r8
  39 .Lsetup:
  40         dcbt    r9,r4
  41         dcbz    r9,r3
  42         add     r9,r9,r12
  43         bdnz    .Lsetup
  44 END_FTR_SECTION_IFSET(CPU_FTR_CP_USE_DCBTZ)
  45         addi    r3,r3,-8
  46         srdi    r8,r5,7         /* page is copied in 128 byte strides */
  47         addi    r8,r8,-1        /* one stride copied outside loop */
  48
  49         mtctr   r8
  50
  51         ld      r5,0(r4)
  52         ld      r6,8(r4)
  53         ld      r7,16(r4)
  54         ldu     r8,24(r4)
  55 1:      std     r5,8(r3)
  56         std     r6,16(r3)
  57         ld      r9,8(r4)
  58         ld      r10,16(r4)
  59         std     r7,24(r3)
  60         std     r8,32(r3)
  61         ld      r11,24(r4)
  62         ld      r12,32(r4)
  63         std     r9,40(r3)
  64         std     r10,48(r3)
  65         ld      r5,40(r4)
  66         ld      r6,48(r4)
  67         std     r11,56(r3)
  68         std     r12,64(r3)
  69         ld      r7,56(r4)
  70         ld      r8,64(r4)
  71         std     r5,72(r3)
  72         std     r6,80(r3)
  73         ld      r9,72(r4)
  74         ld      r10,80(r4)
  75         std     r7,88(r3)
  76         std     r8,96(r3)
  77         ld      r11,88(r4)
  78         ld      r12,96(r4)
  79         std     r9,104(r3)
  80         std     r10,112(r3)
  81         ld      r5,104(r4)
  82         ld      r6,112(r4)
  83         std     r11,120(r3)
  84         stdu    r12,128(r3)
  85         ld      r7,120(r4)
  86         ldu     r8,128(r4)
  87         bdnz    1b
  88
  89         std     r5,8(r3)
  90         std     r6,16(r3)
  91         ld      r9,8(r4)
  92         ld      r10,16(r4)
  93         std     r7,24(r3)
  94         std     r8,32(r3)
  95         ld      r11,24(r4)
  96         ld      r12,32(r4)
  97         std     r9,40(r3)
  98         std     r10,48(r3)
  99         ld      r5,40(r4)
 100         ld      r6,48(r4)
 101         std     r11,56(r3)
 102         std     r12,64(r3)
 103         ld      r7,56(r4)
 104         ld      r8,64(r4)
 105         std     r5,72(r3)
 106         std     r6,80(r3)
 107         ld      r9,72(r4)
 108         ld      r10,80(r4)
 109         std     r7,88(r3)
 110         std     r8,96(r3)
 111         ld      r11,88(r4)
 112         ld      r12,96(r4)
 113         std     r9,104(r3)
 114         std     r10,112(r3)
 115         std     r11,120(r3)
 116         std     r12,128(r3)
 117         blr
 118 EXPORT_SYMBOL(copy_page)