arch/powerpc/net/bpf_jit_64.S

   1 /* bpf_jit.S: Packet/header access helper functions
   2  * for PPC64 BPF compiler.
   3  *
   4  * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; version 2
   9  * of the License.
  10  */
  11
  12 #include <asm/ppc_asm.h>
  13 #include "bpf_jit.h"
  14
  15 /*
  16  * All of these routines are called directly from generated code,
  17  * whose register usage is:
  18  *
  19  * r3           skb
  20  * r4,r5        A,X
  21  * r6           *** address parameter to helper ***
  22  * r7-r10       scratch
  23  * r14          skb->data
  24  * r15          skb headlen
  25  * r16-31       M[]
  26  */
  27
  28 /*
  29  * To consider: These helpers are so small it could be better to just
  30  * generate them inline.  Inline code can do the simple headlen check
  31  * then branch directly to slow_path_XXX if required.  (In fact, could
  32  * load a spare GPR with the address of slow_path_generic and pass size
  33  * as an argument, making the call site a mtlr, li and bllr.)
  34  */
  35         .globl  sk_load_word
  36 sk_load_word:
  37         cmpdi   r_addr, 0
  38         blt     bpf_slow_path_word_neg
  39         .globl  sk_load_word_positive_offset
  40 sk_load_word_positive_offset:
  41         /* Are we accessing past headlen? */
  42         subi    r_scratch1, r_HL, 4
  43         cmpd    r_scratch1, r_addr
  44         blt     bpf_slow_path_word
  45         /* Nope, just hitting the header.  cr0 here is eq or gt! */
  46 #ifdef __LITTLE_ENDIAN__
  47         lwbrx   r_A, r_D, r_addr
  48 #else
  49         lwzx    r_A, r_D, r_addr
  50 #endif
  51         blr     /* Return success, cr0 != LT */
  52
  53         .globl  sk_load_half
  54 sk_load_half:
  55         cmpdi   r_addr, 0
  56         blt     bpf_slow_path_half_neg
  57         .globl  sk_load_half_positive_offset
  58 sk_load_half_positive_offset:
  59         subi    r_scratch1, r_HL, 2
  60         cmpd    r_scratch1, r_addr
  61         blt     bpf_slow_path_half
  62 #ifdef __LITTLE_ENDIAN__
  63         lhbrx   r_A, r_D, r_addr
  64 #else
  65         lhzx    r_A, r_D, r_addr
  66 #endif
  67         blr
  68
  69         .globl  sk_load_byte
  70 sk_load_byte:
  71         cmpdi   r_addr, 0
  72         blt     bpf_slow_path_byte_neg
  73         .globl  sk_load_byte_positive_offset
  74 sk_load_byte_positive_offset:
  75         cmpd    r_HL, r_addr
  76         ble     bpf_slow_path_byte
  77         lbzx    r_A, r_D, r_addr
  78         blr
  79
  80 /*
  81  * BPF_S_LDX_B_MSH: ldxb  4*([offset]&0xf)
  82  * r_addr is the offset value
  83  */
  84         .globl sk_load_byte_msh
  85 sk_load_byte_msh:
  86         cmpdi   r_addr, 0
  87         blt     bpf_slow_path_byte_msh_neg
  88         .globl sk_load_byte_msh_positive_offset
  89 sk_load_byte_msh_positive_offset:
  90         cmpd    r_HL, r_addr
  91         ble     bpf_slow_path_byte_msh
  92         lbzx    r_X, r_D, r_addr
  93         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
  94         blr
  95
  96 /* Call out to skb_copy_bits:
  97  * We'll need to back up our volatile regs first; we have
  98  * local variable space at r1+(BPF_PPC_STACK_BASIC).
  99  * Allocate a new stack frame here to remain ABI-compliant in
 100  * stashing LR.
 101  */
 102 #define bpf_slow_path_common(SIZE)                              \
 103         mflr    r0;                                             \
 104         std     r0, 16(r1);                                     \
 105         /* R3 goes in parameter space of caller's frame */      \
 106         std     r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 107         std     r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 108         std     r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 109         addi    r5, r1, BPF_PPC_STACK_BASIC+(2*8);              \
 110         stdu    r1, -BPF_PPC_SLOWPATH_FRAME(r1);                \
 111         /* R3 = r_skb, as passed */                             \
 112         mr      r4, r_addr;                                     \
 113         li      r6, SIZE;                                       \
 114         bl      skb_copy_bits;                                  \
 115         nop;                                                    \
 116         /* R3 = 0 on success */                                 \
 117         addi    r1, r1, BPF_PPC_SLOWPATH_FRAME;                 \
 118         ld      r0, 16(r1);                                     \
 119         ld      r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 120         ld      r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 121         mtlr    r0;                                             \
 122         cmpdi   r3, 0;                                          \
 123         blt     bpf_error;      /* cr0 = LT */                  \
 124         ld      r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 125         /* Great success! */
 126
 127 bpf_slow_path_word:
 128         bpf_slow_path_common(4)
 129         /* Data value is on stack, and cr0 != LT */
 130         lwz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 131         blr
 132
 133 bpf_slow_path_half:
 134         bpf_slow_path_common(2)
 135         lhz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 136         blr
 137
 138 bpf_slow_path_byte:
 139         bpf_slow_path_common(1)
 140         lbz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 141         blr
 142
 143 bpf_slow_path_byte_msh:
 144         bpf_slow_path_common(1)
 145         lbz     r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
 146         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
 147         blr
 148
 149 /* Call out to bpf_internal_load_pointer_neg_helper:
 150  * We'll need to back up our volatile regs first; we have
 151  * local variable space at r1+(BPF_PPC_STACK_BASIC).
 152  * Allocate a new stack frame here to remain ABI-compliant in
 153  * stashing LR.
 154  */
 155 #define sk_negative_common(SIZE)                                \
 156         mflr    r0;                                             \
 157         std     r0, 16(r1);                                     \
 158         /* R3 goes in parameter space of caller's frame */      \
 159         std     r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 160         std     r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 161         std     r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 162         stdu    r1, -BPF_PPC_SLOWPATH_FRAME(r1);                \
 163         /* R3 = r_skb, as passed */                             \
 164         mr      r4, r_addr;                                     \
 165         li      r5, SIZE;                                       \
 166         bl      bpf_internal_load_pointer_neg_helper;           \
 167         nop;                                                    \
 168         /* R3 != 0 on success */                                \
 169         addi    r1, r1, BPF_PPC_SLOWPATH_FRAME;                 \
 170         ld      r0, 16(r1);                                     \
 171         ld      r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 172         ld      r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 173         mtlr    r0;                                             \
 174         cmpldi  r3, 0;                                          \
 175         beq     bpf_error_slow; /* cr0 = EQ */                  \
 176         mr      r_addr, r3;                                     \
 177         ld      r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 178         /* Great success! */
 179
 180 bpf_slow_path_word_neg:
 181         lis     r_scratch1,-32  /* SKF_LL_OFF */
 182         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 183         blt     bpf_error       /* cr0 = LT */
 184         .globl  sk_load_word_negative_offset
 185 sk_load_word_negative_offset:
 186         sk_negative_common(4)
 187         lwz     r_A, 0(r_addr)
 188         blr
 189
 190 bpf_slow_path_half_neg:
 191         lis     r_scratch1,-32  /* SKF_LL_OFF */
 192         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 193         blt     bpf_error       /* cr0 = LT */
 194         .globl  sk_load_half_negative_offset
 195 sk_load_half_negative_offset:
 196         sk_negative_common(2)
 197         lhz     r_A, 0(r_addr)
 198         blr
 199
 200 bpf_slow_path_byte_neg:
 201         lis     r_scratch1,-32  /* SKF_LL_OFF */
 202         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 203         blt     bpf_error       /* cr0 = LT */
 204         .globl  sk_load_byte_negative_offset
 205 sk_load_byte_negative_offset:
 206         sk_negative_common(1)
 207         lbz     r_A, 0(r_addr)
 208         blr
 209
 210 bpf_slow_path_byte_msh_neg:
 211         lis     r_scratch1,-32  /* SKF_LL_OFF */
 212         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 213         blt     bpf_error       /* cr0 = LT */
 214         .globl  sk_load_byte_msh_negative_offset
 215 sk_load_byte_msh_negative_offset:
 216         sk_negative_common(1)
 217         lbz     r_X, 0(r_addr)
 218         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
 219         blr
 220
 221 bpf_error_slow:
 222         /* fabricate a cr0 = lt */
 223         li      r_scratch1, -1
 224         cmpdi   r_scratch1, 0
 225 bpf_error:
 226         /* Entered with cr0 = lt */
 227         li      r3, 0
 228         /* Generated code will 'blt epilogue', returning 0. */
 229         blr