arch/x86/net/bpf_jit.S

   1 /* bpf_jit.S : BPF JIT helper functions
   2  *
   3  * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; version 2
   8  * of the License.
   9  */
  10 #include <linux/linkage.h>
  11 #include <asm/frame.h>
  12
  13 /*
  14  * Calling convention :
  15  * rbx : skb pointer (callee saved)
  16  * esi : offset of byte(s) to fetch in skb (can be scratched)
  17  * r10 : copy of skb->data
  18  * r9d : hlen = skb->len - skb->data_len
  19  */
  20 #define SKBDATA %r10
  21 #define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
  22
  23 #define FUNC(name) \
  24         .globl name; \
  25         .type name, @function; \
  26         name:
  27
  28 FUNC(sk_load_word)
  29         test    %esi,%esi
  30         js      bpf_slow_path_word_neg
  31
  32 FUNC(sk_load_word_positive_offset)
  33         mov     %r9d,%eax               # hlen
  34         sub     %esi,%eax               # hlen - offset
  35         cmp     $3,%eax
  36         jle     bpf_slow_path_word
  37         mov     (SKBDATA,%rsi),%eax
  38         bswap   %eax                    /* ntohl() */
  39         ret
  40
  41 FUNC(sk_load_half)
  42         test    %esi,%esi
  43         js      bpf_slow_path_half_neg
  44
  45 FUNC(sk_load_half_positive_offset)
  46         mov     %r9d,%eax
  47         sub     %esi,%eax               #       hlen - offset
  48         cmp     $1,%eax
  49         jle     bpf_slow_path_half
  50         movzwl  (SKBDATA,%rsi),%eax
  51         rol     $8,%ax                  # ntohs()
  52         ret
  53
  54 FUNC(sk_load_byte)
  55         test    %esi,%esi
  56         js      bpf_slow_path_byte_neg
  57
  58 FUNC(sk_load_byte_positive_offset)
  59         cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
  60         jle     bpf_slow_path_byte
  61         movzbl  (SKBDATA,%rsi),%eax
  62         ret
  63
  64 /* rsi contains offset and can be scratched */
  65 #define bpf_slow_path_common(LEN)               \
  66         lea     32(%rbp), %rdx;\
  67         FRAME_BEGIN;                            \
  68         mov     %rbx, %rdi; /* arg1 == skb */   \
  69         push    %r9;                            \
  70         push    SKBDATA;                        \
  71 /* rsi already has offset */                    \
  72         mov     $LEN,%ecx;      /* len */       \
  73         call    skb_copy_bits;                  \
  74         test    %eax,%eax;                      \
  75         pop     SKBDATA;                        \
  76         pop     %r9;                            \
  77         FRAME_END
  78
  79
  80 bpf_slow_path_word:
  81         bpf_slow_path_common(4)
  82         js      bpf_error
  83         mov     32(%rbp),%eax
  84         bswap   %eax
  85         ret
  86
  87 bpf_slow_path_half:
  88         bpf_slow_path_common(2)
  89         js      bpf_error
  90         mov     32(%rbp),%ax
  91         rol     $8,%ax
  92         movzwl  %ax,%eax
  93         ret
  94
  95 bpf_slow_path_byte:
  96         bpf_slow_path_common(1)
  97         js      bpf_error
  98         movzbl  32(%rbp),%eax
  99         ret
 100
 101 #define sk_negative_common(SIZE)                                \
 102         FRAME_BEGIN;                                            \
 103         mov     %rbx, %rdi; /* arg1 == skb */                   \
 104         push    %r9;                                            \
 105         push    SKBDATA;                                        \
 106 /* rsi already has offset */                                    \
 107         mov     $SIZE,%edx;     /* size */                      \
 108         call    bpf_internal_load_pointer_neg_helper;           \
 109         test    %rax,%rax;                                      \
 110         pop     SKBDATA;                                        \
 111         pop     %r9;                                            \
 112         FRAME_END;                                              \
 113         jz      bpf_error
 114
 115 bpf_slow_path_word_neg:
 116         cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
 117         jl      bpf_error       /* offset lower -> error  */
 118
 119 FUNC(sk_load_word_negative_offset)
 120         sk_negative_common(4)
 121         mov     (%rax), %eax
 122         bswap   %eax
 123         ret
 124
 125 bpf_slow_path_half_neg:
 126         cmp     SKF_MAX_NEG_OFF, %esi
 127         jl      bpf_error
 128
 129 FUNC(sk_load_half_negative_offset)
 130         sk_negative_common(2)
 131         mov     (%rax),%ax
 132         rol     $8,%ax
 133         movzwl  %ax,%eax
 134         ret
 135
 136 bpf_slow_path_byte_neg:
 137         cmp     SKF_MAX_NEG_OFF, %esi
 138         jl      bpf_error
 139
 140 FUNC(sk_load_byte_negative_offset)
 141         sk_negative_common(1)
 142         movzbl  (%rax), %eax
 143         ret
 144
 145 bpf_error:
 146 # force a return 0 from jit handler
 147         xor     %eax,%eax
 148         mov     (%rbp),%rbx
 149         mov     8(%rbp),%r13
 150         mov     16(%rbp),%r14
 151         mov     24(%rbp),%r15
 152         add     $40, %rbp
 153         leaveq
 154         ret