simd_memory.hpp

   1 //  templated memory simd functions
   2 //  Copyright (C) 2010 Tim Blechmann <tim@klingt.org>
   3 //
   4 //  This program is free software; you can redistribute it and/or modify
   5 //  it under the terms of the GNU General Public License as published by
   6 //  the Free Software Foundation; either version 2 of the License, or
   7 //  (at your option) any later version.
   8 //
   9 //  This program is distributed in the hope that it will be useful,
  10 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 //  GNU General Public License for more details.
  13 //
  14 //  You should have received a copy of the GNU General Public License
  15 //  along with this program; see the file COPYING.  If not, write to
  16 //  the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  17 //  Boston, MA 02111-1307, USA.
  18
  19
  20 #ifndef SIMD_MEMORY_HPP
  21 #define SIMD_MEMORY_HPP
  22
  23 #include <cassert>
  24 #include <cstring>
  25
  26 #include "vec.hpp"
  27
  28 #if defined(__GNUC__) && defined(NDEBUG)
  29 #define always_inline inline  __attribute__((always_inline))
  30 #else
  31 #define always_inline inline
  32 #endif
  33
  34 namespace nova {
  35
  36 template <typename F>
  37 inline void zerovec(F * dest, unsigned int n)
  38 {
  39     std::memset(dest, 0, n*sizeof(F));
  40 }
  41
  42 template <typename F>
  43 inline void setvec(F * dest, F f, unsigned int n)
  44 {
  45     assert(n);
  46     do
  47         *dest++ = f;
  48     while (--n);
  49 }
  50
  51 namespace detail
  52 {
  53
  54 template <bool aligned, typename F>
  55 inline void store_aligned(vec<F> const & value, F * dest)
  56 {
  57     if (aligned)
  58         value.store_aligned(dest);
  59     else
  60         value.store(dest);
  61 }
  62
  63 template <typename F, unsigned int n, bool aligned>
  64 struct setvec
  65 {
  66     static const int offset = vec<F>::size;
  67
  68     static always_inline void mp_iteration(F * dst, vec<F> const & val)
  69     {
  70         store_aligned<aligned>(val, dst);
  71         setvec<F, n-offset, aligned>::mp_iteration(dst+offset, val);
  72     }
  73 };
  74
  75 template <typename F, bool aligned>
  76 struct setvec<F, 0, aligned>
  77 {
  78     static always_inline void mp_iteration(F * dst, vec<F> const & val)
  79     {}
  80 };
  81
  82 template <typename F, bool aligned>
  83 inline void setvec_simd(F * dest, vec<F> const & val, unsigned int n)
  84 {
  85     const unsigned int offset = vec<F>::objects_per_cacheline;
  86     unsigned int unroll = n / offset;
  87
  88     do
  89     {
  90         setvec<F, offset, aligned>::mp_iteration(dest, val);
  91         dest += offset;
  92     }
  93     while (--unroll);
  94 }
  95
  96 } /* namespace detail */
  97
  98 template <typename F>
  99 inline void zerovec_simd(F * dest, unsigned int n)
 100 {
 101     vec<F> zero; zero.clear();
 102     detail::setvec_simd<F, true>(dest, zero, n);
 103 }
 104
 105 template <unsigned int n, typename F>
 106 inline void zerovec_simd(F *dest)
 107 {
 108     vec<F> zero; zero.clear();
 109     detail::setvec<F, n, true>::mp_iteration(dest, zero);
 110 }
 111
 112 template <typename F>
 113 inline void zerovec_na_simd(F * dest, unsigned int n)
 114 {
 115     vec<F> zero; zero.clear();
 116     detail::setvec_simd<F, false>(dest, zero, n);
 117 }
 118
 119 template <unsigned int n, typename F>
 120 inline void zerovec_na_simd(F *dest)
 121 {
 122     vec<F> zero; zero.clear();
 123     detail::setvec<F, n, false>::mp_iteration(dest, zero);
 124 }
 125
 126
 127
 128 template <typename F>
 129 inline void setvec_simd(F * dest, F f, unsigned int n)
 130 {
 131     vec<F> val(f);
 132     detail::setvec_simd<F, true>(dest, val, n);
 133 }
 134
 135 template <unsigned int n, typename F>
 136 inline void setvec_simd(F *dest, F f)
 137 {
 138     vec<F> val(f);
 139     detail::setvec<F, n, true>::mp_iteration(dest, val);
 140 }
 141
 142 template <typename F>
 143 inline void setvec_na_simd(F * dest, F f, unsigned int n)
 144 {
 145     vec<F> val(f);
 146     detail::setvec_simd<F, false>(dest, val, n);
 147 }
 148
 149 template <unsigned int n, typename F>
 150 inline void setvec_na_simd(F *dest, F f)
 151 {
 152     vec<F> val(f);
 153     detail::setvec<F, n, false>::mp_iteration(dest, val);
 154 }
 155
 156
 157 namespace detail
 158 {
 159
 160 template <typename F, unsigned int n>
 161 struct set_ramp
 162 {
 163     static const int offset = vec<F>::size;
 164
 165     static always_inline void slope_mp_iteration(F * dst, vec<F> & vbase, vec<F> const & vslope)
 166     {
 167         vbase.store_aligned(dst);
 168         vbase += vslope;
 169         set_ramp<F, n-offset>::slope_mp_iteration(dst+offset, vbase, vslope);
 170     }
 171
 172     static always_inline void exp_mp_iteration(F * dst, vec<F> & vbase, vec<F> const & vcurve)
 173     {
 174         vbase.store_aligned(dst);
 175         vbase *= vcurve;
 176         set_ramp<F, n-offset>::exp_mp_iteration(dst+offset, vbase, vcurve);
 177     }
 178 };
 179
 180 template <typename F>
 181 struct set_ramp<F, 0>
 182 {
 183     static always_inline void slope_mp_iteration(F * dst, vec<F> & vbase, vec<F> const & vslope)
 184     {}
 185     static always_inline void exp_mp_iteration(F * dst, vec<F> & vbase, vec<F> const & curve)
 186     {}
 187 };
 188
 189 } /* namespace detail */
 190
 191
 192 template <typename F>
 193 inline void set_slope_vec(F * dest, F f, F slope, unsigned int n)
 194 {
 195     assert(n);
 196     do {
 197         *dest++ = f; f += slope;
 198     } while (--n);
 199 }
 200
 201 template <typename F>
 202 inline void set_slope_vec_simd(F * dest, F f, F slope, unsigned int n)
 203 {
 204     vec<F> vbase, vslope;
 205     vbase.set_slope(f, slope);
 206     vslope.set_vec(vec<F>::size * slope);
 207
 208     unsigned int unroll = n / vec<F>::objects_per_cacheline;
 209     do
 210     {
 211         detail::set_ramp<F, vec<F>::objects_per_cacheline>::slope_mp_iteration(dest, vbase, vslope);
 212         dest += vec<F>::objects_per_cacheline;
 213     } while(--unroll);
 214 }
 215
 216 template <typename F>
 217 inline void set_exp_vec(F * dest, F f, F curve, unsigned int n)
 218 {
 219     assert(n);
 220     do {
 221         *dest++ = f; f *= curve;
 222     } while (--n);
 223 }
 224
 225 namespace detail {
 226
 227 template <int Exponent, typename Type>
 228 class pow_i
 229 {
 230 public:
 231     static Type run(Type const & base)
 232     {
 233         return base * pow_i<Exponent - 1, Type>::run(base);
 234     }
 235 };
 236
 237 template <typename Type>
 238 class pow_i<1, Type>
 239 {
 240 public:
 241     static Type run(Type const & base)
 242     {
 243         return base;
 244     }
 245 };
 246
 247 template <size_t Exponent, typename Type>
 248 Type ipow(Type const & base)
 249 {
 250     return pow_i<Exponent, Type>::run(base);
 251 }
 252
 253 }
 254
 255 template <typename F>
 256 inline void set_exp_vec_simd(F * dest, F f, F curve, unsigned int n)
 257 {
 258     vec<F> vbase, vcurve(detail::ipow<vec<F>::size, F>(curve));
 259     vbase.set_exp(f, curve);
 260
 261     unsigned int unroll = n / vec<F>::objects_per_cacheline;
 262     do
 263     {
 264         detail::set_ramp<F, vec<F>::objects_per_cacheline>::exp_mp_iteration(dest, vbase, vcurve);
 265         dest += vec<F>::objects_per_cacheline;
 266     } while(--unroll);
 267 }
 268
 269
 270 template <typename F>
 271 inline void copyvec(F * dest, const F * src, unsigned int n)
 272 {
 273     std::memcpy(dest, src, n*sizeof(F));
 274 }
 275
 276 namespace detail {
 277
 278 template <typename F, bool src_aligned, bool dst_aligned, unsigned int n>
 279 struct copyvec
 280 {
 281     static const int offset = vec<F>::size;
 282
 283     static always_inline void mp_iteration(F * dst, const F * src)
 284     {
 285         vec<F> val;
 286         if (src_aligned)
 287             val.load_aligned(src);
 288         else
 289             val.load(src);
 290
 291         mp_iteration(dst, src + offset, val);
 292     }
 293
 294     static always_inline void mp_iteration(F * dst, const F * src, vec<F> const & loaded_value)
 295     {
 296         vec<F> val;
 297
 298         if (src_aligned)
 299             val.load_aligned(src);
 300         else
 301             val.load(src);
 302
 303         store_aligned<dst_aligned>(loaded_value, dst);
 304         copyvec<F, src_aligned, dst_aligned, n-offset>::mp_iteration(dst+offset, src+offset, val);
 305     }
 306 };
 307
 308 template <typename F, bool src_aligned, bool dst_aligned>
 309 struct copyvec<F, src_aligned, dst_aligned, 0>
 310 {
 311     static always_inline void mp_iteration(F * dst, const F * src, vec<F> loaded_value)
 312     {}
 313 };
 314
 315 }
 316
 317 #define COPYVEC_FUNCTION(name, src_aligned, dst_aligned)                \
 318 template <typename F>                                                   \
 319 inline void name##_simd(F * dest, const F * src, unsigned int n)        \
 320 {                                                                       \
 321     const int per_loop = vec<F>::objects_per_cacheline;                 \
 322     n /= per_loop;                                                      \
 323     do                                                                  \
 324     {                                                                   \
 325         detail::copyvec<F, src_aligned, dst_aligned, per_loop>::mp_iteration(dest, src); \
 326         dest += per_loop; src += per_loop;                              \
 327     }                                                                   \
 328     while (--n);                                                        \
 329 }                                                                       \
 330                                                                         \
 331 template <unsigned int n, typename F>                                   \
 332 inline void name##_simd(F * dest, const F * src)                        \
 333 {                                                                       \
 334     detail::copyvec<F, src_aligned, dst_aligned, n>::mp_iteration(dest, src); \
 335 }
 336
 337 COPYVEC_FUNCTION(copyvec_aa, true, true)
 338 COPYVEC_FUNCTION(copyvec_na, false, true)
 339 COPYVEC_FUNCTION(copyvec_an, true, false)
 340 COPYVEC_FUNCTION(copyvec_nn, false, false)
 341
 342 template <typename F>
 343 inline void copyvec_simd(F * dest, const F * src, unsigned int n)
 344 {
 345     copyvec_aa_simd(dest, src, n);
 346 }
 347
 348 template <unsigned int n, typename F>
 349 inline void copyvec_simd(F * dest, const F * src)
 350 {
 351     copyvec_aa_simd<n, F>(dest, src);
 352 }
 353
 354 template <typename F>
 355 inline void addvec(F * out, const F * in, unsigned int n)
 356 {
 357     do {
 358         *out++ += *in++;
 359     } while (--n);
 360 }
 361
 362 template <typename F>
 363 inline void addvec(F * out, const F in, unsigned int n)
 364 {
 365     do {
 366         *out++ += in;
 367     } while (--n);
 368 }
 369
 370 template <typename F>
 371 inline void addvec(F * out, const F in, const F slope, unsigned int n)
 372 {
 373     do {
 374         *out++ += in; in += slope;
 375     } while (--n);
 376 }
 377
 378 namespace detail
 379 {
 380
 381 template <typename F, unsigned int n>
 382 struct addvec
 383 {
 384     static const int offset = vec<F>::size;
 385
 386     static always_inline void mp_iteration(F * dst, const F * src)
 387     {
 388         vec<F> v1, v2;
 389         v1.load_aligned(dst);
 390         v2.load_aligned(src);
 391         v1 += v2;
 392         v1.store_aligned(dst);
 393         addvec<F, n-offset>::mp_iteration(dst+offset, src+offset);
 394     }
 395
 396     static always_inline void mp_iteration(F * dst, vec<F> const & in)
 397     {
 398         vec<F> v1;
 399         v1.load_aligned(dst);
 400         v1 += in;
 401         v1.store_aligned(dst);
 402         addvec<F, n-offset>::mp_iteration(dst+offset, in);
 403     }
 404
 405     static always_inline void mp_iteration(F * dst, vec<F> & in, vec<F> const & vslope)
 406     {
 407         vec<F> v1;
 408         v1.load_aligned(dst);
 409         v1 += in;
 410         v1.store_aligned(dst);
 411         in += vslope;
 412         addvec<F, n-offset>::mp_iteration(dst+offset, in, vslope);
 413     }
 414 };
 415
 416 template <typename F>
 417 struct addvec<F, 0>
 418 {
 419     static always_inline void mp_iteration(F * dst, const F * src)
 420     {}
 421
 422     static always_inline void mp_iteration(F * dst, vec<F> const & in)
 423     {}
 424
 425     static always_inline void mp_iteration(F * dst, vec<F> & in, vec<F> const & vslope)
 426     {}
 427 };
 428
 429 }
 430
 431 template <typename F>
 432 inline void addvec_simd(F * out, const F * in, unsigned int n)
 433 {
 434     const int per_loop = vec<F>::objects_per_cacheline;
 435     n /= per_loop;
 436     do
 437     {
 438         detail::addvec<F, per_loop>::mp_iteration(out, in);
 439         out += per_loop; in += per_loop;
 440     }
 441     while (--n);
 442 }
 443
 444 template <typename F>
 445 inline void addvec_simd(F * out, const F in, unsigned int n)
 446 {
 447     const int per_loop = vec<F>::objects_per_cacheline;
 448     vec<F> vin(in);
 449     n /= per_loop;
 450     do
 451     {
 452         detail::addvec<F, per_loop>::mp_iteration(out, vin);
 453         out += per_loop;
 454     }
 455     while (--n);
 456 }
 457
 458 template <typename F>
 459 inline void addvec_simd(F * out, const F in, const F slope, unsigned int n)
 460 {
 461     const int per_loop = vec<F>::objects_per_cacheline;
 462     vec<F> vin; vin.set_slope(in, slope);
 463     vec<F> vslope; vslope.set(slope * vec<F>::size);
 464     n /= per_loop;
 465     do
 466     {
 467         detail::addvec<F, per_loop>::mp_iteration(out, vin, vslope);
 468         out += per_loop;
 469     }
 470     while (--n);
 471 }
 472
 473 template <unsigned int n, typename F>
 474 inline void addvec_simd(F * out, const F * in)
 475 {
 476     detail::addvec<F, n>::mp_iteration(out, in);
 477 }
 478
 479 template <unsigned int n, typename F>
 480 inline void addvec_simd(F * out, const F in)
 481 {
 482     vec<F> vin(in);
 483     detail::addvec<F, n>::mp_iteration(out, vin);
 484 }
 485
 486 template <unsigned int n, typename F>
 487 inline void addvec_simd(F * out, const F in, const F slope)
 488 {
 489     vec<F> vin; vin.set_slope(in, slope);
 490     vec<F> vslope; vslope.set(slope * vec<F>::size);
 491     detail::addvec<F, n>::mp_iteration(out, vin, vslope);
 492 }
 493
 494
 495 } /* namespace nova */
 496
 497 #undef always_inline
 498
 499 #endif /* SIMD_MEMORY_HPP */