Line data Source code
1 : /*
2 : * Copyright 2016 Google Inc.
3 : *
4 : * Use of this source code is governed by a BSD-style license that can be
5 : * found in the LICENSE file.
6 : */
7 :
8 : // It is not safe to #include any header file here unless it has been vetted for ODR safety:
9 : // all symbols used must be file-scoped static or in an anonymous namespace. This applies
10 : // to _all_ header files: C standard library, C++ standard library, Skia... everything.
11 :
12 : #include <immintrin.h> // ODR safe
13 : #include <stdint.h> // ODR safe
14 :
15 : #if defined(__AVX2__)
16 :
17 : namespace hsw {
18 :
19 0 : void convolve_vertically(const int16_t* filter, int filterLen,
20 : uint8_t* const* srcRows, int width,
21 : uint8_t* out, bool hasAlpha) {
22 : // It's simpler to work with the output array in terms of 4-byte pixels.
23 0 : auto dst = (int*)out;
24 :
25 : // Output up to eight pixels per iteration.
26 0 : for (int x = 0; x < width; x += 8) {
27 : // Accumulated result for 4 (non-adjacent) pairs of pixels,
28 : // with each channel in signed 17.14 fixed point.
29 0 : auto accum04 = _mm256_setzero_si256(),
30 0 : accum15 = _mm256_setzero_si256(),
31 0 : accum26 = _mm256_setzero_si256(),
32 0 : accum37 = _mm256_setzero_si256();
33 :
34 : // Convolve with the filter. (This inner loop is where we spend ~all our time.)
35 : // While we can, we consume 2 filter coefficients and 2 rows of 8 pixels each at a time.
36 : auto convolve_16_pixels = [&](__m256i interlaced_coeffs,
37 0 : __m256i pixels_01234567, __m256i pixels_89ABCDEF) {
38 : // Interlaced R0R8 G0G8 B0B8 A0A8 R1R9 G1G9... 32 8-bit values each.
39 0 : auto _08194C5D = _mm256_unpacklo_epi8(pixels_01234567, pixels_89ABCDEF),
40 0 : _2A3B6E7F = _mm256_unpackhi_epi8(pixels_01234567, pixels_89ABCDEF);
41 :
42 : // Still interlaced R0R8 G0G8... as above, each channel expanded to 16-bit lanes.
43 0 : auto _084C = _mm256_unpacklo_epi8(_08194C5D, _mm256_setzero_si256()),
44 0 : _195D = _mm256_unpackhi_epi8(_08194C5D, _mm256_setzero_si256()),
45 0 : _2A6E = _mm256_unpacklo_epi8(_2A3B6E7F, _mm256_setzero_si256()),
46 0 : _3B7F = _mm256_unpackhi_epi8(_2A3B6E7F, _mm256_setzero_si256());
47 :
48 : // accum0_R += R0*coeff0 + R8*coeff1, etc.
49 0 : accum04 = _mm256_add_epi32(accum04, _mm256_madd_epi16(_084C, interlaced_coeffs));
50 0 : accum15 = _mm256_add_epi32(accum15, _mm256_madd_epi16(_195D, interlaced_coeffs));
51 0 : accum26 = _mm256_add_epi32(accum26, _mm256_madd_epi16(_2A6E, interlaced_coeffs));
52 0 : accum37 = _mm256_add_epi32(accum37, _mm256_madd_epi16(_3B7F, interlaced_coeffs));
53 0 : };
54 :
55 0 : int i = 0;
56 0 : for (; i < filterLen/2*2; i += 2) {
57 0 : convolve_16_pixels(_mm256_set1_epi32(*(const int32_t*)(filter+i)),
58 0 : _mm256_loadu_si256((const __m256i*)(srcRows[i+0] + x*4)),
59 0 : _mm256_loadu_si256((const __m256i*)(srcRows[i+1] + x*4)));
60 : }
61 0 : if (i < filterLen) {
62 0 : convolve_16_pixels(_mm256_set1_epi32(*(const int16_t*)(filter+i)),
63 0 : _mm256_loadu_si256((const __m256i*)(srcRows[i] + x*4)),
64 0 : _mm256_setzero_si256());
65 : }
66 :
67 : // Trim the fractional parts off the accumulators.
68 0 : accum04 = _mm256_srai_epi32(accum04, 14);
69 0 : accum15 = _mm256_srai_epi32(accum15, 14);
70 0 : accum26 = _mm256_srai_epi32(accum26, 14);
71 0 : accum37 = _mm256_srai_epi32(accum37, 14);
72 :
73 : // Pack back down to 8-bit channels.
74 0 : auto pixels = _mm256_packus_epi16(_mm256_packs_epi32(accum04, accum15),
75 0 : _mm256_packs_epi32(accum26, accum37));
76 :
77 0 : if (hasAlpha) {
78 : // Clamp alpha to the max of r,g,b to make sure we stay premultiplied.
79 0 : __m256i max_rg = _mm256_max_epu8(pixels, _mm256_srli_epi32(pixels, 8)),
80 0 : max_rgb = _mm256_max_epu8(max_rg, _mm256_srli_epi32(pixels, 16));
81 0 : pixels = _mm256_max_epu8(pixels, _mm256_slli_epi32(max_rgb, 24));
82 : } else {
83 : // Force opaque.
84 0 : pixels = _mm256_or_si256(pixels, _mm256_set1_epi32(0xff000000));
85 : }
86 :
87 : // Normal path to store 8 pixels.
88 0 : if (x + 8 <= width) {
89 : _mm256_storeu_si256((__m256i*)dst, pixels);
90 0 : dst += 8;
91 0 : continue;
92 : }
93 :
94 : // Store one pixel at a time on the last iteration.
95 0 : for (int i = x; i < width; i++) {
96 0 : *dst++ = _mm_cvtsi128_si32(_mm256_castsi256_si128(pixels));
97 0 : pixels = _mm256_permutevar8x32_epi32(pixels, _mm256_setr_epi32(1,2,3,4,5,6,7,0));
98 : }
99 : }
100 0 : }
101 :
102 : }
103 :
104 : namespace SkOpts {
105 : // See SkOpts.h, writing SkConvolutionFilter1D::ConvolutionFixed as the underlying type.
106 : extern void (*convolve_vertically)(const int16_t* filter, int filterLen,
107 : uint8_t* const* srcRows, int width,
108 : uint8_t* out, bool hasAlpha);
109 3 : void Init_hsw() {
110 3 : convolve_vertically = hsw::convolve_vertically;
111 3 : }
112 : }
113 :
114 : #else // defined(__AVX2__) is not true...
115 :
116 : namespace SkOpts { void Init_hsw() {} }
117 :
118 : #endif
|
