Line data Source code
1 : /*
2 : * Copyright 2015 Google Inc.
3 : *
4 : * Use of this source code is governed by a BSD-style license that can be
5 : * found in the LICENSE file.
6 : */
7 :
8 : #ifndef SkBlitRow_opts_DEFINED
9 : #define SkBlitRow_opts_DEFINED
10 :
11 : #include "Sk4px.h"
12 : #include "SkColorPriv.h"
13 : #include "SkMSAN.h"
14 :
15 : #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
16 : #include "SkColor_opts_SSE2.h"
17 : #endif
18 :
19 : namespace SK_OPTS_NS {
20 :
21 : // Color32 uses the blend_256_round_alt algorithm from tests/BlendTest.cpp.
22 : // It's not quite perfect, but it's never wrong in the interesting edge cases,
23 : // and it's quite a bit faster than blend_perfect.
24 : //
25 : // blend_256_round_alt is our currently blessed algorithm. Please use it or an analogous one.
26 : static inline
27 4748 : void blit_row_color32(SkPMColor* dst, const SkPMColor* src, int count, SkPMColor color) {
28 4748 : unsigned invA = 255 - SkGetPackedA32(color);
29 4748 : invA += invA >> 7;
30 4748 : SkASSERT(invA < 256); // We've should have already handled alpha == 0 externally.
31 :
32 : #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
33 23740 : __m128i colorHighAndRound = _mm_add_epi16(_mm_unpacklo_epi8(_mm_setzero_si128(), _mm_set1_epi32(color)), _mm_set1_epi16(128));
34 9496 : __m128i invA_16x = _mm_set1_epi16(invA);
35 : #define BLIT_ROW_COLOR32_FN(px, lohi) \
36 : _mm_srli_epi16(_mm_add_epi16(_mm_mullo_epi16(_mm_unpack ## lohi ## _epi8(px, _mm_setzero_si128()), invA_16x), colorHighAndRound), 8)
37 23222 : while (count >= 4) {
38 9237 : __m128i px = _mm_loadu_si128((const __m128i*)src);
39 101607 : _mm_storeu_si128((__m128i*)dst,
40 : _mm_packus_epi16(BLIT_ROW_COLOR32_FN(px, lo), BLIT_ROW_COLOR32_FN(px, hi)));
41 9237 : src += 4;
42 9237 : dst += 4;
43 9237 : count -= 4;
44 : }
45 4748 : if (count >= 2) {
46 476 : __m128i px = _mm_loadl_epi64((const __m128i*)src);
47 3332 : _mm_storel_epi64((__m128i*)dst,
48 : _mm_packus_epi16(BLIT_ROW_COLOR32_FN(px, lo), _mm_setzero_si128()));
49 476 : src += 2;
50 476 : dst += 2;
51 476 : count -= 2;
52 : }
53 4748 : if (count >= 1) {
54 8388 : __m128i px = _mm_cvtsi32_si128(*src);
55 33552 : *dst = _mm_cvtsi128_si32(
56 : _mm_packus_epi16(BLIT_ROW_COLOR32_FN(px, lo), _mm_setzero_si128()));
57 : }
58 : #else
59 : Sk16h colorHighAndRound = Sk4px::DupPMColor(color).widenHi() + Sk16h(128);
60 : Sk16b invA_16x(invA);
61 :
62 : Sk4px::MapSrc(count, dst, src, [&](const Sk4px& src4) -> Sk4px {
63 : return (src4 * invA_16x).addNarrowHi(colorHighAndRound);
64 : });
65 : #endif
66 4748 : }
67 :
68 : static inline
69 10357 : void blit_row_s32a_opaque(SkPMColor* dst, const SkPMColor* src, int len, U8CPU alpha) {
70 10357 : SkASSERT(alpha == 0xFF);
71 10357 : sk_msan_assert_initialized(src, src+len);
72 :
73 : #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41
74 187113 : while (len >= 16) {
75 : // Load 16 source pixels.
76 88378 : auto s0 = _mm_loadu_si128((const __m128i*)(src) + 0),
77 176756 : s1 = _mm_loadu_si128((const __m128i*)(src) + 1),
78 176756 : s2 = _mm_loadu_si128((const __m128i*)(src) + 2),
79 176756 : s3 = _mm_loadu_si128((const __m128i*)(src) + 3);
80 :
81 88378 : const auto alphaMask = _mm_set1_epi32(0xFF000000);
82 :
83 353512 : auto ORed = _mm_or_si128(s3, _mm_or_si128(s2, _mm_or_si128(s1, s0)));
84 88378 : if (_mm_testz_si128(ORed, alphaMask)) {
85 : // All 16 source pixels are transparent. Nothing to do.
86 11090 : src += 16;
87 11090 : dst += 16;
88 11090 : len -= 16;
89 28112 : continue;
90 : }
91 :
92 77288 : auto d0 = (__m128i*)(dst) + 0,
93 77288 : d1 = (__m128i*)(dst) + 1,
94 77288 : d2 = (__m128i*)(dst) + 2,
95 77288 : d3 = (__m128i*)(dst) + 3;
96 :
97 309152 : auto ANDed = _mm_and_si128(s3, _mm_and_si128(s2, _mm_and_si128(s1, s0)));
98 77288 : if (_mm_testc_si128(ANDed, alphaMask)) {
99 : // All 16 source pixels are opaque. SrcOver becomes Src.
100 5932 : _mm_storeu_si128(d0, s0);
101 5932 : _mm_storeu_si128(d1, s1);
102 5932 : _mm_storeu_si128(d2, s2);
103 5932 : _mm_storeu_si128(d3, s3);
104 5932 : src += 16;
105 5932 : dst += 16;
106 5932 : len -= 16;
107 5932 : continue;
108 : }
109 :
110 : // TODO: This math is wrong.
111 : // Do SrcOver.
112 142712 : _mm_storeu_si128(d0, SkPMSrcOver_SSE2(s0, _mm_loadu_si128(d0)));
113 142712 : _mm_storeu_si128(d1, SkPMSrcOver_SSE2(s1, _mm_loadu_si128(d1)));
114 142712 : _mm_storeu_si128(d2, SkPMSrcOver_SSE2(s2, _mm_loadu_si128(d2)));
115 142712 : _mm_storeu_si128(d3, SkPMSrcOver_SSE2(s3, _mm_loadu_si128(d3)));
116 71356 : src += 16;
117 71356 : dst += 16;
118 71356 : len -= 16;
119 : }
120 :
121 : #elif SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
122 0 : while (len >= 16) {
123 : // Load 16 source pixels.
124 0 : auto s0 = _mm_loadu_si128((const __m128i*)(src) + 0),
125 0 : s1 = _mm_loadu_si128((const __m128i*)(src) + 1),
126 0 : s2 = _mm_loadu_si128((const __m128i*)(src) + 2),
127 0 : s3 = _mm_loadu_si128((const __m128i*)(src) + 3);
128 :
129 0 : const auto alphaMask = _mm_set1_epi32(0xFF000000);
130 :
131 0 : auto ORed = _mm_or_si128(s3, _mm_or_si128(s2, _mm_or_si128(s1, s0)));
132 0 : if (0xffff == _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_and_si128(ORed, alphaMask),
133 : _mm_setzero_si128()))) {
134 : // All 16 source pixels are transparent. Nothing to do.
135 0 : src += 16;
136 0 : dst += 16;
137 0 : len -= 16;
138 0 : continue;
139 : }
140 :
141 0 : auto d0 = (__m128i*)(dst) + 0,
142 0 : d1 = (__m128i*)(dst) + 1,
143 0 : d2 = (__m128i*)(dst) + 2,
144 0 : d3 = (__m128i*)(dst) + 3;
145 :
146 0 : auto ANDed = _mm_and_si128(s3, _mm_and_si128(s2, _mm_and_si128(s1, s0)));
147 0 : if (0xffff == _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_and_si128(ANDed, alphaMask),
148 : alphaMask))) {
149 : // All 16 source pixels are opaque. SrcOver becomes Src.
150 0 : _mm_storeu_si128(d0, s0);
151 0 : _mm_storeu_si128(d1, s1);
152 0 : _mm_storeu_si128(d2, s2);
153 0 : _mm_storeu_si128(d3, s3);
154 0 : src += 16;
155 0 : dst += 16;
156 0 : len -= 16;
157 0 : continue;
158 : }
159 :
160 : // TODO: This math is wrong.
161 : // Do SrcOver.
162 0 : _mm_storeu_si128(d0, SkPMSrcOver_SSE2(s0, _mm_loadu_si128(d0)));
163 0 : _mm_storeu_si128(d1, SkPMSrcOver_SSE2(s1, _mm_loadu_si128(d1)));
164 0 : _mm_storeu_si128(d2, SkPMSrcOver_SSE2(s2, _mm_loadu_si128(d2)));
165 0 : _mm_storeu_si128(d3, SkPMSrcOver_SSE2(s3, _mm_loadu_si128(d3)));
166 :
167 0 : src += 16;
168 0 : dst += 16;
169 0 : len -= 16;
170 : }
171 :
172 : #elif defined(SK_ARM_HAS_NEON)
173 : while (len >= 4) {
174 : if ((src[0] | src[1] | src[2] | src[3]) == 0x00000000) {
175 : // All 16 source pixels are transparent. Nothing to do.
176 : src += 4;
177 : dst += 4;
178 : len -= 4;
179 : continue;
180 : }
181 :
182 : if ((src[0] & src[1] & src[2] & src[3]) >= 0xFF000000) {
183 : // All 16 source pixels are opaque. SrcOver becomes Src.
184 : dst[0] = src[0];
185 : dst[1] = src[1];
186 : dst[2] = src[2];
187 : dst[3] = src[3];
188 : src += 4;
189 : dst += 4;
190 : len -= 4;
191 : continue;
192 : }
193 :
194 : // Load 4 source and destination pixels.
195 : auto src0 = vreinterpret_u8_u32(vld1_u32(src+0)),
196 : src2 = vreinterpret_u8_u32(vld1_u32(src+2)),
197 : dst0 = vreinterpret_u8_u32(vld1_u32(dst+0)),
198 : dst2 = vreinterpret_u8_u32(vld1_u32(dst+2));
199 :
200 : // TODO: This math is wrong.
201 : const uint8x8_t alphas = vcreate_u8(0x0707070703030303);
202 : auto invSA0_w = vsubw_u8(vdupq_n_u16(256), vtbl1_u8(src0, alphas)),
203 : invSA2_w = vsubw_u8(vdupq_n_u16(256), vtbl1_u8(src2, alphas));
204 :
205 : auto dstInvSA0 = vmulq_u16(invSA0_w, vmovl_u8(dst0)),
206 : dstInvSA2 = vmulq_u16(invSA2_w, vmovl_u8(dst2));
207 :
208 : dst0 = vadd_u8(src0, vshrn_n_u16(dstInvSA0, 8));
209 : dst2 = vadd_u8(src2, vshrn_n_u16(dstInvSA2, 8));
210 :
211 : vst1_u32(dst+0, vreinterpret_u32_u8(dst0));
212 : vst1_u32(dst+2, vreinterpret_u32_u8(dst2));
213 :
214 : src += 4;
215 : dst += 4;
216 : len -= 4;
217 : }
218 : #endif
219 :
220 165675 : while (len-- > 0) {
221 : // This 0xFF000000 is not semantically necessary, but for compatibility
222 : // with chromium:611002 we need to keep it until we figure out where
223 : // the non-premultiplied src values (like 0x00FFFFFF) are coming from.
224 : // TODO(mtklein): sort this out and assert *src is premul here.
225 77659 : if (*src & 0xFF000000) {
226 51028 : *dst = (*src >= 0xFF000000) ? *src : SkPMSrcOver(*src, *dst);
227 : }
228 77659 : src++;
229 77659 : dst++;
230 : }
231 10357 : }
232 :
233 : } // SK_OPTS_NS
234 :
235 : #endif//SkBlitRow_opts_DEFINED
|
