Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sts=4 et sw=4 tw=99:
3 : * This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "gc/Marking.h"
8 : #include "jit/Disassembler.h"
9 : #include "jit/JitCompartment.h"
10 : #if defined(JS_CODEGEN_X86)
11 : # include "jit/x86/MacroAssembler-x86.h"
12 : #elif defined(JS_CODEGEN_X64)
13 : # include "jit/x64/MacroAssembler-x64.h"
14 : #else
15 : # error "Wrong architecture. Only x86 and x64 should build this file!"
16 : #endif
17 :
18 : #ifdef _MSC_VER
19 : # include <intrin.h> // for __cpuid
20 : # if defined(_M_X64) && (_MSC_FULL_VER >= 160040219)
21 : # include <immintrin.h> // for _xgetbv
22 : # endif
23 : #endif
24 :
25 : using namespace js;
26 : using namespace js::jit;
27 :
28 : void
29 4499 : AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest)
30 : {
31 4499 : if (jumpRelocations_.length())
32 2875 : memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
33 4499 : }
34 :
35 : void
36 4499 : AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest)
37 : {
38 4499 : if (dataRelocations_.length())
39 791 : memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
40 4499 : }
41 :
42 : static void
43 8 : TraceDataRelocations(JSTracer* trc, uint8_t* buffer, CompactBufferReader& reader)
44 : {
45 14 : while (reader.more()) {
46 6 : size_t offset = reader.readUnsigned();
47 6 : void* ptr = X86Encoding::GetPointer(buffer + offset);
48 :
49 : #ifdef JS_PUNBOX64
50 : // All pointers on x64 will have the top bits cleared. If those bits
51 : // are not cleared, this must be a Value.
52 6 : uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
53 6 : if (word >> JSVAL_TAG_SHIFT) {
54 0 : Value v = Value::fromRawBits(word);
55 0 : TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
56 0 : if (word != v.asRawBits()) {
57 : // Only update the code if the Value changed, because the code
58 : // is not writable if we're not moving objects.
59 0 : X86Encoding::SetPointer(buffer + offset, v.bitsAsPunboxPointer());
60 : }
61 0 : continue;
62 : }
63 : #endif
64 :
65 : // No barrier needed since these are constants.
66 6 : gc::Cell* cellPtr = reinterpret_cast<gc::Cell*>(ptr);
67 6 : TraceManuallyBarrieredGenericPointerEdge(trc, &cellPtr, "jit-masm-ptr");
68 6 : if (cellPtr != ptr)
69 0 : X86Encoding::SetPointer(buffer + offset, cellPtr);
70 : }
71 2 : }
72 :
73 :
74 : void
75 2 : AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader)
76 : {
77 2 : ::TraceDataRelocations(trc, code->raw(), reader);
78 2 : }
79 :
80 : void
81 0 : AssemblerX86Shared::trace(JSTracer* trc)
82 : {
83 0 : for (size_t i = 0; i < jumps_.length(); i++) {
84 0 : RelativePatch& rp = jumps_[i];
85 0 : if (rp.kind == Relocation::JITCODE) {
86 0 : JitCode* code = JitCode::FromExecutable((uint8_t*)rp.target);
87 0 : TraceManuallyBarrieredEdge(trc, &code, "masmrel32");
88 0 : MOZ_ASSERT(code == JitCode::FromExecutable((uint8_t*)rp.target));
89 : }
90 : }
91 0 : if (dataRelocations_.length()) {
92 0 : CompactBufferReader reader(dataRelocations_);
93 0 : unsigned char* code = masm.data();
94 0 : ::TraceDataRelocations(trc, code, reader);
95 : }
96 0 : }
97 :
98 : void
99 4499 : AssemblerX86Shared::executableCopy(void* buffer)
100 : {
101 4499 : masm.executableCopy(buffer);
102 :
103 : // Crash diagnostics for bug 1124397. Check the code buffer has not been
104 : // poisoned with 0xE5 bytes.
105 : static const size_t MinPoisoned = 16;
106 4499 : const uint8_t* bytes = (const uint8_t*)buffer;
107 4499 : size_t len = size();
108 :
109 320435 : for (size_t i = 0; i < len; i += MinPoisoned) {
110 315936 : if (bytes[i] != 0xE5)
111 315865 : continue;
112 :
113 71 : size_t startOffset = i;
114 71 : while (startOffset > 0 && bytes[startOffset - 1] == 0xE5)
115 0 : startOffset--;
116 :
117 71 : size_t endOffset = i;
118 73 : while (endOffset + 1 < len && bytes[endOffset + 1] == 0xE5)
119 1 : endOffset++;
120 :
121 71 : if (endOffset - startOffset < MinPoisoned)
122 71 : continue;
123 :
124 : volatile uintptr_t dump[5];
125 0 : blackbox = dump;
126 0 : blackbox[0] = uintptr_t(0xABCD4321);
127 0 : blackbox[1] = uintptr_t(len);
128 0 : blackbox[2] = uintptr_t(startOffset);
129 0 : blackbox[3] = uintptr_t(endOffset);
130 0 : blackbox[4] = uintptr_t(0xFFFF8888);
131 0 : MOZ_CRASH("Corrupt code buffer");
132 : }
133 4499 : }
134 :
135 : void
136 4499 : AssemblerX86Shared::processCodeLabels(uint8_t* rawCode)
137 : {
138 4513 : for (size_t i = 0; i < codeLabels_.length(); i++) {
139 14 : CodeLabel label = codeLabels_[i];
140 14 : Bind(rawCode, label.patchAt(), rawCode + label.target()->offset());
141 : }
142 4499 : }
143 :
144 : AssemblerX86Shared::Condition
145 276 : AssemblerX86Shared::InvertCondition(Condition cond)
146 : {
147 276 : switch (cond) {
148 : case Zero:
149 234 : return NonZero;
150 : case NonZero:
151 32 : return Zero;
152 : case LessThan:
153 5 : return GreaterThanOrEqual;
154 : case LessThanOrEqual:
155 1 : return GreaterThan;
156 : case GreaterThan:
157 1 : return LessThanOrEqual;
158 : case GreaterThanOrEqual:
159 3 : return LessThan;
160 : case Above:
161 0 : return BelowOrEqual;
162 : case AboveOrEqual:
163 0 : return Below;
164 : case Below:
165 0 : return AboveOrEqual;
166 : case BelowOrEqual:
167 0 : return Above;
168 : default:
169 0 : MOZ_CRASH("unexpected condition");
170 : }
171 : }
172 :
173 : AssemblerX86Shared::Condition
174 0 : AssemblerX86Shared::UnsignedCondition(Condition cond)
175 : {
176 0 : switch (cond) {
177 : case Zero:
178 : case NonZero:
179 0 : return cond;
180 : case LessThan:
181 : case Below:
182 0 : return Below;
183 : case LessThanOrEqual:
184 : case BelowOrEqual:
185 0 : return BelowOrEqual;
186 : case GreaterThan:
187 : case Above:
188 0 : return Above;
189 : case AboveOrEqual:
190 : case GreaterThanOrEqual:
191 0 : return AboveOrEqual;
192 : default:
193 0 : MOZ_CRASH("unexpected condition");
194 : }
195 : }
196 :
197 : AssemblerX86Shared::Condition
198 0 : AssemblerX86Shared::ConditionWithoutEqual(Condition cond)
199 : {
200 0 : switch (cond) {
201 : case LessThan:
202 : case LessThanOrEqual:
203 0 : return LessThan;
204 : case Below:
205 : case BelowOrEqual:
206 0 : return Below;
207 : case GreaterThan:
208 : case GreaterThanOrEqual:
209 0 : return GreaterThan;
210 : case Above:
211 : case AboveOrEqual:
212 0 : return Above;
213 : default:
214 0 : MOZ_CRASH("unexpected condition");
215 : }
216 : }
217 :
218 : AssemblerX86Shared::DoubleCondition
219 0 : AssemblerX86Shared::InvertCondition(DoubleCondition cond)
220 : {
221 0 : switch (cond) {
222 : case DoubleEqual:
223 0 : return DoubleNotEqualOrUnordered;
224 : case DoubleEqualOrUnordered:
225 0 : return DoubleNotEqual;
226 : case DoubleNotEqualOrUnordered:
227 0 : return DoubleEqual;
228 : case DoubleNotEqual:
229 0 : return DoubleEqualOrUnordered;
230 : case DoubleLessThan:
231 0 : return DoubleGreaterThanOrEqualOrUnordered;
232 : case DoubleLessThanOrUnordered:
233 0 : return DoubleGreaterThanOrEqual;
234 : case DoubleLessThanOrEqual:
235 0 : return DoubleGreaterThanOrUnordered;
236 : case DoubleLessThanOrEqualOrUnordered:
237 0 : return DoubleGreaterThan;
238 : case DoubleGreaterThan:
239 0 : return DoubleLessThanOrEqualOrUnordered;
240 : case DoubleGreaterThanOrUnordered:
241 0 : return DoubleLessThanOrEqual;
242 : case DoubleGreaterThanOrEqual:
243 0 : return DoubleLessThanOrUnordered;
244 : case DoubleGreaterThanOrEqualOrUnordered:
245 0 : return DoubleLessThan;
246 : default:
247 0 : MOZ_CRASH("unexpected condition");
248 : }
249 : }
250 :
251 : void
252 0 : AssemblerX86Shared::verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
253 : const Disassembler::HeapAccess& heapAccess)
254 : {
255 : #ifdef DEBUG
256 0 : if (masm.oom())
257 0 : return;
258 0 : unsigned char* code = masm.data();
259 0 : Disassembler::VerifyHeapAccess(code + begin, code + end, heapAccess);
260 : #endif
261 : }
262 :
263 : CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE;
264 : CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE;
265 : bool CPUInfo::avxPresent = false;
266 : bool CPUInfo::avxEnabled = false;
267 : bool CPUInfo::popcntPresent = false;
268 : bool CPUInfo::needAmdBugWorkaround = false;
269 :
270 : static uintptr_t
271 0 : ReadXGETBV()
272 : {
273 : // We use a variety of low-level mechanisms to get at the xgetbv
274 : // instruction, including spelling out the xgetbv instruction as bytes,
275 : // because older compilers and assemblers may not recognize the instruction
276 : // by name.
277 0 : size_t xcr0EAX = 0;
278 : #if defined(_XCR_XFEATURE_ENABLED_MASK)
279 : xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
280 : #elif defined(__GNUC__)
281 : // xgetbv returns its results in %eax and %edx, and for our purposes here,
282 : // we're only interested in the %eax value.
283 0 : asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx");
284 : #elif defined(_MSC_VER) && defined(_M_IX86)
285 : __asm {
286 : xor ecx, ecx
287 : _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
288 : mov xcr0EAX, eax
289 : }
290 : #endif
291 0 : return xcr0EAX;
292 : }
293 :
294 : void
295 3 : CPUInfo::SetSSEVersion()
296 : {
297 3 : int flagsEAX = 0;
298 3 : int flagsECX = 0;
299 3 : int flagsEDX = 0;
300 :
301 : #ifdef _MSC_VER
302 : int cpuinfo[4];
303 : __cpuid(cpuinfo, 1);
304 : flagsEAX = cpuinfo[0];
305 : flagsECX = cpuinfo[2];
306 : flagsEDX = cpuinfo[3];
307 : #elif defined(__GNUC__)
308 : # ifdef JS_CODEGEN_X64
309 : asm (
310 : "movl $0x1, %%eax;"
311 : "cpuid;"
312 : : "=a" (flagsEAX), "=c" (flagsECX), "=d" (flagsEDX)
313 : :
314 : : "%ebx"
315 3 : );
316 : # else
317 : // On x86, preserve ebx. The compiler needs it for PIC mode.
318 : // Some older processors don't fill the ecx register with cpuid, so clobber
319 : // it before calling cpuid, so that there's no risk of picking random bits
320 : // indicating SSE3/SSE4 are present.
321 : asm (
322 : "xor %%ecx, %%ecx;"
323 : "movl $0x1, %%eax;"
324 : "pushl %%ebx;"
325 : "cpuid;"
326 : "popl %%ebx;"
327 : : "=a" (flagsEAX), "=c" (flagsECX), "=d" (flagsEDX)
328 : :
329 : :
330 : );
331 : # endif
332 : #else
333 : # error "Unsupported compiler"
334 : #endif
335 :
336 : static const int SSEBit = 1 << 25;
337 : static const int SSE2Bit = 1 << 26;
338 : static const int SSE3Bit = 1 << 0;
339 : static const int SSSE3Bit = 1 << 9;
340 : static const int SSE41Bit = 1 << 19;
341 : static const int SSE42Bit = 1 << 20;
342 :
343 3 : if (flagsECX & SSE42Bit) maxSSEVersion = SSE4_2;
344 0 : else if (flagsECX & SSE41Bit) maxSSEVersion = SSE4_1;
345 0 : else if (flagsECX & SSSE3Bit) maxSSEVersion = SSSE3;
346 0 : else if (flagsECX & SSE3Bit) maxSSEVersion = SSE3;
347 0 : else if (flagsEDX & SSE2Bit) maxSSEVersion = SSE2;
348 0 : else if (flagsEDX & SSEBit) maxSSEVersion = SSE;
349 0 : else maxSSEVersion = NoSSE;
350 :
351 3 : if (maxEnabledSSEVersion != UnknownSSE)
352 0 : maxSSEVersion = Min(maxSSEVersion, maxEnabledSSEVersion);
353 :
354 : static const int AVXBit = 1 << 28;
355 : static const int XSAVEBit = 1 << 27;
356 3 : avxPresent = (flagsECX & AVXBit) && (flagsECX & XSAVEBit) && avxEnabled;
357 :
358 : // If the hardware supports AVX, check whether the OS supports it too.
359 3 : if (avxPresent) {
360 0 : size_t xcr0EAX = ReadXGETBV();
361 : static const int xcr0SSEBit = 1 << 1;
362 : static const int xcr0AVXBit = 1 << 2;
363 0 : avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit);
364 : }
365 :
366 : static const int POPCNTBit = 1 << 23;
367 :
368 3 : popcntPresent = (flagsECX & POPCNTBit);
369 :
370 : // Check if we need to work around an AMD CPU bug (see bug 1281759).
371 : // We check for family 20 models 0-2. Intel doesn't use family 20 at
372 : // this point, so this should only match AMD CPUs.
373 3 : unsigned family = ((flagsEAX >> 20) & 0xff) + ((flagsEAX >> 8) & 0xf);
374 3 : unsigned model = (((flagsEAX >> 16) & 0xf) << 4) + ((flagsEAX >> 4) & 0xf);
375 3 : needAmdBugWorkaround = (family == 20 && model <= 2);
376 3 : }
377 :
378 : volatile uintptr_t* blackbox = nullptr;
|
