Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sts=4 et sw=4 tw=99:
3 : * This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #ifndef jit_x86_shared_Assembler_x86_shared_h
8 : #define jit_x86_shared_Assembler_x86_shared_h
9 :
10 : #include <cstddef>
11 :
12 : #include "jit/shared/Assembler-shared.h"
13 :
14 : #if defined(JS_CODEGEN_X86)
15 : # include "jit/x86/BaseAssembler-x86.h"
16 : #elif defined(JS_CODEGEN_X64)
17 : # include "jit/x64/BaseAssembler-x64.h"
18 : #else
19 : # error "Unknown architecture!"
20 : #endif
21 :
22 : namespace js {
23 : namespace jit {
24 :
25 0 : struct ScratchFloat32Scope : public AutoFloatRegisterScope
26 : {
27 0 : explicit ScratchFloat32Scope(MacroAssembler& masm)
28 0 : : AutoFloatRegisterScope(masm, ScratchFloat32Reg)
29 0 : { }
30 : };
31 :
32 28 : struct ScratchDoubleScope : public AutoFloatRegisterScope
33 : {
34 28 : explicit ScratchDoubleScope(MacroAssembler& masm)
35 28 : : AutoFloatRegisterScope(masm, ScratchDoubleReg)
36 28 : { }
37 : };
38 :
39 0 : struct ScratchSimd128Scope : public AutoFloatRegisterScope
40 : {
41 0 : explicit ScratchSimd128Scope(MacroAssembler& masm)
42 0 : : AutoFloatRegisterScope(masm, ScratchSimd128Reg)
43 0 : { }
44 : };
45 :
46 : class Operand
47 : {
48 : public:
49 : enum Kind {
50 : REG,
51 : MEM_REG_DISP,
52 : FPREG,
53 : MEM_SCALE,
54 : MEM_ADDRESS32
55 : };
56 :
57 : private:
58 : Kind kind_ : 4;
59 : // Used as a Register::Encoding and a FloatRegister::Encoding.
60 : uint32_t base_ : 5;
61 : Scale scale_ : 3;
62 : // We don't use all 8 bits, of course, but GCC complains if the size of
63 : // this field is smaller than the size of Register::Encoding.
64 : Register::Encoding index_ : 8;
65 : int32_t disp_;
66 :
67 : public:
68 19031 : explicit Operand(Register reg)
69 19031 : : kind_(REG),
70 19031 : base_(reg.encoding()),
71 : scale_(TimesOne),
72 : index_(Registers::Invalid),
73 38062 : disp_(0)
74 19031 : { }
75 0 : explicit Operand(FloatRegister reg)
76 0 : : kind_(FPREG),
77 0 : base_(reg.encoding()),
78 : scale_(TimesOne),
79 : index_(Registers::Invalid),
80 0 : disp_(0)
81 0 : { }
82 427731 : explicit Operand(const Address& address)
83 427731 : : kind_(MEM_REG_DISP),
84 427731 : base_(address.base.encoding()),
85 : scale_(TimesOne),
86 : index_(Registers::Invalid),
87 855465 : disp_(address.offset)
88 427734 : { }
89 1166 : explicit Operand(const BaseIndex& address)
90 1166 : : kind_(MEM_SCALE),
91 1166 : base_(address.base.encoding()),
92 1166 : scale_(address.scale),
93 1166 : index_(address.index.encoding()),
94 4664 : disp_(address.offset)
95 1166 : { }
96 28 : Operand(Register base, Register index, Scale scale, int32_t disp = 0)
97 28 : : kind_(MEM_SCALE),
98 28 : base_(base.encoding()),
99 : scale_(scale),
100 28 : index_(index.encoding()),
101 84 : disp_(disp)
102 28 : { }
103 24427 : Operand(Register reg, int32_t disp)
104 24427 : : kind_(MEM_REG_DISP),
105 24427 : base_(reg.encoding()),
106 : scale_(TimesOne),
107 : index_(Registers::Invalid),
108 48854 : disp_(disp)
109 24427 : { }
110 0 : explicit Operand(AbsoluteAddress address)
111 0 : : kind_(MEM_ADDRESS32),
112 : base_(Registers::Invalid),
113 : scale_(TimesOne),
114 : index_(Registers::Invalid),
115 0 : disp_(X86Encoding::AddressImmediate(address.addr))
116 0 : { }
117 : explicit Operand(PatchedAbsoluteAddress address)
118 : : kind_(MEM_ADDRESS32),
119 : base_(Registers::Invalid),
120 : scale_(TimesOne),
121 : index_(Registers::Invalid),
122 : disp_(X86Encoding::AddressImmediate(address.addr))
123 : { }
124 :
125 0 : Address toAddress() const {
126 0 : MOZ_ASSERT(kind() == MEM_REG_DISP);
127 0 : return Address(Register::FromCode(base()), disp());
128 : }
129 :
130 0 : BaseIndex toBaseIndex() const {
131 0 : MOZ_ASSERT(kind() == MEM_SCALE);
132 0 : return BaseIndex(Register::FromCode(base()), Register::FromCode(index()), scale(), disp());
133 : }
134 :
135 1404363 : Kind kind() const {
136 1404363 : return kind_;
137 : }
138 19165 : Register::Encoding reg() const {
139 19165 : MOZ_ASSERT(kind() == REG);
140 19165 : return Register::Encoding(base_);
141 : }
142 453885 : Register::Encoding base() const {
143 453885 : MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
144 453872 : return Register::Encoding(base_);
145 : }
146 1195 : Register::Encoding index() const {
147 1195 : MOZ_ASSERT(kind() == MEM_SCALE);
148 1195 : return index_;
149 : }
150 1194 : Scale scale() const {
151 1194 : MOZ_ASSERT(kind() == MEM_SCALE);
152 1194 : return scale_;
153 : }
154 0 : FloatRegister::Encoding fpu() const {
155 0 : MOZ_ASSERT(kind() == FPREG);
156 0 : return FloatRegister::Encoding(base_);
157 : }
158 453369 : int32_t disp() const {
159 453369 : MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
160 453355 : return disp_;
161 : }
162 0 : void* address() const {
163 0 : MOZ_ASSERT(kind() == MEM_ADDRESS32);
164 0 : return reinterpret_cast<void*>(disp_);
165 : }
166 :
167 665 : bool containsReg(Register r) const {
168 665 : switch (kind()) {
169 155 : case REG: return r.encoding() == reg();
170 507 : case MEM_REG_DISP: return r.encoding() == base();
171 3 : case MEM_SCALE: return r.encoding() == base() || r.encoding() == index();
172 0 : default: return false;
173 : }
174 : }
175 : };
176 :
177 : inline Imm32
178 : Imm64::firstHalf() const
179 : {
180 : return low();
181 : }
182 :
183 : inline Imm32
184 : Imm64::secondHalf() const
185 : {
186 : return hi();
187 : }
188 :
189 : class CPUInfo
190 : {
191 : public:
192 : // As the SSE's were introduced in order, the presence of a later SSE implies
193 : // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2 support.
194 : enum SSEVersion {
195 : UnknownSSE = 0,
196 : NoSSE = 1,
197 : SSE = 2,
198 : SSE2 = 3,
199 : SSE3 = 4,
200 : SSSE3 = 5,
201 : SSE4_1 = 6,
202 : SSE4_2 = 7
203 : };
204 :
205 0 : static SSEVersion GetSSEVersion() {
206 0 : if (maxSSEVersion == UnknownSSE)
207 0 : SetSSEVersion();
208 :
209 0 : MOZ_ASSERT(maxSSEVersion != UnknownSSE);
210 0 : MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE, maxSSEVersion <= maxEnabledSSEVersion);
211 0 : return maxSSEVersion;
212 : }
213 :
214 4503 : static bool IsAVXPresent() {
215 4503 : if (MOZ_UNLIKELY(maxSSEVersion == UnknownSSE))
216 3 : SetSSEVersion();
217 :
218 4503 : MOZ_ASSERT_IF(!avxEnabled, !avxPresent);
219 4503 : return avxPresent;
220 : }
221 :
222 : private:
223 : static SSEVersion maxSSEVersion;
224 : static SSEVersion maxEnabledSSEVersion;
225 : static bool avxPresent;
226 : static bool avxEnabled;
227 : static bool popcntPresent;
228 : static bool needAmdBugWorkaround;
229 :
230 : static void SetSSEVersion();
231 :
232 : public:
233 355998 : static bool IsSSE2Present() {
234 : #ifdef JS_CODEGEN_X64
235 355998 : return true;
236 : #else
237 : return GetSSEVersion() >= SSE2;
238 : #endif
239 : }
240 0 : static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; }
241 0 : static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; }
242 0 : static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; }
243 : static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; }
244 0 : static bool IsPOPCNTPresent() { return popcntPresent; }
245 375 : static bool NeedAmdBugWorkaround() { return needAmdBugWorkaround; }
246 :
247 : static void SetSSE3Disabled() { maxEnabledSSEVersion = SSE2; avxEnabled = false; }
248 : static void SetSSE4Disabled() { maxEnabledSSEVersion = SSSE3; avxEnabled = false; }
249 : static void SetAVXEnabled() { avxEnabled = true; }
250 : };
251 :
252 4503 : class AssemblerX86Shared : public AssemblerShared
253 : {
254 : protected:
255 : struct RelativePatch {
256 : int32_t offset;
257 : void* target;
258 : Relocation::Kind kind;
259 :
260 19650 : RelativePatch(int32_t offset, void* target, Relocation::Kind kind)
261 19650 : : offset(offset),
262 : target(target),
263 19650 : kind(kind)
264 19650 : { }
265 : };
266 :
267 : Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
268 : CompactBufferWriter jumpRelocations_;
269 : CompactBufferWriter dataRelocations_;
270 :
271 4213 : void writeDataRelocation(ImmGCPtr ptr) {
272 4213 : if (ptr.value) {
273 4213 : if (gc::IsInsideNursery(ptr.value))
274 0 : embedsNurseryPointers_ = true;
275 4213 : dataRelocations_.writeUnsigned(masm.currentOffset());
276 : }
277 4213 : }
278 :
279 : protected:
280 : X86Encoding::BaseAssemblerSpecific masm;
281 :
282 : typedef X86Encoding::JmpSrc JmpSrc;
283 : typedef X86Encoding::JmpDst JmpDst;
284 :
285 : public:
286 4503 : AssemblerX86Shared()
287 4503 : {
288 4503 : if (!HasAVX())
289 4503 : masm.disableVEX();
290 4503 : }
291 :
292 : enum Condition {
293 : Equal = X86Encoding::ConditionE,
294 : NotEqual = X86Encoding::ConditionNE,
295 : Above = X86Encoding::ConditionA,
296 : AboveOrEqual = X86Encoding::ConditionAE,
297 : Below = X86Encoding::ConditionB,
298 : BelowOrEqual = X86Encoding::ConditionBE,
299 : GreaterThan = X86Encoding::ConditionG,
300 : GreaterThanOrEqual = X86Encoding::ConditionGE,
301 : LessThan = X86Encoding::ConditionL,
302 : LessThanOrEqual = X86Encoding::ConditionLE,
303 : Overflow = X86Encoding::ConditionO,
304 : CarrySet = X86Encoding::ConditionC,
305 : CarryClear = X86Encoding::ConditionNC,
306 : Signed = X86Encoding::ConditionS,
307 : NotSigned = X86Encoding::ConditionNS,
308 : Zero = X86Encoding::ConditionE,
309 : NonZero = X86Encoding::ConditionNE,
310 : Parity = X86Encoding::ConditionP,
311 : NoParity = X86Encoding::ConditionNP
312 : };
313 :
314 : // If this bit is set, the vucomisd operands have to be inverted.
315 : static const int DoubleConditionBitInvert = 0x10;
316 :
317 : // Bit set when a DoubleCondition does not map to a single x86 condition.
318 : // The macro assembler has to special-case these conditions.
319 : static const int DoubleConditionBitSpecial = 0x20;
320 : static const int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial;
321 :
322 : enum DoubleCondition {
323 : // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN.
324 : DoubleOrdered = NoParity,
325 : DoubleEqual = Equal | DoubleConditionBitSpecial,
326 : DoubleNotEqual = NotEqual,
327 : DoubleGreaterThan = Above,
328 : DoubleGreaterThanOrEqual = AboveOrEqual,
329 : DoubleLessThan = Above | DoubleConditionBitInvert,
330 : DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert,
331 : // If either operand is NaN, these conditions always evaluate to true.
332 : DoubleUnordered = Parity,
333 : DoubleEqualOrUnordered = Equal,
334 : DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial,
335 : DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert,
336 : DoubleGreaterThanOrEqualOrUnordered = BelowOrEqual | DoubleConditionBitInvert,
337 : DoubleLessThanOrUnordered = Below,
338 : DoubleLessThanOrEqualOrUnordered = BelowOrEqual
339 : };
340 :
341 : enum NaNCond {
342 : NaN_HandledByCond,
343 : NaN_IsTrue,
344 : NaN_IsFalse
345 : };
346 :
347 : // If the primary condition returned by ConditionFromDoubleCondition doesn't
348 : // handle NaNs properly, return NaN_IsFalse if the comparison should be
349 : // overridden to return false on NaN, NaN_IsTrue if it should be overridden
350 : // to return true on NaN, or NaN_HandledByCond if no secondary check is
351 : // needed.
352 11 : static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) {
353 11 : switch (cond) {
354 : case DoubleOrdered:
355 : case DoubleNotEqual:
356 : case DoubleGreaterThan:
357 : case DoubleGreaterThanOrEqual:
358 : case DoubleLessThan:
359 : case DoubleLessThanOrEqual:
360 : case DoubleUnordered:
361 : case DoubleEqualOrUnordered:
362 : case DoubleGreaterThanOrUnordered:
363 : case DoubleGreaterThanOrEqualOrUnordered:
364 : case DoubleLessThanOrUnordered:
365 : case DoubleLessThanOrEqualOrUnordered:
366 5 : return NaN_HandledByCond;
367 : case DoubleEqual:
368 3 : return NaN_IsFalse;
369 : case DoubleNotEqualOrUnordered:
370 3 : return NaN_IsTrue;
371 : }
372 :
373 0 : MOZ_CRASH("Unknown double condition");
374 : }
375 :
376 : static void StaticAsserts() {
377 : // DoubleConditionBits should not interfere with x86 condition codes.
378 : JS_STATIC_ASSERT(!((Equal | NotEqual | Above | AboveOrEqual | Below |
379 : BelowOrEqual | Parity | NoParity) & DoubleConditionBits));
380 : }
381 :
382 : static Condition InvertCondition(Condition cond);
383 : static Condition UnsignedCondition(Condition cond);
384 : static Condition ConditionWithoutEqual(Condition cond);
385 :
386 : static DoubleCondition InvertCondition(DoubleCondition cond);
387 :
388 : // Return the primary condition to test. Some primary conditions may not
389 : // handle NaNs properly and may therefore require a secondary condition.
390 : // Use NaNCondFromDoubleCondition to determine what else is needed.
391 11 : static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
392 11 : return static_cast<Condition>(cond & ~DoubleConditionBits);
393 : }
394 :
395 : static void TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader);
396 :
397 : // MacroAssemblers hold onto gcthings, so they are traced by the GC.
398 : void trace(JSTracer* trc);
399 :
400 36937 : bool oom() const {
401 73874 : return AssemblerShared::oom() ||
402 73874 : masm.oom() ||
403 110811 : jumpRelocations_.oom() ||
404 73874 : dataRelocations_.oom();
405 : }
406 :
407 0 : void disableProtection() { masm.disableProtection(); }
408 : void enableProtection() { masm.enableProtection(); }
409 : void setLowerBoundForProtection(size_t size) {
410 : masm.setLowerBoundForProtection(size);
411 : }
412 : void unprotectRegion(unsigned char* first, size_t size) {
413 : masm.unprotectRegion(first, size);
414 : }
415 : void reprotectRegion(unsigned char* first, size_t size) {
416 : masm.reprotectRegion(first, size);
417 : }
418 :
419 0 : void setPrinter(Sprinter* sp) {
420 0 : masm.setPrinter(sp);
421 0 : }
422 :
423 53055 : static const Register getStackPointer() {
424 53055 : return StackPointer;
425 : }
426 :
427 : void executableCopy(void* buffer);
428 0 : bool asmMergeWith(const AssemblerX86Shared& other) {
429 0 : MOZ_ASSERT(other.jumps_.length() == 0);
430 0 : if (!AssemblerShared::asmMergeWith(masm.size(), other))
431 0 : return false;
432 0 : return masm.appendBuffer(other.masm);
433 : }
434 : void processCodeLabels(uint8_t* rawCode);
435 : void copyJumpRelocationTable(uint8_t* dest);
436 : void copyDataRelocationTable(uint8_t* dest);
437 :
438 : // Size of the instruction stream, in bytes.
439 28667 : size_t size() const {
440 28667 : return masm.size();
441 : }
442 : // Size of the jump relocation table, in bytes.
443 9038 : size_t jumpRelocationTableBytes() const {
444 9038 : return jumpRelocations_.length();
445 : }
446 9038 : size_t dataRelocationTableBytes() const {
447 9038 : return dataRelocations_.length();
448 : }
449 : // Size of the data table, in bytes.
450 4539 : size_t bytesNeeded() const {
451 9078 : return size() +
452 4539 : jumpRelocationTableBytes() +
453 4539 : dataRelocationTableBytes();
454 : }
455 :
456 : public:
457 0 : void haltingAlign(int alignment) {
458 0 : masm.haltingAlign(alignment);
459 0 : }
460 0 : void nopAlign(int alignment) {
461 0 : masm.nopAlign(alignment);
462 0 : }
463 0 : void writeCodePointer(CodeOffset* label) {
464 : // A CodeOffset only has one use, bake in the "end of list" value.
465 0 : masm.jumpTablePointer(LabelBase::INVALID_OFFSET);
466 0 : label->bind(masm.size());
467 0 : }
468 0 : void cmovz(const Operand& src, Register dest) {
469 0 : switch (src.kind()) {
470 : case Operand::REG:
471 0 : masm.cmovz_rr(src.reg(), dest.encoding());
472 0 : break;
473 : case Operand::MEM_REG_DISP:
474 0 : masm.cmovz_mr(src.disp(), src.base(), dest.encoding());
475 0 : break;
476 : case Operand::MEM_SCALE:
477 0 : masm.cmovz_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
478 0 : break;
479 : default:
480 0 : MOZ_CRASH("unexpected operand kind");
481 : }
482 0 : }
483 0 : void movl(Imm32 imm32, Register dest) {
484 0 : masm.movl_i32r(imm32.value, dest.encoding());
485 0 : }
486 391 : void movl(Register src, Register dest) {
487 391 : masm.movl_rr(src.encoding(), dest.encoding());
488 391 : }
489 1747 : void movl(const Operand& src, Register dest) {
490 1747 : switch (src.kind()) {
491 : case Operand::REG:
492 26 : masm.movl_rr(src.reg(), dest.encoding());
493 26 : break;
494 : case Operand::MEM_REG_DISP:
495 1670 : masm.movl_mr(src.disp(), src.base(), dest.encoding());
496 1670 : break;
497 : case Operand::MEM_SCALE:
498 51 : masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
499 51 : break;
500 : case Operand::MEM_ADDRESS32:
501 0 : masm.movl_mr(src.address(), dest.encoding());
502 0 : break;
503 : default:
504 0 : MOZ_CRASH("unexpected operand kind");
505 : }
506 1747 : }
507 2601 : void movl(Register src, const Operand& dest) {
508 2601 : switch (dest.kind()) {
509 : case Operand::REG:
510 3 : masm.movl_rr(src.encoding(), dest.reg());
511 3 : break;
512 : case Operand::MEM_REG_DISP:
513 2598 : masm.movl_rm(src.encoding(), dest.disp(), dest.base());
514 2598 : break;
515 : case Operand::MEM_SCALE:
516 0 : masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
517 0 : break;
518 : case Operand::MEM_ADDRESS32:
519 0 : masm.movl_rm(src.encoding(), dest.address());
520 0 : break;
521 : default:
522 0 : MOZ_CRASH("unexpected operand kind");
523 : }
524 2601 : }
525 4123 : void movl(Imm32 imm32, const Operand& dest) {
526 4123 : switch (dest.kind()) {
527 : case Operand::REG:
528 0 : masm.movl_i32r(imm32.value, dest.reg());
529 0 : break;
530 : case Operand::MEM_REG_DISP:
531 4123 : masm.movl_i32m(imm32.value, dest.disp(), dest.base());
532 4123 : break;
533 : case Operand::MEM_SCALE:
534 0 : masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), dest.scale());
535 0 : break;
536 : case Operand::MEM_ADDRESS32:
537 0 : masm.movl_i32m(imm32.value, dest.address());
538 0 : break;
539 : default:
540 0 : MOZ_CRASH("unexpected operand kind");
541 : }
542 4123 : }
543 :
544 : void xchgl(Register src, Register dest) {
545 : masm.xchgl_rr(src.encoding(), dest.encoding());
546 : }
547 :
548 : // Eventually vmovapd should be overloaded to support loads and
549 : // stores too.
550 0 : void vmovapd(FloatRegister src, FloatRegister dest) {
551 0 : MOZ_ASSERT(HasSSE2());
552 0 : masm.vmovapd_rr(src.encoding(), dest.encoding());
553 0 : }
554 :
555 0 : void vmovaps(FloatRegister src, FloatRegister dest) {
556 0 : MOZ_ASSERT(HasSSE2());
557 0 : masm.vmovaps_rr(src.encoding(), dest.encoding());
558 0 : }
559 0 : void vmovaps(const Operand& src, FloatRegister dest) {
560 0 : MOZ_ASSERT(HasSSE2());
561 0 : switch (src.kind()) {
562 : case Operand::MEM_REG_DISP:
563 0 : masm.vmovaps_mr(src.disp(), src.base(), dest.encoding());
564 0 : break;
565 : case Operand::MEM_SCALE:
566 0 : masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
567 0 : break;
568 : case Operand::FPREG:
569 0 : masm.vmovaps_rr(src.fpu(), dest.encoding());
570 0 : break;
571 : default:
572 0 : MOZ_CRASH("unexpected operand kind");
573 : }
574 0 : }
575 0 : void vmovaps(FloatRegister src, const Operand& dest) {
576 0 : MOZ_ASSERT(HasSSE2());
577 0 : switch (dest.kind()) {
578 : case Operand::MEM_REG_DISP:
579 0 : masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base());
580 0 : break;
581 : case Operand::MEM_SCALE:
582 0 : masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
583 0 : break;
584 : default:
585 0 : MOZ_CRASH("unexpected operand kind");
586 : }
587 0 : }
588 165543 : void vmovups(const Operand& src, FloatRegister dest) {
589 165543 : MOZ_ASSERT(HasSSE2());
590 165543 : switch (src.kind()) {
591 : case Operand::MEM_REG_DISP:
592 165543 : masm.vmovups_mr(src.disp(), src.base(), dest.encoding());
593 165522 : break;
594 : case Operand::MEM_SCALE:
595 0 : masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
596 0 : break;
597 : default:
598 0 : MOZ_CRASH("unexpected operand kind");
599 : }
600 165522 : }
601 165673 : void vmovups(FloatRegister src, const Operand& dest) {
602 165673 : MOZ_ASSERT(HasSSE2());
603 165673 : switch (dest.kind()) {
604 : case Operand::MEM_REG_DISP:
605 165673 : masm.vmovups_rm(src.encoding(), dest.disp(), dest.base());
606 165663 : break;
607 : case Operand::MEM_SCALE:
608 0 : masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
609 0 : break;
610 : default:
611 0 : MOZ_CRASH("unexpected operand kind");
612 : }
613 165663 : }
614 :
615 : // vmovsd is only provided in load/store form since the
616 : // register-to-register form has different semantics (it doesn't clobber
617 : // the whole output register) and isn't needed currently.
618 16 : void vmovsd(const Address& src, FloatRegister dest) {
619 16 : masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding());
620 16 : }
621 0 : void vmovsd(const BaseIndex& src, FloatRegister dest) {
622 0 : masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding());
623 0 : }
624 7 : void vmovsd(FloatRegister src, const Address& dest) {
625 7 : masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding());
626 7 : }
627 0 : void vmovsd(FloatRegister src, const BaseIndex& dest) {
628 0 : masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale);
629 0 : }
630 : // Although vmovss is not only provided in load/store form (for the same
631 : // reasons as vmovsd above), the register to register form should be only
632 : // used in contexts where we care about not clearing the higher lanes of
633 : // the FloatRegister.
634 0 : void vmovss(const Address& src, FloatRegister dest) {
635 0 : masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding());
636 0 : }
637 0 : void vmovss(const BaseIndex& src, FloatRegister dest) {
638 0 : masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding());
639 0 : }
640 0 : void vmovss(FloatRegister src, const Address& dest) {
641 0 : masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding());
642 0 : }
643 0 : void vmovss(FloatRegister src, const BaseIndex& dest) {
644 0 : masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale);
645 0 : }
646 0 : void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
647 0 : masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding());
648 0 : }
649 0 : void vmovdqu(const Operand& src, FloatRegister dest) {
650 0 : MOZ_ASSERT(HasSSE2());
651 0 : switch (src.kind()) {
652 : case Operand::MEM_REG_DISP:
653 0 : masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding());
654 0 : break;
655 : case Operand::MEM_SCALE:
656 0 : masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
657 0 : break;
658 : default:
659 0 : MOZ_CRASH("unexpected operand kind");
660 : }
661 0 : }
662 0 : void vmovdqu(FloatRegister src, const Operand& dest) {
663 0 : MOZ_ASSERT(HasSSE2());
664 0 : switch (dest.kind()) {
665 : case Operand::MEM_REG_DISP:
666 0 : masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base());
667 0 : break;
668 : case Operand::MEM_SCALE:
669 0 : masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
670 0 : break;
671 : default:
672 0 : MOZ_CRASH("unexpected operand kind");
673 : }
674 0 : }
675 0 : void vmovdqa(const Operand& src, FloatRegister dest) {
676 0 : MOZ_ASSERT(HasSSE2());
677 0 : switch (src.kind()) {
678 : case Operand::FPREG:
679 0 : masm.vmovdqa_rr(src.fpu(), dest.encoding());
680 0 : break;
681 : case Operand::MEM_REG_DISP:
682 0 : masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding());
683 0 : break;
684 : case Operand::MEM_SCALE:
685 0 : masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
686 0 : break;
687 : default:
688 0 : MOZ_CRASH("unexpected operand kind");
689 : }
690 0 : }
691 0 : void vmovdqa(FloatRegister src, const Operand& dest) {
692 0 : MOZ_ASSERT(HasSSE2());
693 0 : switch (dest.kind()) {
694 : case Operand::MEM_REG_DISP:
695 0 : masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base());
696 0 : break;
697 : case Operand::MEM_SCALE:
698 0 : masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
699 0 : break;
700 : default:
701 0 : MOZ_CRASH("unexpected operand kind");
702 : }
703 0 : }
704 0 : void vmovdqa(FloatRegister src, FloatRegister dest) {
705 0 : MOZ_ASSERT(HasSSE2());
706 0 : masm.vmovdqa_rr(src.encoding(), dest.encoding());
707 0 : }
708 0 : void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
709 0 : MOZ_ASSERT(HasSSE2());
710 0 : masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
711 0 : }
712 0 : void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
713 0 : MOZ_ASSERT(HasSSE2());
714 0 : masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
715 0 : }
716 656 : void movzbl(const Operand& src, Register dest) {
717 656 : switch (src.kind()) {
718 : case Operand::MEM_REG_DISP:
719 124 : masm.movzbl_mr(src.disp(), src.base(), dest.encoding());
720 124 : break;
721 : case Operand::MEM_SCALE:
722 532 : masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
723 532 : break;
724 : default:
725 0 : MOZ_CRASH("unexpected operand kind");
726 : }
727 656 : }
728 0 : void movsbl(Register src, Register dest) {
729 0 : masm.movsbl_rr(src.encoding(), dest.encoding());
730 0 : }
731 0 : void movsbl(const Operand& src, Register dest) {
732 0 : switch (src.kind()) {
733 : case Operand::MEM_REG_DISP:
734 0 : masm.movsbl_mr(src.disp(), src.base(), dest.encoding());
735 0 : break;
736 : case Operand::MEM_SCALE:
737 0 : masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
738 0 : break;
739 : default:
740 0 : MOZ_CRASH("unexpected operand kind");
741 : }
742 0 : }
743 0 : void movb(const Operand& src, Register dest) {
744 0 : switch (src.kind()) {
745 : case Operand::MEM_REG_DISP:
746 0 : masm.movb_mr(src.disp(), src.base(), dest.encoding());
747 0 : break;
748 : case Operand::MEM_SCALE:
749 0 : masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
750 0 : break;
751 : default:
752 0 : MOZ_CRASH("unexpected operand kind");
753 : }
754 0 : }
755 0 : void movb(Imm32 src, Register dest) {
756 0 : masm.movb_ir(src.value & 255, dest.encoding());
757 0 : }
758 11 : void movb(Register src, const Operand& dest) {
759 11 : switch (dest.kind()) {
760 : case Operand::MEM_REG_DISP:
761 8 : masm.movb_rm(src.encoding(), dest.disp(), dest.base());
762 8 : break;
763 : case Operand::MEM_SCALE:
764 3 : masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
765 3 : break;
766 : default:
767 0 : MOZ_CRASH("unexpected operand kind");
768 : }
769 11 : }
770 4 : void movb(Imm32 src, const Operand& dest) {
771 4 : switch (dest.kind()) {
772 : case Operand::MEM_REG_DISP:
773 4 : masm.movb_im(src.value, dest.disp(), dest.base());
774 4 : break;
775 : case Operand::MEM_SCALE:
776 0 : masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale());
777 0 : break;
778 : default:
779 0 : MOZ_CRASH("unexpected operand kind");
780 : }
781 4 : }
782 203 : void movzwl(const Operand& src, Register dest) {
783 203 : switch (src.kind()) {
784 : case Operand::REG:
785 0 : masm.movzwl_rr(src.reg(), dest.encoding());
786 0 : break;
787 : case Operand::MEM_REG_DISP:
788 130 : masm.movzwl_mr(src.disp(), src.base(), dest.encoding());
789 130 : break;
790 : case Operand::MEM_SCALE:
791 73 : masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
792 73 : break;
793 : default:
794 0 : MOZ_CRASH("unexpected operand kind");
795 : }
796 203 : }
797 0 : void movzwl(Register src, Register dest) {
798 0 : masm.movzwl_rr(src.encoding(), dest.encoding());
799 0 : }
800 0 : void movw(const Operand& src, Register dest) {
801 0 : masm.prefix_16_for_32();
802 0 : movl(src, dest);
803 0 : }
804 : void movw(Imm32 src, Register dest) {
805 : masm.prefix_16_for_32();
806 : movl(src, dest);
807 : }
808 36 : void movw(Register src, const Operand& dest) {
809 36 : switch (dest.kind()) {
810 : case Operand::MEM_REG_DISP:
811 36 : masm.movw_rm(src.encoding(), dest.disp(), dest.base());
812 36 : break;
813 : case Operand::MEM_SCALE:
814 0 : masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
815 0 : break;
816 : default:
817 0 : MOZ_CRASH("unexpected operand kind");
818 : }
819 36 : }
820 4 : void movw(Imm32 src, const Operand& dest) {
821 4 : switch (dest.kind()) {
822 : case Operand::MEM_REG_DISP:
823 4 : masm.movw_im(src.value, dest.disp(), dest.base());
824 4 : break;
825 : case Operand::MEM_SCALE:
826 0 : masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale());
827 0 : break;
828 : default:
829 0 : MOZ_CRASH("unexpected operand kind");
830 : }
831 4 : }
832 0 : void movswl(Register src, Register dest) {
833 0 : masm.movswl_rr(src.encoding(), dest.encoding());
834 0 : }
835 0 : void movswl(const Operand& src, Register dest) {
836 0 : switch (src.kind()) {
837 : case Operand::MEM_REG_DISP:
838 0 : masm.movswl_mr(src.disp(), src.base(), dest.encoding());
839 0 : break;
840 : case Operand::MEM_SCALE:
841 0 : masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
842 0 : break;
843 : default:
844 0 : MOZ_CRASH("unexpected operand kind");
845 : }
846 0 : }
847 0 : void leal(const Operand& src, Register dest) {
848 0 : switch (src.kind()) {
849 : case Operand::MEM_REG_DISP:
850 0 : masm.leal_mr(src.disp(), src.base(), dest.encoding());
851 0 : break;
852 : case Operand::MEM_SCALE:
853 0 : masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
854 0 : break;
855 : default:
856 0 : MOZ_CRASH("unexpected operand kind");
857 : }
858 0 : }
859 :
860 : protected:
861 40904 : void jSrc(Condition cond, Label* label) {
862 40904 : if (label->bound()) {
863 : // The jump can be immediately encoded to the correct destination.
864 535 : masm.jCC_i(static_cast<X86Encoding::Condition>(cond), JmpDst(label->offset()));
865 : } else {
866 : // Thread the jump list through the unpatched jump targets.
867 40369 : JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond));
868 40368 : JmpSrc prev = JmpSrc(label->use(j.offset()));
869 40368 : masm.setNextJump(j, prev);
870 : }
871 40903 : }
872 7280 : void jmpSrc(Label* label) {
873 7280 : if (label->bound()) {
874 : // The jump can be immediately encoded to the correct destination.
875 695 : masm.jmp_i(JmpDst(label->offset()));
876 : } else {
877 : // Thread the jump list through the unpatched jump targets.
878 6585 : JmpSrc j = masm.jmp();
879 6585 : JmpSrc prev = JmpSrc(label->use(j.offset()));
880 6585 : masm.setNextJump(j, prev);
881 : }
882 7280 : }
883 :
884 : // Comparison of EAX against the address given by a Label.
885 : JmpSrc cmpSrc(Label* label) {
886 : JmpSrc j = masm.cmp_eax();
887 : if (label->bound()) {
888 : // The jump can be immediately patched to the correct destination.
889 : masm.linkJump(j, JmpDst(label->offset()));
890 : } else {
891 : // Thread the jump list through the unpatched jump targets.
892 : JmpSrc prev = JmpSrc(label->use(j.offset()));
893 : masm.setNextJump(j, prev);
894 : }
895 : return j;
896 : }
897 :
898 0 : JmpSrc jSrc(Condition cond, RepatchLabel* label) {
899 0 : JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond));
900 0 : if (label->bound()) {
901 : // The jump can be immediately patched to the correct destination.
902 0 : masm.linkJump(j, JmpDst(label->offset()));
903 : } else {
904 0 : label->use(j.offset());
905 : }
906 0 : return j;
907 : }
908 21 : JmpSrc jmpSrc(RepatchLabel* label) {
909 21 : JmpSrc j = masm.jmp();
910 21 : if (label->bound()) {
911 : // The jump can be immediately patched to the correct destination.
912 0 : masm.linkJump(j, JmpDst(label->offset()));
913 : } else {
914 : // Thread the jump list through the unpatched jump targets.
915 21 : label->use(j.offset());
916 : }
917 21 : return j;
918 : }
919 :
920 : public:
921 64 : void nop() { masm.nop(); }
922 0 : void nop(size_t n) { masm.insert_nop(n); }
923 40904 : void j(Condition cond, Label* label) { jSrc(cond, label); }
924 7280 : void jmp(Label* label) { jmpSrc(label); }
925 : void j(Condition cond, RepatchLabel* label) { jSrc(cond, label); }
926 : void jmp(RepatchLabel* label) { jmpSrc(label); }
927 :
928 0 : void j(Condition cond, wasm::TrapDesc target) {
929 0 : Label l;
930 0 : j(cond, &l);
931 0 : bindLater(&l, target);
932 0 : }
933 0 : void jmp(wasm::TrapDesc target) {
934 0 : Label l;
935 0 : jmp(&l);
936 0 : bindLater(&l, target);
937 0 : }
938 :
939 2583 : void jmp(const Operand& op) {
940 2583 : switch (op.kind()) {
941 : case Operand::MEM_REG_DISP:
942 2384 : masm.jmp_m(op.disp(), op.base());
943 2384 : break;
944 : case Operand::MEM_SCALE:
945 0 : masm.jmp_m(op.disp(), op.base(), op.index(), op.scale());
946 0 : break;
947 : case Operand::REG:
948 199 : masm.jmp_r(op.reg());
949 199 : break;
950 : default:
951 0 : MOZ_CRASH("unexpected operand kind");
952 : }
953 2583 : }
954 : void cmpEAX(Label* label) { cmpSrc(label); }
955 114023 : void bind(Label* label) {
956 114023 : JmpDst dst(masm.label());
957 114025 : if (label->used()) {
958 : bool more;
959 40344 : JmpSrc jmp(label->offset());
960 47324 : do {
961 47323 : JmpSrc next;
962 47323 : more = masm.nextJump(jmp, &next);
963 47323 : masm.linkJump(jmp, dst);
964 47324 : jmp = next;
965 : } while (more);
966 : }
967 114025 : label->bind(dst.offset());
968 114025 : }
969 0 : void bindLater(Label* label, wasm::TrapDesc target) {
970 0 : if (label->used()) {
971 0 : JmpSrc jmp(label->offset());
972 0 : do {
973 0 : append(wasm::TrapSite(target, jmp.offset()));
974 0 : } while (masm.nextJump(jmp, &jmp));
975 : }
976 0 : label->reset();
977 0 : }
978 21 : void bind(RepatchLabel* label) {
979 21 : JmpDst dst(masm.label());
980 21 : if (label->used()) {
981 21 : JmpSrc jmp(label->offset());
982 21 : masm.linkJump(jmp, dst);
983 : }
984 21 : label->bind(dst.offset());
985 21 : }
986 17 : void use(CodeOffset* label) {
987 17 : label->bind(currentOffset());
988 17 : }
989 86849 : uint32_t currentOffset() {
990 86849 : return masm.label().offset();
991 : }
992 :
993 : // Re-routes pending jumps to a new label.
994 279 : void retarget(Label* label, Label* target) {
995 279 : if (!label->used())
996 0 : return;
997 : bool more;
998 279 : JmpSrc jmp(label->offset());
999 305 : do {
1000 305 : JmpSrc next;
1001 305 : more = masm.nextJump(jmp, &next);
1002 305 : if (target->bound()) {
1003 : // The jump can be immediately patched to the correct destination.
1004 0 : masm.linkJump(jmp, JmpDst(target->offset()));
1005 : } else {
1006 : // Thread the jump list through the unpatched jump targets.
1007 305 : JmpSrc prev(target->use(jmp.offset()));
1008 305 : masm.setNextJump(jmp, prev);
1009 : }
1010 305 : jmp = JmpSrc(next.offset());
1011 : } while (more);
1012 279 : label->reset();
1013 : }
1014 :
1015 14 : static void Bind(uint8_t* raw, CodeOffset* label, const void* address) {
1016 14 : if (label->bound()) {
1017 14 : intptr_t offset = label->offset();
1018 14 : X86Encoding::SetPointer(raw + offset, address);
1019 : }
1020 14 : }
1021 :
1022 : // See Bind and X86Encoding::setPointer.
1023 0 : size_t labelToPatchOffset(CodeOffset label) {
1024 0 : return label.offset() - sizeof(void*);
1025 : }
1026 :
1027 3552 : void ret() {
1028 3552 : masm.ret();
1029 3552 : }
1030 1036 : void retn(Imm32 n) {
1031 : // Remove the size of the return address which is included in the frame.
1032 1036 : masm.ret_i(n.value - sizeof(void*));
1033 1036 : }
1034 373 : CodeOffset call(Label* label) {
1035 373 : if (label->bound()) {
1036 0 : masm.linkJump(masm.call(), JmpDst(label->offset()));
1037 : } else {
1038 373 : JmpSrc j = masm.call();
1039 373 : JmpSrc prev = JmpSrc(label->use(j.offset()));
1040 373 : masm.setNextJump(j, prev);
1041 : }
1042 373 : return CodeOffset(masm.currentOffset());
1043 : }
1044 94 : CodeOffset call(Register reg) {
1045 94 : masm.call_r(reg.encoding());
1046 94 : return CodeOffset(masm.currentOffset());
1047 : }
1048 19474 : void call(const Operand& op) {
1049 19474 : switch (op.kind()) {
1050 : case Operand::REG:
1051 0 : masm.call_r(op.reg());
1052 0 : break;
1053 : case Operand::MEM_REG_DISP:
1054 19474 : masm.call_m(op.disp(), op.base());
1055 19474 : break;
1056 : default:
1057 0 : MOZ_CRASH("unexpected operand kind");
1058 : }
1059 19474 : }
1060 :
1061 0 : CodeOffset callWithPatch() {
1062 0 : return CodeOffset(masm.call().offset());
1063 : }
1064 :
1065 0 : void patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
1066 0 : unsigned char* code = masm.data();
1067 0 : X86Encoding::SetRel32(code + callerOffset, code + calleeOffset);
1068 0 : }
1069 0 : CodeOffset farJumpWithPatch() {
1070 0 : return CodeOffset(masm.jmp().offset());
1071 : }
1072 0 : void patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
1073 0 : unsigned char* code = masm.data();
1074 0 : X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset);
1075 0 : }
1076 0 : static void repatchFarJump(uint8_t* code, uint32_t farJumpOffset, uint32_t targetOffset) {
1077 0 : X86Encoding::SetRel32(code + farJumpOffset, code + targetOffset);
1078 0 : }
1079 :
1080 : // This is for patching during code generation, not after.
1081 : void patchAddl(CodeOffset offset, int32_t n) {
1082 : unsigned char* code = masm.data();
1083 : X86Encoding::SetInt32(code + offset.offset(), n);
1084 : }
1085 :
1086 0 : CodeOffset twoByteNop() {
1087 0 : return CodeOffset(masm.twoByteNop().offset());
1088 : }
1089 0 : static void patchTwoByteNopToJump(uint8_t* jump, uint8_t* target) {
1090 0 : X86Encoding::BaseAssembler::patchTwoByteNopToJump(jump, target);
1091 0 : }
1092 0 : static void patchJumpToTwoByteNop(uint8_t* jump) {
1093 0 : X86Encoding::BaseAssembler::patchJumpToTwoByteNop(jump);
1094 0 : }
1095 :
1096 0 : static void patchFiveByteNopToCall(uint8_t* callsite, uint8_t* target) {
1097 0 : X86Encoding::BaseAssembler::patchFiveByteNopToCall(callsite, target);
1098 0 : }
1099 0 : static void patchCallToFiveByteNop(uint8_t* callsite) {
1100 0 : X86Encoding::BaseAssembler::patchCallToFiveByteNop(callsite);
1101 0 : }
1102 :
1103 21541 : void breakpoint() {
1104 21541 : masm.int3();
1105 21541 : }
1106 :
1107 331376 : static bool HasSSE2() { return CPUInfo::IsSSE2Present(); }
1108 0 : static bool HasSSE3() { return CPUInfo::IsSSE3Present(); }
1109 0 : static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); }
1110 0 : static bool HasSSE41() { return CPUInfo::IsSSE41Present(); }
1111 0 : static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); }
1112 4 : static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); }
1113 4 : static bool SupportsUnalignedAccesses() { return true; }
1114 24627 : static bool SupportsSimd() { return CPUInfo::IsSSE2Present(); }
1115 4503 : static bool HasAVX() { return CPUInfo::IsAVXPresent(); }
1116 :
1117 0 : static bool HasRoundInstruction(RoundingMode mode) {
1118 0 : switch (mode) {
1119 : case RoundingMode::Up:
1120 : case RoundingMode::Down:
1121 : case RoundingMode::NearestTiesToEven:
1122 : case RoundingMode::TowardsZero:
1123 0 : return CPUInfo::IsSSE41Present();
1124 : }
1125 0 : MOZ_CRASH("unexpected mode");
1126 : }
1127 :
1128 221 : void cmpl(Register rhs, Register lhs) {
1129 221 : masm.cmpl_rr(rhs.encoding(), lhs.encoding());
1130 221 : }
1131 8 : void cmpl(const Operand& rhs, Register lhs) {
1132 8 : switch (rhs.kind()) {
1133 : case Operand::REG:
1134 5 : masm.cmpl_rr(rhs.reg(), lhs.encoding());
1135 5 : break;
1136 : case Operand::MEM_REG_DISP:
1137 3 : masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding());
1138 3 : break;
1139 : case Operand::MEM_ADDRESS32:
1140 0 : masm.cmpl_mr(rhs.address(), lhs.encoding());
1141 0 : break;
1142 : default:
1143 0 : MOZ_CRASH("unexpected operand kind");
1144 : }
1145 8 : }
1146 101 : void cmpl(Register rhs, const Operand& lhs) {
1147 101 : switch (lhs.kind()) {
1148 : case Operand::REG:
1149 0 : masm.cmpl_rr(rhs.encoding(), lhs.reg());
1150 0 : break;
1151 : case Operand::MEM_REG_DISP:
1152 101 : masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base());
1153 101 : break;
1154 : case Operand::MEM_ADDRESS32:
1155 0 : masm.cmpl_rm(rhs.encoding(), lhs.address());
1156 0 : break;
1157 : default:
1158 0 : MOZ_CRASH("unexpected operand kind");
1159 : }
1160 101 : }
1161 7659 : void cmpl(Imm32 rhs, Register lhs) {
1162 7659 : masm.cmpl_ir(rhs.value, lhs.encoding());
1163 7659 : }
1164 1658 : void cmpl(Imm32 rhs, const Operand& lhs) {
1165 1658 : switch (lhs.kind()) {
1166 : case Operand::REG:
1167 0 : masm.cmpl_ir(rhs.value, lhs.reg());
1168 0 : break;
1169 : case Operand::MEM_REG_DISP:
1170 1658 : masm.cmpl_im(rhs.value, lhs.disp(), lhs.base());
1171 1658 : break;
1172 : case Operand::MEM_SCALE:
1173 0 : masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), lhs.scale());
1174 0 : break;
1175 : case Operand::MEM_ADDRESS32:
1176 0 : masm.cmpl_im(rhs.value, lhs.address());
1177 0 : break;
1178 : default:
1179 0 : MOZ_CRASH("unexpected operand kind");
1180 : }
1181 1658 : }
1182 : CodeOffset cmplWithPatch(Imm32 rhs, Register lhs) {
1183 : masm.cmpl_i32r(rhs.value, lhs.encoding());
1184 : return CodeOffset(masm.currentOffset());
1185 : }
1186 : void cmpw(Register rhs, Register lhs) {
1187 : masm.cmpw_rr(rhs.encoding(), lhs.encoding());
1188 : }
1189 186 : void setCC(Condition cond, Register r) {
1190 186 : masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding());
1191 186 : }
1192 796 : void testb(Register rhs, Register lhs) {
1193 796 : MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs));
1194 796 : MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs));
1195 796 : masm.testb_rr(rhs.encoding(), lhs.encoding());
1196 796 : }
1197 : void testw(Register rhs, Register lhs) {
1198 : masm.testw_rr(lhs.encoding(), rhs.encoding());
1199 : }
1200 3248 : void testl(Register rhs, Register lhs) {
1201 3248 : masm.testl_rr(lhs.encoding(), rhs.encoding());
1202 3248 : }
1203 199 : void testl(Imm32 rhs, Register lhs) {
1204 199 : masm.testl_ir(rhs.value, lhs.encoding());
1205 199 : }
1206 7960 : void testl(Imm32 rhs, const Operand& lhs) {
1207 7960 : switch (lhs.kind()) {
1208 : case Operand::REG:
1209 0 : masm.testl_ir(rhs.value, lhs.reg());
1210 0 : break;
1211 : case Operand::MEM_REG_DISP:
1212 7960 : masm.testl_i32m(rhs.value, lhs.disp(), lhs.base());
1213 7960 : break;
1214 : case Operand::MEM_ADDRESS32:
1215 0 : masm.testl_i32m(rhs.value, lhs.address());
1216 0 : break;
1217 : default:
1218 0 : MOZ_CRASH("unexpected operand kind");
1219 : break;
1220 : }
1221 7960 : }
1222 :
1223 1155 : void addl(Imm32 imm, Register dest) {
1224 1155 : masm.addl_ir(imm.value, dest.encoding());
1225 1155 : }
1226 : CodeOffset addlWithPatch(Imm32 imm, Register dest) {
1227 : masm.addl_i32r(imm.value, dest.encoding());
1228 : return CodeOffset(masm.currentOffset());
1229 : }
1230 16 : void addl(Imm32 imm, const Operand& op) {
1231 16 : switch (op.kind()) {
1232 : case Operand::REG:
1233 8 : masm.addl_ir(imm.value, op.reg());
1234 8 : break;
1235 : case Operand::MEM_REG_DISP:
1236 8 : masm.addl_im(imm.value, op.disp(), op.base());
1237 8 : break;
1238 : case Operand::MEM_ADDRESS32:
1239 0 : masm.addl_im(imm.value, op.address());
1240 0 : break;
1241 : case Operand::MEM_SCALE:
1242 0 : masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1243 0 : break;
1244 : default:
1245 0 : MOZ_CRASH("unexpected operand kind");
1246 : }
1247 16 : }
1248 0 : void addw(Imm32 imm, const Operand& op) {
1249 0 : switch (op.kind()) {
1250 : case Operand::REG:
1251 0 : masm.addw_ir(imm.value, op.reg());
1252 0 : break;
1253 : case Operand::MEM_REG_DISP:
1254 0 : masm.addw_im(imm.value, op.disp(), op.base());
1255 0 : break;
1256 : case Operand::MEM_ADDRESS32:
1257 0 : masm.addw_im(imm.value, op.address());
1258 0 : break;
1259 : case Operand::MEM_SCALE:
1260 0 : masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1261 0 : break;
1262 : default:
1263 0 : MOZ_CRASH("unexpected operand kind");
1264 : }
1265 0 : }
1266 663 : void subl(Imm32 imm, Register dest) {
1267 663 : masm.subl_ir(imm.value, dest.encoding());
1268 663 : }
1269 0 : void subl(Imm32 imm, const Operand& op) {
1270 0 : switch (op.kind()) {
1271 : case Operand::REG:
1272 0 : masm.subl_ir(imm.value, op.reg());
1273 0 : break;
1274 : case Operand::MEM_REG_DISP:
1275 0 : masm.subl_im(imm.value, op.disp(), op.base());
1276 0 : break;
1277 : case Operand::MEM_SCALE:
1278 0 : masm.subl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1279 0 : break;
1280 : default:
1281 0 : MOZ_CRASH("unexpected operand kind");
1282 : }
1283 0 : }
1284 0 : void subw(Imm32 imm, const Operand& op) {
1285 0 : switch (op.kind()) {
1286 : case Operand::REG:
1287 0 : masm.subw_ir(imm.value, op.reg());
1288 0 : break;
1289 : case Operand::MEM_REG_DISP:
1290 0 : masm.subw_im(imm.value, op.disp(), op.base());
1291 0 : break;
1292 : case Operand::MEM_SCALE:
1293 0 : masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1294 0 : break;
1295 : default:
1296 0 : MOZ_CRASH("unexpected operand kind");
1297 : }
1298 0 : }
1299 47 : void addl(Register src, Register dest) {
1300 47 : masm.addl_rr(src.encoding(), dest.encoding());
1301 47 : }
1302 0 : void addl(Register src, const Operand& dest) {
1303 0 : switch (dest.kind()) {
1304 : case Operand::REG:
1305 0 : masm.addl_rr(src.encoding(), dest.reg());
1306 0 : break;
1307 : case Operand::MEM_REG_DISP:
1308 0 : masm.addl_rm(src.encoding(), dest.disp(), dest.base());
1309 0 : break;
1310 : case Operand::MEM_SCALE:
1311 0 : masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1312 0 : break;
1313 : default:
1314 0 : MOZ_CRASH("unexpected operand kind");
1315 : }
1316 0 : }
1317 0 : void addw(Register src, const Operand& dest) {
1318 0 : switch (dest.kind()) {
1319 : case Operand::REG:
1320 0 : masm.addw_rr(src.encoding(), dest.reg());
1321 0 : break;
1322 : case Operand::MEM_REG_DISP:
1323 0 : masm.addw_rm(src.encoding(), dest.disp(), dest.base());
1324 0 : break;
1325 : case Operand::MEM_SCALE:
1326 0 : masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1327 0 : break;
1328 : default:
1329 0 : MOZ_CRASH("unexpected operand kind");
1330 : }
1331 0 : }
1332 15 : void subl(Register src, Register dest) {
1333 15 : masm.subl_rr(src.encoding(), dest.encoding());
1334 15 : }
1335 0 : void subl(const Operand& src, Register dest) {
1336 0 : switch (src.kind()) {
1337 : case Operand::REG:
1338 0 : masm.subl_rr(src.reg(), dest.encoding());
1339 0 : break;
1340 : case Operand::MEM_REG_DISP:
1341 0 : masm.subl_mr(src.disp(), src.base(), dest.encoding());
1342 0 : break;
1343 : default:
1344 0 : MOZ_CRASH("unexpected operand kind");
1345 : }
1346 0 : }
1347 0 : void subl(Register src, const Operand& dest) {
1348 0 : switch (dest.kind()) {
1349 : case Operand::REG:
1350 0 : masm.subl_rr(src.encoding(), dest.reg());
1351 0 : break;
1352 : case Operand::MEM_REG_DISP:
1353 0 : masm.subl_rm(src.encoding(), dest.disp(), dest.base());
1354 0 : break;
1355 : case Operand::MEM_SCALE:
1356 0 : masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1357 0 : break;
1358 : default:
1359 0 : MOZ_CRASH("unexpected operand kind");
1360 : }
1361 0 : }
1362 0 : void subw(Register src, const Operand& dest) {
1363 0 : switch (dest.kind()) {
1364 : case Operand::REG:
1365 0 : masm.subw_rr(src.encoding(), dest.reg());
1366 0 : break;
1367 : case Operand::MEM_REG_DISP:
1368 0 : masm.subw_rm(src.encoding(), dest.disp(), dest.base());
1369 0 : break;
1370 : case Operand::MEM_SCALE:
1371 0 : masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1372 0 : break;
1373 : default:
1374 0 : MOZ_CRASH("unexpected operand kind");
1375 : }
1376 0 : }
1377 0 : void orl(Register reg, Register dest) {
1378 0 : masm.orl_rr(reg.encoding(), dest.encoding());
1379 0 : }
1380 0 : void orl(Register src, const Operand& dest) {
1381 0 : switch (dest.kind()) {
1382 : case Operand::REG:
1383 0 : masm.orl_rr(src.encoding(), dest.reg());
1384 0 : break;
1385 : case Operand::MEM_REG_DISP:
1386 0 : masm.orl_rm(src.encoding(), dest.disp(), dest.base());
1387 0 : break;
1388 : case Operand::MEM_SCALE:
1389 0 : masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1390 0 : break;
1391 : default:
1392 0 : MOZ_CRASH("unexpected operand kind");
1393 : }
1394 0 : }
1395 0 : void orw(Register src, const Operand& dest) {
1396 0 : switch (dest.kind()) {
1397 : case Operand::REG:
1398 0 : masm.orw_rr(src.encoding(), dest.reg());
1399 0 : break;
1400 : case Operand::MEM_REG_DISP:
1401 0 : masm.orw_rm(src.encoding(), dest.disp(), dest.base());
1402 0 : break;
1403 : case Operand::MEM_SCALE:
1404 0 : masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1405 0 : break;
1406 : default:
1407 0 : MOZ_CRASH("unexpected operand kind");
1408 : }
1409 0 : }
1410 0 : void orl(Imm32 imm, Register reg) {
1411 0 : masm.orl_ir(imm.value, reg.encoding());
1412 0 : }
1413 185 : void orl(Imm32 imm, const Operand& op) {
1414 185 : switch (op.kind()) {
1415 : case Operand::REG:
1416 0 : masm.orl_ir(imm.value, op.reg());
1417 0 : break;
1418 : case Operand::MEM_REG_DISP:
1419 185 : masm.orl_im(imm.value, op.disp(), op.base());
1420 185 : break;
1421 : case Operand::MEM_SCALE:
1422 0 : masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1423 0 : break;
1424 : default:
1425 0 : MOZ_CRASH("unexpected operand kind");
1426 : }
1427 185 : }
1428 0 : void orw(Imm32 imm, const Operand& op) {
1429 0 : switch (op.kind()) {
1430 : case Operand::REG:
1431 0 : masm.orw_ir(imm.value, op.reg());
1432 0 : break;
1433 : case Operand::MEM_REG_DISP:
1434 0 : masm.orw_im(imm.value, op.disp(), op.base());
1435 0 : break;
1436 : case Operand::MEM_SCALE:
1437 0 : masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1438 0 : break;
1439 : default:
1440 0 : MOZ_CRASH("unexpected operand kind");
1441 : }
1442 0 : }
1443 1199 : void xorl(Register src, Register dest) {
1444 1199 : masm.xorl_rr(src.encoding(), dest.encoding());
1445 1199 : }
1446 0 : void xorl(Register src, const Operand& dest) {
1447 0 : switch (dest.kind()) {
1448 : case Operand::REG:
1449 0 : masm.xorl_rr(src.encoding(), dest.reg());
1450 0 : break;
1451 : case Operand::MEM_REG_DISP:
1452 0 : masm.xorl_rm(src.encoding(), dest.disp(), dest.base());
1453 0 : break;
1454 : case Operand::MEM_SCALE:
1455 0 : masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1456 0 : break;
1457 : default:
1458 0 : MOZ_CRASH("unexpected operand kind");
1459 : }
1460 0 : }
1461 0 : void xorw(Register src, const Operand& dest) {
1462 0 : switch (dest.kind()) {
1463 : case Operand::REG:
1464 0 : masm.xorw_rr(src.encoding(), dest.reg());
1465 0 : break;
1466 : case Operand::MEM_REG_DISP:
1467 0 : masm.xorw_rm(src.encoding(), dest.disp(), dest.base());
1468 0 : break;
1469 : case Operand::MEM_SCALE:
1470 0 : masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1471 0 : break;
1472 : default:
1473 0 : MOZ_CRASH("unexpected operand kind");
1474 : }
1475 0 : }
1476 37 : void xorl(Imm32 imm, Register reg) {
1477 37 : masm.xorl_ir(imm.value, reg.encoding());
1478 37 : }
1479 0 : void xorl(Imm32 imm, const Operand& op) {
1480 0 : switch (op.kind()) {
1481 : case Operand::REG:
1482 0 : masm.xorl_ir(imm.value, op.reg());
1483 0 : break;
1484 : case Operand::MEM_REG_DISP:
1485 0 : masm.xorl_im(imm.value, op.disp(), op.base());
1486 0 : break;
1487 : case Operand::MEM_SCALE:
1488 0 : masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1489 0 : break;
1490 : default:
1491 0 : MOZ_CRASH("unexpected operand kind");
1492 : }
1493 0 : }
1494 0 : void xorw(Imm32 imm, const Operand& op) {
1495 0 : switch (op.kind()) {
1496 : case Operand::REG:
1497 0 : masm.xorw_ir(imm.value, op.reg());
1498 0 : break;
1499 : case Operand::MEM_REG_DISP:
1500 0 : masm.xorw_im(imm.value, op.disp(), op.base());
1501 0 : break;
1502 : case Operand::MEM_SCALE:
1503 0 : masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1504 0 : break;
1505 : default:
1506 0 : MOZ_CRASH("unexpected operand kind");
1507 : }
1508 0 : }
1509 164 : void andl(Register src, Register dest) {
1510 164 : masm.andl_rr(src.encoding(), dest.encoding());
1511 164 : }
1512 0 : void andl(Register src, const Operand& dest) {
1513 0 : switch (dest.kind()) {
1514 : case Operand::REG:
1515 0 : masm.andl_rr(src.encoding(), dest.reg());
1516 0 : break;
1517 : case Operand::MEM_REG_DISP:
1518 0 : masm.andl_rm(src.encoding(), dest.disp(), dest.base());
1519 0 : break;
1520 : case Operand::MEM_SCALE:
1521 0 : masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1522 0 : break;
1523 : default:
1524 0 : MOZ_CRASH("unexpected operand kind");
1525 : }
1526 0 : }
1527 0 : void andw(Register src, const Operand& dest) {
1528 0 : switch (dest.kind()) {
1529 : case Operand::REG:
1530 0 : masm.andw_rr(src.encoding(), dest.reg());
1531 0 : break;
1532 : case Operand::MEM_REG_DISP:
1533 0 : masm.andw_rm(src.encoding(), dest.disp(), dest.base());
1534 0 : break;
1535 : case Operand::MEM_SCALE:
1536 0 : masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
1537 0 : break;
1538 : default:
1539 0 : MOZ_CRASH("unexpected operand kind");
1540 : }
1541 0 : }
1542 204 : void andl(Imm32 imm, Register dest) {
1543 204 : masm.andl_ir(imm.value, dest.encoding());
1544 204 : }
1545 0 : void andl(Imm32 imm, const Operand& op) {
1546 0 : switch (op.kind()) {
1547 : case Operand::REG:
1548 0 : masm.andl_ir(imm.value, op.reg());
1549 0 : break;
1550 : case Operand::MEM_REG_DISP:
1551 0 : masm.andl_im(imm.value, op.disp(), op.base());
1552 0 : break;
1553 : case Operand::MEM_SCALE:
1554 0 : masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1555 0 : break;
1556 : default:
1557 0 : MOZ_CRASH("unexpected operand kind");
1558 : }
1559 0 : }
1560 0 : void andw(Imm32 imm, const Operand& op) {
1561 0 : switch (op.kind()) {
1562 : case Operand::REG:
1563 0 : masm.andw_ir(imm.value, op.reg());
1564 0 : break;
1565 : case Operand::MEM_REG_DISP:
1566 0 : masm.andw_im(imm.value, op.disp(), op.base());
1567 0 : break;
1568 : case Operand::MEM_SCALE:
1569 0 : masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1570 0 : break;
1571 : default:
1572 0 : MOZ_CRASH("unexpected operand kind");
1573 : }
1574 0 : }
1575 0 : void addl(const Operand& src, Register dest) {
1576 0 : switch (src.kind()) {
1577 : case Operand::REG:
1578 0 : masm.addl_rr(src.reg(), dest.encoding());
1579 0 : break;
1580 : case Operand::MEM_REG_DISP:
1581 0 : masm.addl_mr(src.disp(), src.base(), dest.encoding());
1582 0 : break;
1583 : default:
1584 0 : MOZ_CRASH("unexpected operand kind");
1585 : }
1586 0 : }
1587 0 : void orl(const Operand& src, Register dest) {
1588 0 : switch (src.kind()) {
1589 : case Operand::REG:
1590 0 : masm.orl_rr(src.reg(), dest.encoding());
1591 0 : break;
1592 : case Operand::MEM_REG_DISP:
1593 0 : masm.orl_mr(src.disp(), src.base(), dest.encoding());
1594 0 : break;
1595 : default:
1596 0 : MOZ_CRASH("unexpected operand kind");
1597 : }
1598 0 : }
1599 0 : void xorl(const Operand& src, Register dest) {
1600 0 : switch (src.kind()) {
1601 : case Operand::REG:
1602 0 : masm.xorl_rr(src.reg(), dest.encoding());
1603 0 : break;
1604 : case Operand::MEM_REG_DISP:
1605 0 : masm.xorl_mr(src.disp(), src.base(), dest.encoding());
1606 0 : break;
1607 : default:
1608 0 : MOZ_CRASH("unexpected operand kind");
1609 : }
1610 0 : }
1611 4 : void andl(const Operand& src, Register dest) {
1612 4 : switch (src.kind()) {
1613 : case Operand::REG:
1614 0 : masm.andl_rr(src.reg(), dest.encoding());
1615 0 : break;
1616 : case Operand::MEM_REG_DISP:
1617 4 : masm.andl_mr(src.disp(), src.base(), dest.encoding());
1618 4 : break;
1619 : default:
1620 0 : MOZ_CRASH("unexpected operand kind");
1621 : }
1622 4 : }
1623 0 : void bsrl(const Register& src, const Register& dest) {
1624 0 : masm.bsrl_rr(src.encoding(), dest.encoding());
1625 0 : }
1626 0 : void bsfl(const Register& src, const Register& dest) {
1627 0 : masm.bsfl_rr(src.encoding(), dest.encoding());
1628 0 : }
1629 0 : void popcntl(const Register& src, const Register& dest) {
1630 0 : masm.popcntl_rr(src.encoding(), dest.encoding());
1631 0 : }
1632 0 : void imull(Register multiplier) {
1633 0 : masm.imull_r(multiplier.encoding());
1634 0 : }
1635 0 : void umull(Register multiplier) {
1636 0 : masm.mull_r(multiplier.encoding());
1637 0 : }
1638 0 : void imull(Imm32 imm, Register dest) {
1639 0 : masm.imull_ir(imm.value, dest.encoding(), dest.encoding());
1640 0 : }
1641 0 : void imull(Register src, Register dest) {
1642 0 : masm.imull_rr(src.encoding(), dest.encoding());
1643 0 : }
1644 0 : void imull(Imm32 imm, Register src, Register dest) {
1645 0 : masm.imull_ir(imm.value, src.encoding(), dest.encoding());
1646 0 : }
1647 0 : void imull(const Operand& src, Register dest) {
1648 0 : switch (src.kind()) {
1649 : case Operand::REG:
1650 0 : masm.imull_rr(src.reg(), dest.encoding());
1651 0 : break;
1652 : case Operand::MEM_REG_DISP:
1653 0 : masm.imull_mr(src.disp(), src.base(), dest.encoding());
1654 0 : break;
1655 : default:
1656 0 : MOZ_CRASH("unexpected operand kind");
1657 : }
1658 0 : }
1659 0 : void negl(const Operand& src) {
1660 0 : switch (src.kind()) {
1661 : case Operand::REG:
1662 0 : masm.negl_r(src.reg());
1663 0 : break;
1664 : case Operand::MEM_REG_DISP:
1665 0 : masm.negl_m(src.disp(), src.base());
1666 0 : break;
1667 : default:
1668 0 : MOZ_CRASH("unexpected operand kind");
1669 : }
1670 0 : }
1671 0 : void negl(Register reg) {
1672 0 : masm.negl_r(reg.encoding());
1673 0 : }
1674 0 : void notl(const Operand& src) {
1675 0 : switch (src.kind()) {
1676 : case Operand::REG:
1677 0 : masm.notl_r(src.reg());
1678 0 : break;
1679 : case Operand::MEM_REG_DISP:
1680 0 : masm.notl_m(src.disp(), src.base());
1681 0 : break;
1682 : default:
1683 0 : MOZ_CRASH("unexpected operand kind");
1684 : }
1685 0 : }
1686 1 : void notl(Register reg) {
1687 1 : masm.notl_r(reg.encoding());
1688 1 : }
1689 2 : void shrl(const Imm32 imm, Register dest) {
1690 2 : masm.shrl_ir(imm.value, dest.encoding());
1691 2 : }
1692 12 : void shll(const Imm32 imm, Register dest) {
1693 12 : masm.shll_ir(imm.value, dest.encoding());
1694 12 : }
1695 0 : void sarl(const Imm32 imm, Register dest) {
1696 0 : masm.sarl_ir(imm.value, dest.encoding());
1697 0 : }
1698 0 : void shrl_cl(Register dest) {
1699 0 : masm.shrl_CLr(dest.encoding());
1700 0 : }
1701 0 : void shll_cl(Register dest) {
1702 0 : masm.shll_CLr(dest.encoding());
1703 0 : }
1704 0 : void sarl_cl(Register dest) {
1705 0 : masm.sarl_CLr(dest.encoding());
1706 0 : }
1707 : void shrdl_cl(Register src, Register dest) {
1708 : masm.shrdl_CLr(src.encoding(), dest.encoding());
1709 : }
1710 : void shldl_cl(Register src, Register dest) {
1711 : masm.shldl_CLr(src.encoding(), dest.encoding());
1712 : }
1713 :
1714 0 : void roll(const Imm32 imm, Register dest) {
1715 0 : masm.roll_ir(imm.value, dest.encoding());
1716 0 : }
1717 0 : void roll_cl(Register dest) {
1718 0 : masm.roll_CLr(dest.encoding());
1719 0 : }
1720 0 : void rorl(const Imm32 imm, Register dest) {
1721 0 : masm.rorl_ir(imm.value, dest.encoding());
1722 0 : }
1723 0 : void rorl_cl(Register dest) {
1724 0 : masm.rorl_CLr(dest.encoding());
1725 0 : }
1726 :
1727 : void incl(const Operand& op) {
1728 : switch (op.kind()) {
1729 : case Operand::MEM_REG_DISP:
1730 : masm.incl_m32(op.disp(), op.base());
1731 : break;
1732 : default:
1733 : MOZ_CRASH("unexpected operand kind");
1734 : }
1735 : }
1736 : void lock_incl(const Operand& op) {
1737 : masm.prefix_lock();
1738 : incl(op);
1739 : }
1740 :
1741 : void decl(const Operand& op) {
1742 : switch (op.kind()) {
1743 : case Operand::MEM_REG_DISP:
1744 : masm.decl_m32(op.disp(), op.base());
1745 : break;
1746 : default:
1747 : MOZ_CRASH("unexpected operand kind");
1748 : }
1749 : }
1750 : void lock_decl(const Operand& op) {
1751 : masm.prefix_lock();
1752 : decl(op);
1753 : }
1754 :
1755 0 : void addb(Imm32 imm, const Operand& op) {
1756 0 : switch (op.kind()) {
1757 : case Operand::MEM_REG_DISP:
1758 0 : masm.addb_im(imm.value, op.disp(), op.base());
1759 0 : break;
1760 : case Operand::MEM_SCALE:
1761 0 : masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1762 0 : break;
1763 : default:
1764 0 : MOZ_CRASH("unexpected operand kind");
1765 : break;
1766 : }
1767 0 : }
1768 0 : void addb(Register src, const Operand& op) {
1769 0 : switch (op.kind()) {
1770 : case Operand::MEM_REG_DISP:
1771 0 : masm.addb_rm(src.encoding(), op.disp(), op.base());
1772 0 : break;
1773 : case Operand::MEM_SCALE:
1774 0 : masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
1775 0 : break;
1776 : default:
1777 0 : MOZ_CRASH("unexpected operand kind");
1778 : break;
1779 : }
1780 0 : }
1781 :
1782 0 : void subb(Imm32 imm, const Operand& op) {
1783 0 : switch (op.kind()) {
1784 : case Operand::MEM_REG_DISP:
1785 0 : masm.subb_im(imm.value, op.disp(), op.base());
1786 0 : break;
1787 : case Operand::MEM_SCALE:
1788 0 : masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1789 0 : break;
1790 : default:
1791 0 : MOZ_CRASH("unexpected operand kind");
1792 : break;
1793 : }
1794 0 : }
1795 0 : void subb(Register src, const Operand& op) {
1796 0 : switch (op.kind()) {
1797 : case Operand::MEM_REG_DISP:
1798 0 : masm.subb_rm(src.encoding(), op.disp(), op.base());
1799 0 : break;
1800 : case Operand::MEM_SCALE:
1801 0 : masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
1802 0 : break;
1803 : default:
1804 0 : MOZ_CRASH("unexpected operand kind");
1805 : break;
1806 : }
1807 0 : }
1808 :
1809 0 : void andb(Imm32 imm, const Operand& op) {
1810 0 : switch (op.kind()) {
1811 : case Operand::MEM_REG_DISP:
1812 0 : masm.andb_im(imm.value, op.disp(), op.base());
1813 0 : break;
1814 : case Operand::MEM_SCALE:
1815 0 : masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1816 0 : break;
1817 : default:
1818 0 : MOZ_CRASH("unexpected operand kind");
1819 : break;
1820 : }
1821 0 : }
1822 0 : void andb(Register src, const Operand& op) {
1823 0 : switch (op.kind()) {
1824 : case Operand::MEM_REG_DISP:
1825 0 : masm.andb_rm(src.encoding(), op.disp(), op.base());
1826 0 : break;
1827 : case Operand::MEM_SCALE:
1828 0 : masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
1829 0 : break;
1830 : default:
1831 0 : MOZ_CRASH("unexpected operand kind");
1832 : break;
1833 : }
1834 0 : }
1835 :
1836 0 : void orb(Imm32 imm, const Operand& op) {
1837 0 : switch (op.kind()) {
1838 : case Operand::MEM_REG_DISP:
1839 0 : masm.orb_im(imm.value, op.disp(), op.base());
1840 0 : break;
1841 : case Operand::MEM_SCALE:
1842 0 : masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1843 0 : break;
1844 : default:
1845 0 : MOZ_CRASH("unexpected operand kind");
1846 : break;
1847 : }
1848 0 : }
1849 0 : void orb(Register src, const Operand& op) {
1850 0 : switch (op.kind()) {
1851 : case Operand::MEM_REG_DISP:
1852 0 : masm.orb_rm(src.encoding(), op.disp(), op.base());
1853 0 : break;
1854 : case Operand::MEM_SCALE:
1855 0 : masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
1856 0 : break;
1857 : default:
1858 0 : MOZ_CRASH("unexpected operand kind");
1859 : break;
1860 : }
1861 0 : }
1862 :
1863 0 : void xorb(Imm32 imm, const Operand& op) {
1864 0 : switch (op.kind()) {
1865 : case Operand::MEM_REG_DISP:
1866 0 : masm.xorb_im(imm.value, op.disp(), op.base());
1867 0 : break;
1868 : case Operand::MEM_SCALE:
1869 0 : masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
1870 0 : break;
1871 : default:
1872 0 : MOZ_CRASH("unexpected operand kind");
1873 : break;
1874 : }
1875 0 : }
1876 0 : void xorb(Register src, const Operand& op) {
1877 0 : switch (op.kind()) {
1878 : case Operand::MEM_REG_DISP:
1879 0 : masm.xorb_rm(src.encoding(), op.disp(), op.base());
1880 0 : break;
1881 : case Operand::MEM_SCALE:
1882 0 : masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
1883 0 : break;
1884 : default:
1885 0 : MOZ_CRASH("unexpected operand kind");
1886 : break;
1887 : }
1888 0 : }
1889 :
1890 : template<typename T>
1891 0 : void lock_addb(T src, const Operand& op) {
1892 0 : masm.prefix_lock();
1893 0 : addb(src, op);
1894 0 : }
1895 : template<typename T>
1896 0 : void lock_subb(T src, const Operand& op) {
1897 0 : masm.prefix_lock();
1898 0 : subb(src, op);
1899 0 : }
1900 : template<typename T>
1901 0 : void lock_andb(T src, const Operand& op) {
1902 0 : masm.prefix_lock();
1903 0 : andb(src, op);
1904 0 : }
1905 : template<typename T>
1906 0 : void lock_orb(T src, const Operand& op) {
1907 0 : masm.prefix_lock();
1908 0 : orb(src, op);
1909 0 : }
1910 : template<typename T>
1911 0 : void lock_xorb(T src, const Operand& op) {
1912 0 : masm.prefix_lock();
1913 0 : xorb(src, op);
1914 0 : }
1915 :
1916 : template<typename T>
1917 0 : void lock_addw(T src, const Operand& op) {
1918 0 : masm.prefix_lock();
1919 0 : addw(src, op);
1920 0 : }
1921 : template<typename T>
1922 0 : void lock_subw(T src, const Operand& op) {
1923 0 : masm.prefix_lock();
1924 0 : subw(src, op);
1925 0 : }
1926 : template<typename T>
1927 0 : void lock_andw(T src, const Operand& op) {
1928 0 : masm.prefix_lock();
1929 0 : andw(src, op);
1930 0 : }
1931 : template<typename T>
1932 0 : void lock_orw(T src, const Operand& op) {
1933 0 : masm.prefix_lock();
1934 0 : orw(src, op);
1935 0 : }
1936 : template<typename T>
1937 0 : void lock_xorw(T src, const Operand& op) {
1938 0 : masm.prefix_lock();
1939 0 : xorw(src, op);
1940 0 : }
1941 :
1942 : // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems,
1943 : // among other things. Do not optimize, replace by XADDL, or similar.
1944 : template<typename T>
1945 0 : void lock_addl(T src, const Operand& op) {
1946 0 : masm.prefix_lock();
1947 0 : addl(src, op);
1948 0 : }
1949 : template<typename T>
1950 0 : void lock_subl(T src, const Operand& op) {
1951 0 : masm.prefix_lock();
1952 0 : subl(src, op);
1953 0 : }
1954 : template<typename T>
1955 0 : void lock_andl(T src, const Operand& op) {
1956 0 : masm.prefix_lock();
1957 0 : andl(src, op);
1958 0 : }
1959 : template<typename T>
1960 0 : void lock_orl(T src, const Operand& op) {
1961 0 : masm.prefix_lock();
1962 0 : orl(src, op);
1963 0 : }
1964 : template<typename T>
1965 0 : void lock_xorl(T src, const Operand& op) {
1966 0 : masm.prefix_lock();
1967 0 : xorl(src, op);
1968 0 : }
1969 :
1970 0 : void lock_cmpxchgb(Register src, const Operand& mem) {
1971 0 : masm.prefix_lock();
1972 0 : switch (mem.kind()) {
1973 : case Operand::MEM_REG_DISP:
1974 0 : masm.cmpxchgb(src.encoding(), mem.disp(), mem.base());
1975 0 : break;
1976 : case Operand::MEM_SCALE:
1977 0 : masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
1978 0 : break;
1979 : default:
1980 0 : MOZ_CRASH("unexpected operand kind");
1981 : }
1982 0 : }
1983 0 : void lock_cmpxchgw(Register src, const Operand& mem) {
1984 0 : masm.prefix_lock();
1985 0 : switch (mem.kind()) {
1986 : case Operand::MEM_REG_DISP:
1987 0 : masm.cmpxchgw(src.encoding(), mem.disp(), mem.base());
1988 0 : break;
1989 : case Operand::MEM_SCALE:
1990 0 : masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
1991 0 : break;
1992 : default:
1993 0 : MOZ_CRASH("unexpected operand kind");
1994 : }
1995 0 : }
1996 0 : void lock_cmpxchgl(Register src, const Operand& mem) {
1997 0 : masm.prefix_lock();
1998 0 : switch (mem.kind()) {
1999 : case Operand::MEM_REG_DISP:
2000 0 : masm.cmpxchgl(src.encoding(), mem.disp(), mem.base());
2001 0 : break;
2002 : case Operand::MEM_SCALE:
2003 0 : masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
2004 0 : break;
2005 : default:
2006 0 : MOZ_CRASH("unexpected operand kind");
2007 : }
2008 0 : }
2009 :
2010 0 : void xchgb(Register src, const Operand& mem) {
2011 0 : switch (mem.kind()) {
2012 : case Operand::MEM_REG_DISP:
2013 0 : masm.xchgb_rm(src.encoding(), mem.disp(), mem.base());
2014 0 : break;
2015 : case Operand::MEM_SCALE:
2016 0 : masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
2017 0 : break;
2018 : default:
2019 0 : MOZ_CRASH("unexpected operand kind");
2020 : }
2021 0 : }
2022 0 : void xchgw(Register src, const Operand& mem) {
2023 0 : switch (mem.kind()) {
2024 : case Operand::MEM_REG_DISP:
2025 0 : masm.xchgw_rm(src.encoding(), mem.disp(), mem.base());
2026 0 : break;
2027 : case Operand::MEM_SCALE:
2028 0 : masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
2029 0 : break;
2030 : default:
2031 0 : MOZ_CRASH("unexpected operand kind");
2032 : }
2033 0 : }
2034 0 : void xchgl(Register src, const Operand& mem) {
2035 0 : switch (mem.kind()) {
2036 : case Operand::MEM_REG_DISP:
2037 0 : masm.xchgl_rm(src.encoding(), mem.disp(), mem.base());
2038 0 : break;
2039 : case Operand::MEM_SCALE:
2040 0 : masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
2041 0 : break;
2042 : default:
2043 0 : MOZ_CRASH("unexpected operand kind");
2044 : }
2045 0 : }
2046 :
2047 0 : void lock_xaddb(Register srcdest, const Operand& mem) {
2048 0 : switch (mem.kind()) {
2049 : case Operand::MEM_REG_DISP:
2050 0 : masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base());
2051 0 : break;
2052 : case Operand::MEM_SCALE:
2053 0 : masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
2054 0 : break;
2055 : default:
2056 0 : MOZ_CRASH("unexpected operand kind");
2057 : }
2058 0 : }
2059 0 : void lock_xaddw(Register srcdest, const Operand& mem) {
2060 0 : masm.prefix_16_for_32();
2061 0 : lock_xaddl(srcdest, mem);
2062 0 : }
2063 0 : void lock_xaddl(Register srcdest, const Operand& mem) {
2064 0 : switch (mem.kind()) {
2065 : case Operand::MEM_REG_DISP:
2066 0 : masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base());
2067 0 : break;
2068 : case Operand::MEM_SCALE:
2069 0 : masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
2070 0 : break;
2071 : default:
2072 0 : MOZ_CRASH("unexpected operand kind");
2073 : }
2074 0 : }
2075 :
2076 6210 : void push(const Imm32 imm) {
2077 6210 : masm.push_i(imm.value);
2078 6210 : }
2079 :
2080 4030 : void push(const Operand& src) {
2081 4030 : switch (src.kind()) {
2082 : case Operand::REG:
2083 0 : masm.push_r(src.reg());
2084 0 : break;
2085 : case Operand::MEM_REG_DISP:
2086 4030 : masm.push_m(src.disp(), src.base());
2087 4030 : break;
2088 : default:
2089 0 : MOZ_CRASH("unexpected operand kind");
2090 : }
2091 4030 : }
2092 143086 : void push(Register src) {
2093 143086 : masm.push_r(src.encoding());
2094 143090 : }
2095 24 : void push(const Address& src) {
2096 24 : masm.push_m(src.offset, src.base.encoding());
2097 24 : }
2098 :
2099 665 : void pop(const Operand& src) {
2100 665 : switch (src.kind()) {
2101 : case Operand::REG:
2102 0 : masm.pop_r(src.reg());
2103 0 : break;
2104 : case Operand::MEM_REG_DISP:
2105 665 : masm.pop_m(src.disp(), src.base());
2106 665 : break;
2107 : default:
2108 0 : MOZ_CRASH("unexpected operand kind");
2109 : }
2110 665 : }
2111 117823 : void pop(Register src) {
2112 117823 : masm.pop_r(src.encoding());
2113 117824 : }
2114 : void pop(const Address& src) {
2115 : masm.pop_m(src.offset, src.base.encoding());
2116 : }
2117 :
2118 0 : void pushFlags() {
2119 0 : masm.push_flags();
2120 0 : }
2121 0 : void popFlags() {
2122 0 : masm.pop_flags();
2123 0 : }
2124 :
2125 : #ifdef JS_CODEGEN_X86
2126 : void pushAllRegs() {
2127 : masm.pusha();
2128 : }
2129 : void popAllRegs() {
2130 : masm.popa();
2131 : }
2132 : #endif
2133 :
2134 : // Zero-extend byte to 32-bit integer.
2135 61 : void movzbl(Register src, Register dest) {
2136 61 : masm.movzbl_rr(src.encoding(), dest.encoding());
2137 61 : }
2138 :
2139 0 : void cdq() {
2140 0 : masm.cdq();
2141 0 : }
2142 0 : void idiv(Register divisor) {
2143 0 : masm.idivl_r(divisor.encoding());
2144 0 : }
2145 0 : void udiv(Register divisor) {
2146 0 : masm.divl_r(divisor.encoding());
2147 0 : }
2148 :
2149 0 : void vpinsrb(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) {
2150 0 : MOZ_ASSERT(HasSSE41());
2151 0 : masm.vpinsrb_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
2152 0 : }
2153 0 : void vpinsrw(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) {
2154 0 : masm.vpinsrw_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
2155 0 : }
2156 :
2157 0 : void vpinsrd(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) {
2158 0 : MOZ_ASSERT(HasSSE41());
2159 0 : masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
2160 0 : }
2161 :
2162 0 : void vpextrb(unsigned lane, FloatRegister src, Register dest) {
2163 0 : MOZ_ASSERT(HasSSE41());
2164 0 : masm.vpextrb_irr(lane, src.encoding(), dest.encoding());
2165 0 : }
2166 0 : void vpextrw(unsigned lane, FloatRegister src, Register dest) {
2167 0 : masm.vpextrw_irr(lane, src.encoding(), dest.encoding());
2168 0 : }
2169 0 : void vpextrd(unsigned lane, FloatRegister src, Register dest) {
2170 0 : MOZ_ASSERT(HasSSE41());
2171 0 : masm.vpextrd_irr(lane, src.encoding(), dest.encoding());
2172 0 : }
2173 : void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
2174 : MOZ_ASSERT(HasSSE2());
2175 : masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding());
2176 : }
2177 0 : void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
2178 0 : MOZ_ASSERT(HasSSE2());
2179 0 : masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding());
2180 0 : }
2181 : void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
2182 : MOZ_ASSERT(HasSSE2());
2183 : masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding());
2184 : }
2185 0 : void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2186 0 : MOZ_ASSERT(HasSSE2());
2187 0 : masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding());
2188 0 : }
2189 0 : void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) {
2190 0 : MOZ_ASSERT(HasSSE2());
2191 0 : masm.vpslld_ir(count.value, src0.encoding(), dest.encoding());
2192 0 : }
2193 0 : void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2194 0 : MOZ_ASSERT(HasSSE2());
2195 0 : masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding());
2196 0 : }
2197 0 : void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) {
2198 0 : MOZ_ASSERT(HasSSE2());
2199 0 : masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding());
2200 0 : }
2201 0 : void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2202 0 : MOZ_ASSERT(HasSSE2());
2203 0 : masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding());
2204 0 : }
2205 0 : void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) {
2206 0 : MOZ_ASSERT(HasSSE2());
2207 0 : masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding());
2208 0 : }
2209 :
2210 0 : void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2211 0 : MOZ_ASSERT(HasSSE2());
2212 0 : masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding());
2213 0 : }
2214 0 : void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) {
2215 0 : MOZ_ASSERT(HasSSE2());
2216 0 : masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding());
2217 0 : }
2218 0 : void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2219 0 : MOZ_ASSERT(HasSSE2());
2220 0 : masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding());
2221 0 : }
2222 0 : void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) {
2223 0 : MOZ_ASSERT(HasSSE2());
2224 0 : masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding());
2225 0 : }
2226 0 : void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2227 0 : MOZ_ASSERT(HasSSE2());
2228 0 : masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding());
2229 0 : }
2230 0 : void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) {
2231 0 : MOZ_ASSERT(HasSSE2());
2232 0 : masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding());
2233 0 : }
2234 :
2235 0 : void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2236 0 : MOZ_ASSERT(HasSSE2());
2237 0 : switch (src1.kind()) {
2238 : case Operand::REG:
2239 0 : masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding());
2240 0 : break;
2241 : case Operand::MEM_REG_DISP:
2242 0 : masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2243 0 : break;
2244 : case Operand::MEM_SCALE:
2245 0 : masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding());
2246 0 : break;
2247 : default:
2248 0 : MOZ_CRASH("unexpected operand kind");
2249 : }
2250 0 : }
2251 26 : void vcvttsd2si(FloatRegister src, Register dest) {
2252 26 : MOZ_ASSERT(HasSSE2());
2253 26 : masm.vcvttsd2si_rr(src.encoding(), dest.encoding());
2254 26 : }
2255 0 : void vcvttss2si(FloatRegister src, Register dest) {
2256 0 : MOZ_ASSERT(HasSSE2());
2257 0 : masm.vcvttss2si_rr(src.encoding(), dest.encoding());
2258 0 : }
2259 : void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2260 : MOZ_ASSERT(HasSSE2());
2261 : switch (src1.kind()) {
2262 : case Operand::REG:
2263 : masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding());
2264 : break;
2265 : case Operand::MEM_REG_DISP:
2266 : masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2267 : break;
2268 : case Operand::MEM_SCALE:
2269 : masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding());
2270 : break;
2271 : default:
2272 : MOZ_CRASH("unexpected operand kind");
2273 : }
2274 : }
2275 0 : void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) {
2276 0 : MOZ_ASSERT(HasSSE2());
2277 0 : masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
2278 0 : }
2279 58 : void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) {
2280 58 : MOZ_ASSERT(HasSSE2());
2281 58 : masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
2282 58 : }
2283 0 : void vcvttps2dq(FloatRegister src, FloatRegister dest) {
2284 0 : MOZ_ASSERT(HasSSE2());
2285 0 : masm.vcvttps2dq_rr(src.encoding(), dest.encoding());
2286 0 : }
2287 0 : void vcvtdq2ps(FloatRegister src, FloatRegister dest) {
2288 0 : MOZ_ASSERT(HasSSE2());
2289 0 : masm.vcvtdq2ps_rr(src.encoding(), dest.encoding());
2290 0 : }
2291 0 : void vmovmskpd(FloatRegister src, Register dest) {
2292 0 : MOZ_ASSERT(HasSSE2());
2293 0 : masm.vmovmskpd_rr(src.encoding(), dest.encoding());
2294 0 : }
2295 0 : void vmovmskps(FloatRegister src, Register dest) {
2296 0 : MOZ_ASSERT(HasSSE2());
2297 0 : masm.vmovmskps_rr(src.encoding(), dest.encoding());
2298 0 : }
2299 0 : void vpmovmskb(FloatRegister src, Register dest) {
2300 0 : MOZ_ASSERT(HasSSE2());
2301 0 : masm.vpmovmskb_rr(src.encoding(), dest.encoding());
2302 0 : }
2303 : void vptest(FloatRegister rhs, FloatRegister lhs) {
2304 : MOZ_ASSERT(HasSSE41());
2305 : masm.vptest_rr(rhs.encoding(), lhs.encoding());
2306 : }
2307 38 : void vucomisd(FloatRegister rhs, FloatRegister lhs) {
2308 38 : MOZ_ASSERT(HasSSE2());
2309 38 : masm.vucomisd_rr(rhs.encoding(), lhs.encoding());
2310 38 : }
2311 0 : void vucomiss(FloatRegister rhs, FloatRegister lhs) {
2312 0 : MOZ_ASSERT(HasSSE2());
2313 0 : masm.vucomiss_rr(rhs.encoding(), lhs.encoding());
2314 0 : }
2315 :
2316 0 : void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
2317 0 : MOZ_ASSERT(HasSSE2());
2318 0 : switch (rhs.kind()) {
2319 : case Operand::FPREG:
2320 0 : masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
2321 0 : break;
2322 : case Operand::MEM_REG_DISP:
2323 0 : masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
2324 0 : break;
2325 : case Operand::MEM_ADDRESS32:
2326 0 : masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding());
2327 0 : break;
2328 : default:
2329 0 : MOZ_CRASH("unexpected operand kind");
2330 : }
2331 0 : }
2332 0 : void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
2333 0 : MOZ_ASSERT(HasSSE2());
2334 0 : switch (rhs.kind()) {
2335 : case Operand::FPREG:
2336 0 : masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
2337 0 : break;
2338 : case Operand::MEM_REG_DISP:
2339 0 : masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
2340 0 : break;
2341 : case Operand::MEM_ADDRESS32:
2342 0 : masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding());
2343 0 : break;
2344 : default:
2345 0 : MOZ_CRASH("unexpected operand kind");
2346 : }
2347 0 : }
2348 :
2349 0 : void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
2350 0 : MOZ_ASSERT(HasSSE2());
2351 0 : switch (rhs.kind()) {
2352 : case Operand::FPREG:
2353 0 : masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
2354 0 : break;
2355 : case Operand::MEM_REG_DISP:
2356 0 : masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
2357 0 : break;
2358 : case Operand::MEM_ADDRESS32:
2359 0 : masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding());
2360 0 : break;
2361 : default:
2362 0 : MOZ_CRASH("unexpected operand kind");
2363 : }
2364 0 : }
2365 0 : void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
2366 0 : MOZ_ASSERT(HasSSE2());
2367 0 : switch (rhs.kind()) {
2368 : case Operand::FPREG:
2369 0 : masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
2370 0 : break;
2371 : case Operand::MEM_REG_DISP:
2372 0 : masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
2373 0 : break;
2374 : case Operand::MEM_ADDRESS32:
2375 0 : masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding());
2376 0 : break;
2377 : default:
2378 0 : MOZ_CRASH("unexpected operand kind");
2379 : }
2380 0 : }
2381 :
2382 0 : void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
2383 0 : MOZ_ASSERT(HasSSE2());
2384 0 : switch (rhs.kind()) {
2385 : case Operand::FPREG:
2386 0 : masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
2387 0 : break;
2388 : case Operand::MEM_REG_DISP:
2389 0 : masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
2390 0 : break;
2391 : case Operand::MEM_ADDRESS32:
2392 0 : masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding());
2393 0 : break;
2394 : default:
2395 0 : MOZ_CRASH("unexpected operand kind");
2396 : }
2397 0 : }
2398 0 : void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
2399 0 : MOZ_ASSERT(HasSSE2());
2400 0 : switch (rhs.kind()) {
2401 : case Operand::FPREG:
2402 0 : masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
2403 0 : break;
2404 : case Operand::MEM_REG_DISP:
2405 0 : masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
2406 0 : break;
2407 : case Operand::MEM_ADDRESS32:
2408 0 : masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding());
2409 0 : break;
2410 : default:
2411 0 : MOZ_CRASH("unexpected operand kind");
2412 : }
2413 0 : }
2414 :
2415 0 : void vcmpps(uint8_t order, Operand src1, FloatRegister src0, FloatRegister dest) {
2416 0 : MOZ_ASSERT(HasSSE2());
2417 : // :TODO: (Bug 1132894) See LIRGeneratorX86Shared::lowerForFPU
2418 : // FIXME: This logic belongs in the MacroAssembler.
2419 0 : if (!HasAVX() && !src0.aliases(dest)) {
2420 0 : if (src1.kind() == Operand::FPREG &&
2421 0 : dest.aliases(FloatRegister::FromCode(src1.fpu())))
2422 : {
2423 0 : vmovdqa(src1, ScratchSimd128Reg);
2424 0 : src1 = Operand(ScratchSimd128Reg);
2425 : }
2426 0 : vmovdqa(src0, dest);
2427 0 : src0 = dest;
2428 : }
2429 0 : switch (src1.kind()) {
2430 : case Operand::FPREG:
2431 0 : masm.vcmpps_rr(order, src1.fpu(), src0.encoding(), dest.encoding());
2432 0 : break;
2433 : case Operand::MEM_REG_DISP:
2434 0 : masm.vcmpps_mr(order, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2435 0 : break;
2436 : case Operand::MEM_ADDRESS32:
2437 0 : masm.vcmpps_mr(order, src1.address(), src0.encoding(), dest.encoding());
2438 0 : break;
2439 : default:
2440 0 : MOZ_CRASH("unexpected operand kind");
2441 : }
2442 0 : }
2443 0 : void vcmpeqps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2444 0 : vcmpps(X86Encoding::ConditionCmp_EQ, src1, src0, dest);
2445 0 : }
2446 0 : void vcmpltps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2447 0 : vcmpps(X86Encoding::ConditionCmp_LT, src1, src0, dest);
2448 0 : }
2449 0 : void vcmpleps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2450 0 : vcmpps(X86Encoding::ConditionCmp_LE, src1, src0, dest);
2451 0 : }
2452 0 : void vcmpunordps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2453 0 : vcmpps(X86Encoding::ConditionCmp_UNORD, src1, src0, dest);
2454 0 : }
2455 0 : void vcmpneqps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2456 0 : vcmpps(X86Encoding::ConditionCmp_NEQ, src1, src0, dest);
2457 0 : }
2458 : void vcmpordps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2459 : vcmpps(X86Encoding::ConditionCmp_ORD, src1, src0, dest);
2460 : }
2461 0 : void vrcpps(const Operand& src, FloatRegister dest) {
2462 0 : MOZ_ASSERT(HasSSE2());
2463 0 : switch (src.kind()) {
2464 : case Operand::FPREG:
2465 0 : masm.vrcpps_rr(src.fpu(), dest.encoding());
2466 0 : break;
2467 : case Operand::MEM_REG_DISP:
2468 0 : masm.vrcpps_mr(src.disp(), src.base(), dest.encoding());
2469 0 : break;
2470 : case Operand::MEM_ADDRESS32:
2471 0 : masm.vrcpps_mr(src.address(), dest.encoding());
2472 0 : break;
2473 : default:
2474 0 : MOZ_CRASH("unexpected operand kind");
2475 : }
2476 0 : }
2477 0 : void vsqrtps(const Operand& src, FloatRegister dest) {
2478 0 : MOZ_ASSERT(HasSSE2());
2479 0 : switch (src.kind()) {
2480 : case Operand::FPREG:
2481 0 : masm.vsqrtps_rr(src.fpu(), dest.encoding());
2482 0 : break;
2483 : case Operand::MEM_REG_DISP:
2484 0 : masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding());
2485 0 : break;
2486 : case Operand::MEM_ADDRESS32:
2487 0 : masm.vsqrtps_mr(src.address(), dest.encoding());
2488 0 : break;
2489 : default:
2490 0 : MOZ_CRASH("unexpected operand kind");
2491 : }
2492 0 : }
2493 0 : void vrsqrtps(const Operand& src, FloatRegister dest) {
2494 0 : MOZ_ASSERT(HasSSE2());
2495 0 : switch (src.kind()) {
2496 : case Operand::FPREG:
2497 0 : masm.vrsqrtps_rr(src.fpu(), dest.encoding());
2498 0 : break;
2499 : case Operand::MEM_REG_DISP:
2500 0 : masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding());
2501 0 : break;
2502 : case Operand::MEM_ADDRESS32:
2503 0 : masm.vrsqrtps_mr(src.address(), dest.encoding());
2504 0 : break;
2505 : default:
2506 0 : MOZ_CRASH("unexpected operand kind");
2507 : }
2508 0 : }
2509 0 : void vmovd(Register src, FloatRegister dest) {
2510 0 : MOZ_ASSERT(HasSSE2());
2511 0 : masm.vmovd_rr(src.encoding(), dest.encoding());
2512 0 : }
2513 0 : void vmovd(FloatRegister src, Register dest) {
2514 0 : MOZ_ASSERT(HasSSE2());
2515 0 : masm.vmovd_rr(src.encoding(), dest.encoding());
2516 0 : }
2517 0 : void vmovd(const Operand& src, FloatRegister dest) {
2518 0 : MOZ_ASSERT(HasSSE2());
2519 0 : switch (src.kind()) {
2520 : case Operand::MEM_REG_DISP:
2521 0 : masm.vmovd_mr(src.disp(), src.base(), dest.encoding());
2522 0 : break;
2523 : case Operand::MEM_SCALE:
2524 0 : masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
2525 0 : break;
2526 : default:
2527 0 : MOZ_CRASH("unexpected operand kind");
2528 : }
2529 0 : }
2530 0 : void vmovd(FloatRegister src, const Operand& dest) {
2531 0 : MOZ_ASSERT(HasSSE2());
2532 0 : switch (dest.kind()) {
2533 : case Operand::MEM_REG_DISP:
2534 0 : masm.vmovd_rm(src.encoding(), dest.disp(), dest.base());
2535 0 : break;
2536 : case Operand::MEM_SCALE:
2537 0 : masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
2538 0 : break;
2539 : case Operand::MEM_ADDRESS32:
2540 0 : masm.vmovq_rm(src.encoding(), dest.address());
2541 0 : break;
2542 : default:
2543 0 : MOZ_CRASH("unexpected operand kind");
2544 : }
2545 0 : }
2546 0 : void vmovq(const Operand& src, FloatRegister dest) {
2547 0 : MOZ_ASSERT(HasSSE2());
2548 0 : switch (src.kind()) {
2549 : case Operand::MEM_REG_DISP:
2550 0 : masm.vmovq_mr(src.disp(), src.base(), dest.encoding());
2551 0 : break;
2552 : case Operand::MEM_SCALE:
2553 0 : masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
2554 0 : break;
2555 : case Operand::MEM_ADDRESS32:
2556 0 : masm.vmovq_mr(src.address(), dest.encoding());
2557 0 : break;
2558 : default:
2559 0 : MOZ_CRASH("unexpected operand kind");
2560 : }
2561 0 : }
2562 0 : void vmovq(FloatRegister src, const Operand& dest) {
2563 0 : MOZ_ASSERT(HasSSE2());
2564 0 : switch (dest.kind()) {
2565 : case Operand::MEM_REG_DISP:
2566 0 : masm.vmovq_rm(src.encoding(), dest.disp(), dest.base());
2567 0 : break;
2568 : case Operand::MEM_SCALE:
2569 0 : masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
2570 0 : break;
2571 : default:
2572 0 : MOZ_CRASH("unexpected operand kind");
2573 : }
2574 0 : }
2575 0 : void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2576 0 : MOZ_ASSERT(HasSSE2());
2577 0 : switch (src1.kind()) {
2578 : case Operand::FPREG:
2579 0 : masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding());
2580 0 : break;
2581 : case Operand::MEM_REG_DISP:
2582 0 : masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2583 0 : break;
2584 : case Operand::MEM_ADDRESS32:
2585 0 : masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding());
2586 0 : break;
2587 : default:
2588 0 : MOZ_CRASH("unexpected operand kind");
2589 : }
2590 0 : }
2591 0 : void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2592 0 : MOZ_ASSERT(HasSSE2());
2593 0 : switch (src1.kind()) {
2594 : case Operand::FPREG:
2595 0 : masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding());
2596 0 : break;
2597 : case Operand::MEM_REG_DISP:
2598 0 : masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2599 0 : break;
2600 : case Operand::MEM_ADDRESS32:
2601 0 : masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding());
2602 0 : break;
2603 : default:
2604 0 : MOZ_CRASH("unexpected operand kind");
2605 : }
2606 0 : }
2607 0 : void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2608 0 : MOZ_ASSERT(HasSSE2());
2609 0 : switch (src1.kind()) {
2610 : case Operand::FPREG:
2611 0 : masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
2612 0 : break;
2613 : case Operand::MEM_REG_DISP:
2614 0 : masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2615 0 : break;
2616 : case Operand::MEM_ADDRESS32:
2617 0 : masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding());
2618 0 : break;
2619 : default:
2620 0 : MOZ_CRASH("unexpected operand kind");
2621 : }
2622 0 : }
2623 0 : void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2624 0 : MOZ_ASSERT(HasSSE2());
2625 0 : switch (src1.kind()) {
2626 : case Operand::FPREG:
2627 0 : masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
2628 0 : break;
2629 : case Operand::MEM_REG_DISP:
2630 0 : masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2631 0 : break;
2632 : case Operand::MEM_ADDRESS32:
2633 0 : masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding());
2634 0 : break;
2635 : default:
2636 0 : MOZ_CRASH("unexpected operand kind");
2637 : }
2638 0 : }
2639 0 : void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2640 0 : MOZ_ASSERT(HasSSE2());
2641 0 : switch (src1.kind()) {
2642 : case Operand::FPREG:
2643 0 : masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
2644 0 : break;
2645 : case Operand::MEM_REG_DISP:
2646 0 : masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2647 0 : break;
2648 : case Operand::MEM_ADDRESS32:
2649 0 : masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding());
2650 0 : break;
2651 : default:
2652 0 : MOZ_CRASH("unexpected operand kind");
2653 : }
2654 0 : }
2655 0 : void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2656 0 : MOZ_ASSERT(HasSSE2());
2657 0 : switch (src1.kind()) {
2658 : case Operand::FPREG:
2659 0 : masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
2660 0 : break;
2661 : case Operand::MEM_REG_DISP:
2662 0 : masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2663 0 : break;
2664 : case Operand::MEM_ADDRESS32:
2665 0 : masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding());
2666 0 : break;
2667 : default:
2668 0 : MOZ_CRASH("unexpected operand kind");
2669 : }
2670 0 : }
2671 0 : void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2672 0 : MOZ_ASSERT(HasSSE2());
2673 0 : switch (src1.kind()) {
2674 : case Operand::FPREG:
2675 0 : masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2676 0 : break;
2677 : case Operand::MEM_REG_DISP:
2678 0 : masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2679 0 : break;
2680 : case Operand::MEM_ADDRESS32:
2681 0 : masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding());
2682 0 : break;
2683 : default:
2684 0 : MOZ_CRASH("unexpected operand kind");
2685 : }
2686 0 : }
2687 0 : void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2688 0 : MOZ_ASSERT(HasSSE2());
2689 0 : switch (src1.kind()) {
2690 : case Operand::FPREG:
2691 0 : masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2692 0 : break;
2693 : case Operand::MEM_REG_DISP:
2694 0 : masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2695 0 : break;
2696 : case Operand::MEM_ADDRESS32:
2697 0 : masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding());
2698 0 : break;
2699 : default:
2700 0 : MOZ_CRASH("unexpected operand kind");
2701 : }
2702 0 : }
2703 0 : void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2704 0 : MOZ_ASSERT(HasSSE2());
2705 0 : switch (src1.kind()) {
2706 : case Operand::FPREG:
2707 0 : masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2708 0 : break;
2709 : case Operand::MEM_REG_DISP:
2710 0 : masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2711 0 : break;
2712 : case Operand::MEM_ADDRESS32:
2713 0 : masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding());
2714 0 : break;
2715 : default:
2716 0 : MOZ_CRASH("unexpected operand kind");
2717 : }
2718 0 : }
2719 0 : void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2720 0 : MOZ_ASSERT(HasSSE2());
2721 0 : switch (src1.kind()) {
2722 : case Operand::FPREG:
2723 0 : masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2724 0 : break;
2725 : case Operand::MEM_REG_DISP:
2726 0 : masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2727 0 : break;
2728 : case Operand::MEM_ADDRESS32:
2729 0 : masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding());
2730 0 : break;
2731 : default:
2732 0 : MOZ_CRASH("unexpected operand kind");
2733 : }
2734 0 : }
2735 0 : void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2736 0 : MOZ_ASSERT(HasSSE2());
2737 0 : switch (src1.kind()) {
2738 : case Operand::FPREG:
2739 0 : masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2740 0 : break;
2741 : case Operand::MEM_REG_DISP:
2742 0 : masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2743 0 : break;
2744 : case Operand::MEM_ADDRESS32:
2745 0 : masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding());
2746 0 : break;
2747 : default:
2748 0 : MOZ_CRASH("unexpected operand kind");
2749 : }
2750 0 : }
2751 0 : void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2752 0 : MOZ_ASSERT(HasSSE2());
2753 0 : switch (src1.kind()) {
2754 : case Operand::FPREG:
2755 0 : masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2756 0 : break;
2757 : case Operand::MEM_REG_DISP:
2758 0 : masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2759 0 : break;
2760 : case Operand::MEM_ADDRESS32:
2761 0 : masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding());
2762 0 : break;
2763 : default:
2764 0 : MOZ_CRASH("unexpected operand kind");
2765 : }
2766 0 : }
2767 0 : void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2768 0 : MOZ_ASSERT(HasSSE2());
2769 0 : switch (src1.kind()) {
2770 : case Operand::FPREG:
2771 0 : masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding());
2772 0 : break;
2773 : case Operand::MEM_REG_DISP:
2774 0 : masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2775 0 : break;
2776 : case Operand::MEM_ADDRESS32:
2777 0 : masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding());
2778 0 : break;
2779 : default:
2780 0 : MOZ_CRASH("unexpected operand kind");
2781 : }
2782 0 : }
2783 0 : void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2784 0 : MOZ_ASSERT(HasSSE2());
2785 0 : switch (src1.kind()) {
2786 : case Operand::FPREG:
2787 0 : masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding());
2788 0 : break;
2789 : case Operand::MEM_REG_DISP:
2790 0 : masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2791 0 : break;
2792 : case Operand::MEM_ADDRESS32:
2793 0 : masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding());
2794 0 : break;
2795 : default:
2796 0 : MOZ_CRASH("unexpected operand kind");
2797 : }
2798 0 : }
2799 0 : void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
2800 0 : MOZ_ASSERT(HasSSE2());
2801 0 : masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding());
2802 0 : }
2803 : void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2804 : MOZ_ASSERT(HasSSE2());
2805 : switch (src1.kind()) {
2806 : case Operand::FPREG:
2807 : masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding());
2808 : break;
2809 : case Operand::MEM_REG_DISP:
2810 : masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2811 : break;
2812 : default:
2813 : MOZ_CRASH("unexpected operand kind");
2814 : }
2815 : }
2816 0 : void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2817 0 : MOZ_ASSERT(HasSSE2());
2818 0 : switch (src1.kind()) {
2819 : case Operand::FPREG:
2820 0 : masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding());
2821 0 : break;
2822 : case Operand::MEM_REG_DISP:
2823 0 : masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2824 0 : break;
2825 : default:
2826 0 : MOZ_CRASH("unexpected operand kind");
2827 : }
2828 0 : }
2829 0 : void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2830 0 : MOZ_ASSERT(HasSSE41());
2831 0 : switch (src1.kind()) {
2832 : case Operand::FPREG:
2833 0 : masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding());
2834 0 : break;
2835 : case Operand::MEM_REG_DISP:
2836 0 : masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2837 0 : break;
2838 : case Operand::MEM_ADDRESS32:
2839 0 : masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding());
2840 0 : break;
2841 : default:
2842 0 : MOZ_CRASH("unexpected operand kind");
2843 : }
2844 0 : }
2845 0 : void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2846 0 : MOZ_ASSERT(HasSSE2());
2847 0 : switch (src1.kind()) {
2848 : case Operand::FPREG:
2849 0 : masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2850 0 : break;
2851 : case Operand::MEM_REG_DISP:
2852 0 : masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2853 0 : break;
2854 : case Operand::MEM_ADDRESS32:
2855 0 : masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding());
2856 0 : break;
2857 : default:
2858 0 : MOZ_CRASH("unexpected operand kind");
2859 : }
2860 0 : }
2861 0 : void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2862 0 : MOZ_ASSERT(HasSSE2());
2863 0 : switch (src1.kind()) {
2864 : case Operand::FPREG:
2865 0 : masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2866 0 : break;
2867 : case Operand::MEM_REG_DISP:
2868 0 : masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2869 0 : break;
2870 : case Operand::MEM_ADDRESS32:
2871 0 : masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding());
2872 0 : break;
2873 : default:
2874 0 : MOZ_CRASH("unexpected operand kind");
2875 : }
2876 0 : }
2877 0 : void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2878 0 : MOZ_ASSERT(HasSSE2());
2879 0 : switch (src1.kind()) {
2880 : case Operand::FPREG:
2881 0 : masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2882 0 : break;
2883 : case Operand::MEM_REG_DISP:
2884 0 : masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2885 0 : break;
2886 : case Operand::MEM_ADDRESS32:
2887 0 : masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding());
2888 0 : break;
2889 : default:
2890 0 : MOZ_CRASH("unexpected operand kind");
2891 : }
2892 0 : }
2893 0 : void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2894 0 : MOZ_ASSERT(HasSSE2());
2895 0 : switch (src1.kind()) {
2896 : case Operand::FPREG:
2897 0 : masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2898 0 : break;
2899 : case Operand::MEM_REG_DISP:
2900 0 : masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2901 0 : break;
2902 : case Operand::MEM_ADDRESS32:
2903 0 : masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding());
2904 0 : break;
2905 : default:
2906 0 : MOZ_CRASH("unexpected operand kind");
2907 : }
2908 0 : }
2909 0 : void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2910 0 : MOZ_ASSERT(HasSSE2());
2911 0 : switch (src1.kind()) {
2912 : case Operand::FPREG:
2913 0 : masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2914 0 : break;
2915 : case Operand::MEM_REG_DISP:
2916 0 : masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2917 0 : break;
2918 : case Operand::MEM_ADDRESS32:
2919 0 : masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding());
2920 0 : break;
2921 : default:
2922 0 : MOZ_CRASH("unexpected operand kind");
2923 : }
2924 0 : }
2925 0 : void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2926 0 : MOZ_ASSERT(HasSSE2());
2927 0 : switch (src1.kind()) {
2928 : case Operand::FPREG:
2929 0 : masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2930 0 : break;
2931 : case Operand::MEM_REG_DISP:
2932 0 : masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2933 0 : break;
2934 : case Operand::MEM_ADDRESS32:
2935 0 : masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding());
2936 0 : break;
2937 : default:
2938 0 : MOZ_CRASH("unexpected operand kind");
2939 : }
2940 0 : }
2941 0 : void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2942 0 : MOZ_ASSERT(HasSSE2());
2943 0 : switch (src1.kind()) {
2944 : case Operand::FPREG:
2945 0 : masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2946 0 : break;
2947 : case Operand::MEM_REG_DISP:
2948 0 : masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2949 0 : break;
2950 : case Operand::MEM_ADDRESS32:
2951 0 : masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding());
2952 0 : break;
2953 : default:
2954 0 : MOZ_CRASH("unexpected operand kind");
2955 : }
2956 0 : }
2957 0 : void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2958 : // Negates bits of dest and then applies AND
2959 0 : MOZ_ASSERT(HasSSE2());
2960 0 : switch (src1.kind()) {
2961 : case Operand::FPREG:
2962 0 : masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2963 0 : break;
2964 : case Operand::MEM_REG_DISP:
2965 0 : masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2966 0 : break;
2967 : case Operand::MEM_ADDRESS32:
2968 0 : masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding());
2969 0 : break;
2970 : default:
2971 0 : MOZ_CRASH("unexpected operand kind");
2972 : }
2973 0 : }
2974 0 : void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2975 0 : MOZ_ASSERT(HasSSE2());
2976 0 : switch (src1.kind()) {
2977 : case Operand::FPREG:
2978 0 : masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2979 0 : break;
2980 : case Operand::MEM_REG_DISP:
2981 0 : masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2982 0 : break;
2983 : case Operand::MEM_ADDRESS32:
2984 0 : masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding());
2985 0 : break;
2986 : default:
2987 0 : MOZ_CRASH("unexpected operand kind");
2988 : }
2989 0 : }
2990 0 : void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
2991 0 : MOZ_ASSERT(HasSSE2());
2992 0 : switch (src1.kind()) {
2993 : case Operand::FPREG:
2994 0 : masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
2995 0 : break;
2996 : case Operand::MEM_REG_DISP:
2997 0 : masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
2998 0 : break;
2999 : case Operand::MEM_ADDRESS32:
3000 0 : masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding());
3001 0 : break;
3002 : default:
3003 0 : MOZ_CRASH("unexpected operand kind");
3004 : }
3005 0 : }
3006 : void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3007 : MOZ_ASSERT(HasSSE2());
3008 : masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding());
3009 : }
3010 0 : void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3011 0 : MOZ_ASSERT(HasSSE2());
3012 0 : switch (src1.kind()) {
3013 : case Operand::FPREG:
3014 0 : masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding());
3015 0 : break;
3016 : case Operand::MEM_REG_DISP:
3017 0 : masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3018 0 : break;
3019 : case Operand::MEM_ADDRESS32:
3020 0 : masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding());
3021 0 : break;
3022 : default:
3023 0 : MOZ_CRASH("unexpected operand kind");
3024 : }
3025 0 : }
3026 0 : void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3027 0 : MOZ_ASSERT(HasSSE2());
3028 0 : masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding());
3029 0 : }
3030 0 : void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3031 0 : MOZ_ASSERT(HasSSE2());
3032 0 : switch (src1.kind()) {
3033 : case Operand::FPREG:
3034 0 : masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding());
3035 0 : break;
3036 : case Operand::MEM_REG_DISP:
3037 0 : masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3038 0 : break;
3039 : case Operand::MEM_ADDRESS32:
3040 0 : masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding());
3041 0 : break;
3042 : default:
3043 0 : MOZ_CRASH("unexpected operand kind");
3044 : }
3045 0 : }
3046 0 : void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3047 0 : MOZ_ASSERT(HasSSE2());
3048 0 : masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding());
3049 0 : }
3050 0 : void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3051 0 : MOZ_ASSERT(HasSSE2());
3052 0 : switch (src1.kind()) {
3053 : case Operand::FPREG:
3054 0 : masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding());
3055 0 : break;
3056 : case Operand::MEM_REG_DISP:
3057 0 : masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3058 0 : break;
3059 : case Operand::MEM_ADDRESS32:
3060 0 : masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding());
3061 0 : break;
3062 : default:
3063 0 : MOZ_CRASH("unexpected operand kind");
3064 : }
3065 0 : }
3066 : void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3067 : MOZ_ASSERT(HasSSE2());
3068 : masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding());
3069 : }
3070 : void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3071 : MOZ_ASSERT(HasSSE2());
3072 : switch (src1.kind()) {
3073 : case Operand::FPREG:
3074 : masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding());
3075 : break;
3076 : case Operand::MEM_REG_DISP:
3077 : masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3078 : break;
3079 : case Operand::MEM_ADDRESS32:
3080 : masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding());
3081 : break;
3082 : default:
3083 : MOZ_CRASH("unexpected operand kind");
3084 : }
3085 : }
3086 :
3087 0 : void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) {
3088 0 : MOZ_ASSERT(HasSSE2());
3089 0 : masm.vpshufd_irr(mask, src.encoding(), dest.encoding());
3090 0 : }
3091 0 : void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) {
3092 0 : MOZ_ASSERT(HasSSE2());
3093 0 : switch (src1.kind()) {
3094 : case Operand::FPREG:
3095 0 : masm.vpshufd_irr(mask, src1.fpu(), dest.encoding());
3096 0 : break;
3097 : case Operand::MEM_REG_DISP:
3098 0 : masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding());
3099 0 : break;
3100 : case Operand::MEM_ADDRESS32:
3101 0 : masm.vpshufd_imr(mask, src1.address(), dest.encoding());
3102 0 : break;
3103 : default:
3104 0 : MOZ_CRASH("unexpected operand kind");
3105 : }
3106 0 : }
3107 :
3108 0 : void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) {
3109 0 : MOZ_ASSERT(HasSSE2());
3110 0 : masm.vpshuflw_irr(mask, src.encoding(), dest.encoding());
3111 0 : }
3112 : void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) {
3113 : MOZ_ASSERT(HasSSE2());
3114 : masm.vpshufhw_irr(mask, src.encoding(), dest.encoding());
3115 : }
3116 0 : void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) {
3117 0 : MOZ_ASSERT(HasSSSE3());
3118 0 : masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding());
3119 0 : }
3120 0 : void vmovddup(FloatRegister src, FloatRegister dest) {
3121 0 : MOZ_ASSERT(HasSSE3());
3122 0 : masm.vmovddup_rr(src.encoding(), dest.encoding());
3123 0 : }
3124 0 : void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3125 0 : MOZ_ASSERT(HasSSE2());
3126 0 : masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3127 0 : }
3128 0 : void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3129 0 : MOZ_ASSERT(HasSSE2());
3130 0 : masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3131 0 : }
3132 0 : void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3133 0 : MOZ_ASSERT(HasSSE2());
3134 0 : masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3135 0 : }
3136 0 : void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3137 0 : MOZ_ASSERT(HasSSE2());
3138 0 : switch (src1.kind()) {
3139 : case Operand::FPREG:
3140 0 : masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding());
3141 0 : break;
3142 : case Operand::MEM_REG_DISP:
3143 0 : masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3144 0 : break;
3145 : case Operand::MEM_ADDRESS32:
3146 0 : masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding());
3147 0 : break;
3148 : default:
3149 0 : MOZ_CRASH("unexpected operand kind");
3150 : }
3151 0 : }
3152 0 : void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3153 0 : MOZ_ASSERT(HasSSE2());
3154 0 : masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3155 0 : }
3156 0 : void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3157 0 : MOZ_ASSERT(HasSSE2());
3158 0 : switch (src1.kind()) {
3159 : case Operand::FPREG:
3160 0 : masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding());
3161 0 : break;
3162 : case Operand::MEM_REG_DISP:
3163 0 : masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3164 0 : break;
3165 : case Operand::MEM_ADDRESS32:
3166 0 : masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding());
3167 0 : break;
3168 : default:
3169 0 : MOZ_CRASH("unexpected operand kind");
3170 : }
3171 0 : }
3172 0 : void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3173 0 : MOZ_ASSERT(HasSSE2());
3174 0 : masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
3175 0 : }
3176 0 : void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
3177 0 : MOZ_ASSERT(HasSSE2());
3178 0 : switch (src1.kind()) {
3179 : case Operand::FPREG:
3180 0 : masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
3181 0 : break;
3182 : case Operand::MEM_REG_DISP:
3183 0 : masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3184 0 : break;
3185 : case Operand::MEM_ADDRESS32:
3186 0 : masm.vshufps_imr(mask, src1.address(), src0.encoding(), dest.encoding());
3187 0 : break;
3188 : default:
3189 0 : MOZ_CRASH("unexpected operand kind");
3190 : }
3191 0 : }
3192 0 : void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3193 0 : MOZ_ASSERT(HasSSE2());
3194 0 : masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3195 0 : }
3196 0 : void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3197 0 : MOZ_ASSERT(HasSSE2());
3198 0 : masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3199 0 : }
3200 0 : void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3201 0 : MOZ_ASSERT(HasSSE2());
3202 0 : switch (src1.kind()) {
3203 : case Operand::FPREG:
3204 0 : masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
3205 0 : break;
3206 : case Operand::MEM_REG_DISP:
3207 0 : masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3208 0 : break;
3209 : case Operand::MEM_ADDRESS32:
3210 0 : masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding());
3211 0 : break;
3212 : default:
3213 0 : MOZ_CRASH("unexpected operand kind");
3214 : }
3215 0 : }
3216 0 : void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3217 0 : MOZ_ASSERT(HasSSE2());
3218 0 : switch (src1.kind()) {
3219 : case Operand::FPREG:
3220 0 : masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding());
3221 0 : break;
3222 : case Operand::MEM_REG_DISP:
3223 0 : masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3224 0 : break;
3225 : case Operand::MEM_ADDRESS32:
3226 0 : masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding());
3227 0 : break;
3228 : default:
3229 0 : MOZ_CRASH("unexpected operand kind");
3230 : }
3231 0 : }
3232 0 : void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3233 0 : MOZ_ASSERT(HasSSE2());
3234 0 : masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3235 0 : }
3236 0 : void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3237 0 : MOZ_ASSERT(HasSSE2());
3238 0 : masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3239 0 : }
3240 0 : void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3241 0 : MOZ_ASSERT(HasSSE2());
3242 0 : switch (src1.kind()) {
3243 : case Operand::FPREG:
3244 0 : masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
3245 0 : break;
3246 : case Operand::MEM_REG_DISP:
3247 0 : masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3248 0 : break;
3249 : default:
3250 0 : MOZ_CRASH("unexpected operand kind");
3251 : }
3252 0 : }
3253 0 : void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3254 0 : MOZ_ASSERT(HasSSE2());
3255 0 : switch (src1.kind()) {
3256 : case Operand::FPREG:
3257 0 : masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding());
3258 0 : break;
3259 : case Operand::MEM_REG_DISP:
3260 0 : masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3261 0 : break;
3262 : default:
3263 0 : MOZ_CRASH("unexpected operand kind");
3264 : }
3265 0 : }
3266 0 : void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3267 0 : MOZ_ASSERT(HasSSE2());
3268 0 : masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3269 0 : }
3270 0 : void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3271 0 : MOZ_ASSERT(HasSSE2());
3272 0 : switch (src1.kind()) {
3273 : case Operand::FPREG:
3274 0 : masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
3275 0 : break;
3276 : case Operand::MEM_REG_DISP:
3277 0 : masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3278 0 : break;
3279 : default:
3280 0 : MOZ_CRASH("unexpected operand kind");
3281 : }
3282 0 : }
3283 0 : void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3284 0 : MOZ_ASSERT(HasSSE2());
3285 0 : switch (src1.kind()) {
3286 : case Operand::FPREG:
3287 0 : masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding());
3288 0 : break;
3289 : case Operand::MEM_REG_DISP:
3290 0 : masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3291 0 : break;
3292 : default:
3293 0 : MOZ_CRASH("unexpected operand kind");
3294 : }
3295 0 : }
3296 0 : void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3297 0 : MOZ_ASSERT(HasSSE2());
3298 0 : masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3299 0 : }
3300 1 : void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3301 1 : MOZ_ASSERT(HasSSE2());
3302 1 : masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3303 1 : }
3304 0 : void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3305 0 : MOZ_ASSERT(HasSSE2());
3306 0 : masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3307 0 : }
3308 0 : void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3309 0 : MOZ_ASSERT(HasSSE2());
3310 0 : switch (src1.kind()) {
3311 : case Operand::FPREG:
3312 0 : masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
3313 0 : break;
3314 : case Operand::MEM_REG_DISP:
3315 0 : masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3316 0 : break;
3317 : default:
3318 0 : MOZ_CRASH("unexpected operand kind");
3319 : }
3320 0 : }
3321 0 : void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3322 0 : MOZ_ASSERT(HasSSE2());
3323 0 : switch (src1.kind()) {
3324 : case Operand::FPREG:
3325 0 : masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding());
3326 0 : break;
3327 : case Operand::MEM_REG_DISP:
3328 0 : masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3329 0 : break;
3330 : default:
3331 0 : MOZ_CRASH("unexpected operand kind");
3332 : }
3333 0 : }
3334 59 : void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3335 59 : MOZ_ASSERT(HasSSE2());
3336 59 : masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3337 59 : }
3338 0 : void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3339 0 : MOZ_ASSERT(HasSSE2());
3340 0 : masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3341 0 : }
3342 0 : void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3343 0 : MOZ_ASSERT(HasSSE2());
3344 0 : masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3345 0 : }
3346 0 : void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3347 0 : MOZ_ASSERT(HasSSE2());
3348 0 : masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3349 0 : }
3350 0 : void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3351 0 : MOZ_ASSERT(HasSSE2());
3352 0 : masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3353 0 : }
3354 0 : void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3355 0 : MOZ_ASSERT(HasSSE2());
3356 0 : masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding());
3357 0 : }
3358 0 : void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3359 0 : MOZ_ASSERT(HasSSE2());
3360 0 : masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3361 0 : }
3362 0 : void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3363 0 : MOZ_ASSERT(HasSSE2());
3364 0 : masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3365 0 : }
3366 :
3367 : static X86Encoding::RoundingMode
3368 0 : ToX86RoundingMode(RoundingMode mode) {
3369 0 : switch (mode) {
3370 : case RoundingMode::Up:
3371 0 : return X86Encoding::RoundUp;
3372 : case RoundingMode::Down:
3373 0 : return X86Encoding::RoundDown;
3374 : case RoundingMode::NearestTiesToEven:
3375 0 : return X86Encoding::RoundToNearest;
3376 : case RoundingMode::TowardsZero:
3377 0 : return X86Encoding::RoundToZero;
3378 : }
3379 0 : MOZ_CRASH("unexpected mode");
3380 : }
3381 0 : void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3382 0 : MOZ_ASSERT(HasSSE41());
3383 0 : masm.vroundsd_irr(mode, src1.encoding(), src0.encoding(), dest.encoding());
3384 0 : }
3385 0 : void vroundss(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3386 0 : MOZ_ASSERT(HasSSE41());
3387 0 : masm.vroundss_irr(mode, src1.encoding(), src0.encoding(), dest.encoding());
3388 0 : }
3389 :
3390 0 : unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane, unsigned zeroMask = 0)
3391 : {
3392 : // Note that the sourceLane bits are ignored in the case of a source
3393 : // memory operand, and the source is the given 32-bits memory location.
3394 0 : MOZ_ASSERT(zeroMask < 16);
3395 0 : unsigned ret = zeroMask ;
3396 0 : ret |= destLane << 4;
3397 0 : ret |= sourceLane << 6;
3398 0 : MOZ_ASSERT(ret < 256);
3399 0 : return ret;
3400 : }
3401 0 : void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3402 0 : MOZ_ASSERT(HasSSE41());
3403 0 : masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
3404 0 : }
3405 0 : void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
3406 0 : MOZ_ASSERT(HasSSE41());
3407 0 : switch (src1.kind()) {
3408 : case Operand::FPREG:
3409 0 : masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
3410 0 : break;
3411 : case Operand::MEM_REG_DISP:
3412 0 : masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3413 0 : break;
3414 : default:
3415 0 : MOZ_CRASH("unexpected operand kind");
3416 : }
3417 0 : }
3418 0 : unsigned blendpsMask(bool x, bool y, bool z, bool w) {
3419 0 : return (x << 0) | (y << 1) | (z << 2) | (w << 3);
3420 : }
3421 : void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3422 : MOZ_ASSERT(HasSSE41());
3423 : masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
3424 : }
3425 0 : void vblendps(unsigned mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
3426 0 : MOZ_ASSERT(HasSSE41());
3427 0 : switch (src1.kind()) {
3428 : case Operand::FPREG:
3429 0 : masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
3430 0 : break;
3431 : case Operand::MEM_REG_DISP:
3432 0 : masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3433 0 : break;
3434 : default:
3435 0 : MOZ_CRASH("unexpected operand kind");
3436 : }
3437 0 : }
3438 0 : void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3439 0 : MOZ_ASSERT(HasSSE41());
3440 0 : masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(), dest.encoding());
3441 0 : }
3442 : void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
3443 : MOZ_ASSERT(HasSSE41());
3444 : switch (src1.kind()) {
3445 : case Operand::FPREG:
3446 : masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(), dest.encoding());
3447 : break;
3448 : case Operand::MEM_REG_DISP:
3449 : masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3450 : break;
3451 : default:
3452 : MOZ_CRASH("unexpected operand kind");
3453 : }
3454 : }
3455 0 : void vmovsldup(FloatRegister src, FloatRegister dest) {
3456 0 : MOZ_ASSERT(HasSSE3());
3457 0 : masm.vmovsldup_rr(src.encoding(), dest.encoding());
3458 0 : }
3459 : void vmovsldup(const Operand& src, FloatRegister dest) {
3460 : MOZ_ASSERT(HasSSE3());
3461 : switch (src.kind()) {
3462 : case Operand::FPREG:
3463 : masm.vmovsldup_rr(src.fpu(), dest.encoding());
3464 : break;
3465 : case Operand::MEM_REG_DISP:
3466 : masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding());
3467 : break;
3468 : default:
3469 : MOZ_CRASH("unexpected operand kind");
3470 : }
3471 : }
3472 0 : void vmovshdup(FloatRegister src, FloatRegister dest) {
3473 0 : MOZ_ASSERT(HasSSE3());
3474 0 : masm.vmovshdup_rr(src.encoding(), dest.encoding());
3475 0 : }
3476 : void vmovshdup(const Operand& src, FloatRegister dest) {
3477 : MOZ_ASSERT(HasSSE3());
3478 : switch (src.kind()) {
3479 : case Operand::FPREG:
3480 : masm.vmovshdup_rr(src.fpu(), dest.encoding());
3481 : break;
3482 : case Operand::MEM_REG_DISP:
3483 : masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding());
3484 : break;
3485 : default:
3486 : MOZ_CRASH("unexpected operand kind");
3487 : }
3488 : }
3489 0 : void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3490 0 : MOZ_ASSERT(HasSSE2());
3491 0 : masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3492 0 : }
3493 : void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3494 : MOZ_ASSERT(HasSSE2());
3495 : switch (src1.kind()) {
3496 : case Operand::FPREG:
3497 : masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
3498 : break;
3499 : case Operand::MEM_REG_DISP:
3500 : masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3501 : break;
3502 : default:
3503 : MOZ_CRASH("unexpected operand kind");
3504 : }
3505 : }
3506 0 : void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3507 0 : MOZ_ASSERT(HasSSE2());
3508 0 : masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3509 0 : }
3510 0 : void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3511 0 : MOZ_ASSERT(HasSSE2());
3512 0 : masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
3513 0 : }
3514 : void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
3515 : MOZ_ASSERT(HasSSE2());
3516 : switch (src1.kind()) {
3517 : case Operand::FPREG:
3518 : masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
3519 : break;
3520 : case Operand::MEM_REG_DISP:
3521 : masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
3522 : break;
3523 : default:
3524 : MOZ_CRASH("unexpected operand kind");
3525 : }
3526 : }
3527 0 : void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
3528 0 : MOZ_ASSERT(HasSSE2());
3529 0 : masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding());
3530 0 : }
3531 : void fisttp(const Operand& dest) {
3532 : MOZ_ASSERT(HasSSE3());
3533 : switch (dest.kind()) {
3534 : case Operand::MEM_REG_DISP:
3535 : masm.fisttp_m(dest.disp(), dest.base());
3536 : break;
3537 : default:
3538 : MOZ_CRASH("unexpected operand kind");
3539 : }
3540 : }
3541 : void fistp(const Operand& dest) {
3542 : switch (dest.kind()) {
3543 : case Operand::MEM_REG_DISP:
3544 : masm.fistp_m(dest.disp(), dest.base());
3545 : break;
3546 : default:
3547 : MOZ_CRASH("unexpected operand kind");
3548 : }
3549 : }
3550 : void fnstcw(const Operand& dest) {
3551 : switch (dest.kind()) {
3552 : case Operand::MEM_REG_DISP:
3553 : masm.fnstcw_m(dest.disp(), dest.base());
3554 : break;
3555 : default:
3556 : MOZ_CRASH("unexpected operand kind");
3557 : }
3558 : }
3559 : void fldcw(const Operand& dest) {
3560 : switch (dest.kind()) {
3561 : case Operand::MEM_REG_DISP:
3562 : masm.fldcw_m(dest.disp(), dest.base());
3563 : break;
3564 : default:
3565 : MOZ_CRASH("unexpected operand kind");
3566 : }
3567 : }
3568 : void fnstsw(const Operand& dest) {
3569 : switch (dest.kind()) {
3570 : case Operand::MEM_REG_DISP:
3571 : masm.fnstsw_m(dest.disp(), dest.base());
3572 : break;
3573 : default:
3574 : MOZ_CRASH("unexpected operand kind");
3575 : }
3576 : }
3577 : void fld(const Operand& dest) {
3578 : switch (dest.kind()) {
3579 : case Operand::MEM_REG_DISP:
3580 : masm.fld_m(dest.disp(), dest.base());
3581 : break;
3582 : default:
3583 : MOZ_CRASH("unexpected operand kind");
3584 : }
3585 : }
3586 : void fld32(const Operand& dest) {
3587 : switch (dest.kind()) {
3588 : case Operand::MEM_REG_DISP:
3589 : masm.fld32_m(dest.disp(), dest.base());
3590 : break;
3591 : default:
3592 : MOZ_CRASH("unexpected operand kind");
3593 : }
3594 : }
3595 : void fstp(const Operand& src) {
3596 : switch (src.kind()) {
3597 : case Operand::MEM_REG_DISP:
3598 : masm.fstp_m(src.disp(), src.base());
3599 : break;
3600 : default:
3601 : MOZ_CRASH("unexpected operand kind");
3602 : }
3603 : }
3604 : void fstp32(const Operand& src) {
3605 : switch (src.kind()) {
3606 : case Operand::MEM_REG_DISP:
3607 : masm.fstp32_m(src.disp(), src.base());
3608 : break;
3609 : default:
3610 : MOZ_CRASH("unexpected operand kind");
3611 : }
3612 : }
3613 :
3614 : // Defined for compatibility with ARM's assembler
3615 21 : uint32_t actualIndex(uint32_t x) {
3616 21 : return x;
3617 : }
3618 :
3619 19 : void flushBuffer() {
3620 19 : }
3621 :
3622 : // Patching.
3623 :
3624 9508 : static size_t PatchWrite_NearCallSize() {
3625 9508 : return 5;
3626 : }
3627 0 : static uintptr_t GetPointer(uint8_t* instPtr) {
3628 0 : uintptr_t* ptr = ((uintptr_t*) instPtr) - 1;
3629 0 : return *ptr;
3630 : }
3631 : // Write a relative call at the start location |dataLabel|.
3632 : // Note that this DOES NOT patch data that comes before |label|.
3633 0 : static void PatchWrite_NearCall(CodeLocationLabel startLabel, CodeLocationLabel target) {
3634 0 : uint8_t* start = startLabel.raw();
3635 0 : *start = 0xE8;
3636 0 : ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize();
3637 0 : MOZ_ASSERT(int32_t(offset) == offset);
3638 0 : *((int32_t*) (start + 1)) = offset;
3639 0 : }
3640 :
3641 0 : static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) {
3642 0 : *((int32_t*) dataLabel.raw() - 1) = toWrite.value;
3643 0 : }
3644 :
3645 20756 : static void PatchDataWithValueCheck(CodeLocationLabel data, PatchedImmPtr newData,
3646 : PatchedImmPtr expectedData) {
3647 : // The pointer given is a pointer to *after* the data.
3648 20756 : uintptr_t* ptr = ((uintptr_t*) data.raw()) - 1;
3649 20756 : MOZ_ASSERT(*ptr == (uintptr_t)expectedData.value);
3650 20756 : *ptr = (uintptr_t)newData.value;
3651 20756 : }
3652 20756 : static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData, ImmPtr expectedData) {
3653 20756 : PatchDataWithValueCheck(data, PatchedImmPtr(newData.value), PatchedImmPtr(expectedData.value));
3654 20756 : }
3655 :
3656 0 : static void PatchInstructionImmediate(uint8_t* code, PatchedImmPtr imm) {
3657 0 : MOZ_CRASH("Unused.");
3658 : }
3659 :
3660 64 : static uint32_t NopSize() {
3661 64 : return 1;
3662 : }
3663 : static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) {
3664 : MOZ_CRASH("nextInstruction NYI on x86");
3665 : }
3666 :
3667 : // Toggle a jmp or cmp emitted by toggledJump().
3668 0 : static void ToggleToJmp(CodeLocationLabel inst) {
3669 0 : uint8_t* ptr = (uint8_t*)inst.raw();
3670 0 : MOZ_ASSERT(*ptr == 0x3D);
3671 0 : *ptr = 0xE9;
3672 0 : }
3673 0 : static void ToggleToCmp(CodeLocationLabel inst) {
3674 0 : uint8_t* ptr = (uint8_t*)inst.raw();
3675 0 : MOZ_ASSERT(*ptr == 0xE9);
3676 0 : *ptr = 0x3D;
3677 0 : }
3678 0 : static void ToggleCall(CodeLocationLabel inst, bool enabled) {
3679 0 : uint8_t* ptr = (uint8_t*)inst.raw();
3680 0 : MOZ_ASSERT(*ptr == 0x3D || // CMP
3681 : *ptr == 0xE8); // CALL
3682 0 : *ptr = enabled ? 0xE8 : 0x3D;
3683 0 : }
3684 :
3685 : MOZ_COLD void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
3686 : const Disassembler::HeapAccess& heapAccess);
3687 : };
3688 :
3689 : } // namespace jit
3690 : } // namespace js
3691 :
3692 : #endif /* jit_x86_shared_Assembler_x86_shared_h */
|
