Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sts=4 et sw=4 tw=99:
3 : * This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "jit/Safepoints.h"
8 :
9 : #include "mozilla/MathAlgorithms.h"
10 : #include "mozilla/SizePrintfMacros.h"
11 :
12 : #include "jit/BitSet.h"
13 : #include "jit/JitSpewer.h"
14 : #include "jit/LIR.h"
15 :
16 : using namespace js;
17 : using namespace jit;
18 :
19 : using mozilla::FloorLog2;
20 :
21 8 : SafepointWriter::SafepointWriter(uint32_t slotCount, uint32_t argumentCount)
22 8 : : frameSlots_((slotCount / sizeof(intptr_t)) + 1), // Stack slot counts are inclusive.
23 16 : argumentSlots_(argumentCount / sizeof(intptr_t))
24 8 : { }
25 :
26 : bool
27 8 : SafepointWriter::init(TempAllocator& alloc)
28 : {
29 8 : return frameSlots_.init(alloc) && argumentSlots_.init(alloc);
30 : }
31 :
32 : uint32_t
33 39 : SafepointWriter::startEntry()
34 : {
35 39 : JitSpew(JitSpew_Safepoints, "Encoding safepoint (position %" PRIuSIZE "):", stream_.length());
36 39 : return uint32_t(stream_.length());
37 : }
38 :
39 : void
40 39 : SafepointWriter::writeOsiCallPointOffset(uint32_t osiCallPointOffset)
41 : {
42 39 : stream_.writeUnsigned(osiCallPointOffset);
43 39 : }
44 :
45 : static void
46 117 : WriteRegisterMask(CompactBufferWriter& stream, uint32_t bits)
47 : {
48 : if (sizeof(PackedRegisterMask) == 1)
49 : stream.writeByte(bits);
50 : else
51 117 : stream.writeUnsigned(bits);
52 117 : }
53 :
54 : static int32_t
55 1680 : ReadRegisterMask(CompactBufferReader& stream)
56 : {
57 : if (sizeof(PackedRegisterMask) == 1)
58 : return stream.readByte();
59 1680 : return stream.readUnsigned();
60 : }
61 :
62 : static void
63 39 : WriteFloatRegisterMask(CompactBufferWriter& stream, uint64_t bits)
64 : {
65 : if (sizeof(FloatRegisters::SetType) == 1) {
66 : stream.writeByte(bits);
67 : } else if (sizeof(FloatRegisters::SetType) == 4) {
68 : stream.writeUnsigned(bits);
69 : } else {
70 : MOZ_ASSERT(sizeof(FloatRegisters::SetType) == 8);
71 39 : stream.writeUnsigned(bits & 0xffffffff);
72 39 : stream.writeUnsigned(bits >> 32);
73 : }
74 39 : }
75 :
76 : static int64_t
77 1680 : ReadFloatRegisterMask(CompactBufferReader& stream)
78 : {
79 : if (sizeof(FloatRegisters::SetType) == 1)
80 : return stream.readByte();
81 : if (sizeof(FloatRegisters::SetType) <= 4)
82 : return stream.readUnsigned();
83 : MOZ_ASSERT(sizeof(FloatRegisters::SetType) == 8);
84 1680 : uint64_t ret = stream.readUnsigned();
85 1680 : ret |= uint64_t(stream.readUnsigned()) << 32;
86 1680 : return ret;
87 : }
88 :
89 : void
90 39 : SafepointWriter::writeGcRegs(LSafepoint* safepoint)
91 : {
92 39 : LiveGeneralRegisterSet gc(safepoint->gcRegs());
93 39 : LiveGeneralRegisterSet spilledGpr(safepoint->liveRegs().gprs());
94 39 : LiveFloatRegisterSet spilledFloat(safepoint->liveRegs().fpus());
95 39 : LiveGeneralRegisterSet slots(safepoint->slotsOrElementsRegs());
96 39 : LiveGeneralRegisterSet valueRegs;
97 :
98 39 : WriteRegisterMask(stream_, spilledGpr.bits());
99 39 : if (!spilledGpr.empty()) {
100 26 : WriteRegisterMask(stream_, gc.bits());
101 26 : WriteRegisterMask(stream_, slots.bits());
102 :
103 : #ifdef JS_PUNBOX64
104 26 : valueRegs = safepoint->valueRegs();
105 26 : WriteRegisterMask(stream_, valueRegs.bits());
106 : #endif
107 : }
108 :
109 : // GC registers are a subset of the spilled registers.
110 39 : MOZ_ASSERT((valueRegs.bits() & ~spilledGpr.bits()) == 0);
111 39 : MOZ_ASSERT((gc.bits() & ~spilledGpr.bits()) == 0);
112 :
113 39 : WriteFloatRegisterMask(stream_, spilledFloat.bits());
114 :
115 : #ifdef JS_JITSPEW
116 39 : if (JitSpewEnabled(JitSpew_Safepoints)) {
117 0 : for (GeneralRegisterForwardIterator iter(spilledGpr); iter.more(); ++iter) {
118 0 : const char* type = gc.has(*iter)
119 0 : ? "gc"
120 0 : : slots.has(*iter)
121 0 : ? "slots"
122 0 : : valueRegs.has(*iter)
123 0 : ? "value"
124 0 : : "any";
125 0 : JitSpew(JitSpew_Safepoints, " %s reg: %s", type, (*iter).name());
126 : }
127 0 : for (FloatRegisterForwardIterator iter(spilledFloat); iter.more(); ++iter)
128 0 : JitSpew(JitSpew_Safepoints, " float reg: %s", (*iter).name());
129 : }
130 : #endif
131 39 : }
132 :
133 : static void
134 156 : WriteBitset(const BitSet& set, CompactBufferWriter& stream)
135 : {
136 156 : size_t count = set.rawLength();
137 156 : const uint32_t* words = set.raw();
138 312 : for (size_t i = 0; i < count; i++)
139 156 : stream.writeUnsigned(words[i]);
140 156 : }
141 :
142 : static void
143 78 : MapSlotsToBitset(BitSet& stackSet, BitSet& argumentSet,
144 : CompactBufferWriter& stream, const LSafepoint::SlotList& slots)
145 : {
146 78 : stackSet.clear();
147 78 : argumentSet.clear();
148 :
149 263 : for (uint32_t i = 0; i < slots.length(); i++) {
150 : // Slots are represented at a distance from |fp|. We divide by the
151 : // pointer size, since we only care about pointer-sized/aligned slots
152 : // here.
153 185 : MOZ_ASSERT(slots[i].slot % sizeof(intptr_t) == 0);
154 185 : size_t index = slots[i].slot / sizeof(intptr_t);
155 185 : (slots[i].stack ? stackSet : argumentSet).insert(index);
156 : }
157 :
158 78 : WriteBitset(stackSet, stream);
159 78 : WriteBitset(argumentSet, stream);
160 78 : }
161 :
162 : void
163 39 : SafepointWriter::writeGcSlots(LSafepoint* safepoint)
164 : {
165 39 : LSafepoint::SlotList& slots = safepoint->gcSlots();
166 :
167 : #ifdef JS_JITSPEW
168 151 : for (uint32_t i = 0; i < slots.length(); i++)
169 112 : JitSpew(JitSpew_Safepoints, " gc slot: %u", slots[i].slot);
170 : #endif
171 :
172 39 : MapSlotsToBitset(frameSlots_, argumentSlots_, stream_, slots);
173 39 : }
174 :
175 : void
176 39 : SafepointWriter::writeSlotsOrElementsSlots(LSafepoint* safepoint)
177 : {
178 39 : LSafepoint::SlotList& slots = safepoint->slotsOrElementsSlots();
179 :
180 39 : stream_.writeUnsigned(slots.length());
181 :
182 39 : for (uint32_t i = 0; i < slots.length(); i++) {
183 0 : if (!slots[i].stack)
184 0 : MOZ_CRASH();
185 : #ifdef JS_JITSPEW
186 0 : JitSpew(JitSpew_Safepoints, " slots/elements slot: %d", slots[i].slot);
187 : #endif
188 0 : stream_.writeUnsigned(slots[i].slot);
189 : }
190 39 : }
191 :
192 : void
193 39 : SafepointWriter::writeValueSlots(LSafepoint* safepoint)
194 : {
195 39 : LSafepoint::SlotList& slots = safepoint->valueSlots();
196 :
197 : #ifdef JS_JITSPEW
198 112 : for (uint32_t i = 0; i < slots.length(); i++)
199 73 : JitSpew(JitSpew_Safepoints, " gc value: %u", slots[i].slot);
200 : #endif
201 :
202 39 : MapSlotsToBitset(frameSlots_, argumentSlots_, stream_, slots);
203 39 : }
204 :
205 : #if defined(JS_JITSPEW) && defined(JS_NUNBOX32)
206 : static void
207 : DumpNunboxPart(const LAllocation& a)
208 : {
209 : Fprinter& out = JitSpewPrinter();
210 : if (a.isStackSlot()) {
211 : out.printf("stack %d", a.toStackSlot()->slot());
212 : } else if (a.isArgument()) {
213 : out.printf("arg %d", a.toArgument()->index());
214 : } else {
215 : out.printf("reg %s", a.toGeneralReg()->reg().name());
216 : }
217 : }
218 : #endif // DEBUG
219 :
220 : // Nunbox part encoding:
221 : //
222 : // Reg = 000
223 : // Stack = 001
224 : // Arg = 010
225 : //
226 : // [vwu] nentries:
227 : // uint16_t: tttp ppXX XXXY YYYY
228 : //
229 : // If ttt = Reg, type is reg XXXXX
230 : // If ppp = Reg, payload is reg YYYYY
231 : //
232 : // If ttt != Reg, type is:
233 : // XXXXX if not 11111, otherwise followed by [vwu]
234 : // If ppp != Reg, payload is:
235 : // YYYYY if not 11111, otherwise followed by [vwu]
236 : //
237 : enum NunboxPartKind {
238 : Part_Reg,
239 : Part_Stack,
240 : Part_Arg
241 : };
242 :
243 : static const uint32_t PART_KIND_BITS = 3;
244 : static const uint32_t PART_KIND_MASK = (1 << PART_KIND_BITS) - 1;
245 : static const uint32_t PART_INFO_BITS = 5;
246 : static const uint32_t PART_INFO_MASK = (1 << PART_INFO_BITS) - 1;
247 :
248 : static const uint32_t MAX_INFO_VALUE = (1 << PART_INFO_BITS) - 1;
249 : static const uint32_t TYPE_KIND_SHIFT = 16 - PART_KIND_BITS;
250 : static const uint32_t PAYLOAD_KIND_SHIFT = TYPE_KIND_SHIFT - PART_KIND_BITS;
251 : static const uint32_t TYPE_INFO_SHIFT = PAYLOAD_KIND_SHIFT - PART_INFO_BITS;
252 : static const uint32_t PAYLOAD_INFO_SHIFT = TYPE_INFO_SHIFT - PART_INFO_BITS;
253 :
254 : JS_STATIC_ASSERT(PAYLOAD_INFO_SHIFT == 0);
255 :
256 : #ifdef JS_NUNBOX32
257 : static inline NunboxPartKind
258 : AllocationToPartKind(const LAllocation& a)
259 : {
260 : if (a.isRegister())
261 : return Part_Reg;
262 : if (a.isStackSlot())
263 : return Part_Stack;
264 : MOZ_ASSERT(a.isArgument());
265 : return Part_Arg;
266 : }
267 :
268 : // gcc 4.5 doesn't actually inline CanEncodeInfoInHeader when only
269 : // using the "inline" keyword, and miscompiles the function as well
270 : // when doing block reordering with branch prediction information.
271 : // See bug 799295 comment 71.
272 : static MOZ_ALWAYS_INLINE bool
273 : CanEncodeInfoInHeader(const LAllocation& a, uint32_t* out)
274 : {
275 : if (a.isGeneralReg()) {
276 : *out = a.toGeneralReg()->reg().code();
277 : return true;
278 : }
279 :
280 : if (a.isStackSlot())
281 : *out = a.toStackSlot()->slot();
282 : else
283 : *out = a.toArgument()->index();
284 :
285 : return *out < MAX_INFO_VALUE;
286 : }
287 :
288 : void
289 : SafepointWriter::writeNunboxParts(LSafepoint* safepoint)
290 : {
291 : LSafepoint::NunboxList& entries = safepoint->nunboxParts();
292 :
293 : # ifdef JS_JITSPEW
294 : if (JitSpewEnabled(JitSpew_Safepoints)) {
295 : for (uint32_t i = 0; i < entries.length(); i++) {
296 : SafepointNunboxEntry& entry = entries[i];
297 : if (entry.type.isUse() || entry.payload.isUse())
298 : continue;
299 : JitSpewHeader(JitSpew_Safepoints);
300 : Fprinter& out = JitSpewPrinter();
301 : out.printf(" nunbox (type in ");
302 : DumpNunboxPart(entry.type);
303 : out.printf(", payload in ");
304 : DumpNunboxPart(entry.payload);
305 : out.printf(")\n");
306 : }
307 : }
308 : # endif
309 :
310 : // Safepoints are permitted to have partially filled in entries for nunboxes,
311 : // provided that only the type is live and not the payload. Omit these from
312 : // the written safepoint.
313 :
314 : size_t pos = stream_.length();
315 : stream_.writeUnsigned(entries.length());
316 :
317 : size_t count = 0;
318 : for (size_t i = 0; i < entries.length(); i++) {
319 : SafepointNunboxEntry& entry = entries[i];
320 :
321 : if (entry.payload.isUse()) {
322 : // No allocation associated with the payload.
323 : continue;
324 : }
325 :
326 : if (entry.type.isUse()) {
327 : // No allocation associated with the type. Look for another
328 : // safepoint entry with an allocation for the type.
329 : entry.type = safepoint->findTypeAllocation(entry.typeVreg);
330 : if (entry.type.isUse())
331 : continue;
332 : }
333 :
334 : count++;
335 :
336 : uint16_t header = 0;
337 :
338 : header |= (AllocationToPartKind(entry.type) << TYPE_KIND_SHIFT);
339 : header |= (AllocationToPartKind(entry.payload) << PAYLOAD_KIND_SHIFT);
340 :
341 : uint32_t typeVal;
342 : bool typeExtra = !CanEncodeInfoInHeader(entry.type, &typeVal);
343 : if (!typeExtra)
344 : header |= (typeVal << TYPE_INFO_SHIFT);
345 : else
346 : header |= (MAX_INFO_VALUE << TYPE_INFO_SHIFT);
347 :
348 : uint32_t payloadVal;
349 : bool payloadExtra = !CanEncodeInfoInHeader(entry.payload, &payloadVal);
350 : if (!payloadExtra)
351 : header |= (payloadVal << PAYLOAD_INFO_SHIFT);
352 : else
353 : header |= (MAX_INFO_VALUE << PAYLOAD_INFO_SHIFT);
354 :
355 : stream_.writeFixedUint16_t(header);
356 : if (typeExtra)
357 : stream_.writeUnsigned(typeVal);
358 : if (payloadExtra)
359 : stream_.writeUnsigned(payloadVal);
360 : }
361 :
362 : // Update the stream with the actual number of safepoint entries written.
363 : stream_.writeUnsignedAt(pos, count, entries.length());
364 : }
365 : #endif
366 :
367 : void
368 39 : SafepointWriter::encode(LSafepoint* safepoint)
369 : {
370 39 : uint32_t safepointOffset = startEntry();
371 :
372 39 : MOZ_ASSERT(safepoint->osiCallPointOffset());
373 :
374 39 : writeOsiCallPointOffset(safepoint->osiCallPointOffset());
375 39 : writeGcRegs(safepoint);
376 39 : writeGcSlots(safepoint);
377 39 : writeValueSlots(safepoint);
378 :
379 : #ifdef JS_NUNBOX32
380 : writeNunboxParts(safepoint);
381 : #endif
382 :
383 39 : writeSlotsOrElementsSlots(safepoint);
384 :
385 39 : endEntry();
386 39 : safepoint->setOffset(safepointOffset);
387 39 : }
388 :
389 : void
390 39 : SafepointWriter::endEntry()
391 : {
392 39 : JitSpew(JitSpew_Safepoints, " -- entry ended at %d", uint32_t(stream_.length()));
393 39 : }
394 :
395 1680 : SafepointReader::SafepointReader(IonScript* script, const SafepointIndex* si)
396 1680 : : stream_(script->safepoints() + si->safepointOffset(),
397 1680 : script->safepoints() + script->safepointsSize()),
398 1680 : frameSlots_((script->frameSlots() / sizeof(intptr_t)) + 1), // Stack slot counts are inclusive.
399 5040 : argumentSlots_(script->argumentSlots() / sizeof(intptr_t))
400 : {
401 1680 : osiCallPointOffset_ = stream_.readUnsigned();
402 :
403 : // gcSpills is a subset of allGprSpills.
404 1680 : allGprSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
405 1680 : if (allGprSpills_.empty()) {
406 1680 : gcSpills_ = allGprSpills_;
407 1680 : valueSpills_ = allGprSpills_;
408 1680 : slotsOrElementsSpills_ = allGprSpills_;
409 : } else {
410 0 : gcSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
411 0 : slotsOrElementsSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
412 : #ifdef JS_PUNBOX64
413 0 : valueSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
414 : #endif
415 : }
416 1680 : allFloatSpills_ = FloatRegisterSet(ReadFloatRegisterMask(stream_));
417 :
418 1680 : advanceFromGcRegs();
419 1680 : }
420 :
421 : uint32_t
422 840 : SafepointReader::osiReturnPointOffset() const
423 : {
424 840 : return osiCallPointOffset_ + Assembler::PatchWrite_NearCallSize();
425 : }
426 :
427 : CodeLocationLabel
428 0 : SafepointReader::InvalidationPatchPoint(IonScript* script, const SafepointIndex* si)
429 : {
430 0 : SafepointReader reader(script, si);
431 :
432 0 : return CodeLocationLabel(script->method(), CodeOffset(reader.osiCallPointOffset()));
433 : }
434 :
435 : void
436 1680 : SafepointReader::advanceFromGcRegs()
437 : {
438 1680 : currentSlotChunk_ = 0;
439 1680 : nextSlotChunkNumber_ = 0;
440 1680 : currentSlotsAreStack_ = true;
441 1680 : }
442 :
443 : bool
444 0 : SafepointReader::getSlotFromBitmap(SafepointSlotEntry* entry)
445 : {
446 0 : while (currentSlotChunk_ == 0) {
447 : // Are there any more chunks to read?
448 0 : if (currentSlotsAreStack_) {
449 0 : if (nextSlotChunkNumber_ == BitSet::RawLengthForBits(frameSlots_)) {
450 0 : nextSlotChunkNumber_ = 0;
451 0 : currentSlotsAreStack_ = false;
452 0 : continue;
453 : }
454 0 : } else if (nextSlotChunkNumber_ == BitSet::RawLengthForBits(argumentSlots_)) {
455 0 : return false;
456 : }
457 :
458 : // Yes, read the next chunk.
459 0 : currentSlotChunk_ = stream_.readUnsigned();
460 0 : nextSlotChunkNumber_++;
461 : }
462 :
463 : // The current chunk still has bits in it, so get the next bit, then mask
464 : // it out of the slot chunk.
465 0 : uint32_t bit = FloorLog2(currentSlotChunk_);
466 0 : currentSlotChunk_ &= ~(1 << bit);
467 :
468 : // Return the slot, and re-scale it by the pointer size, reversing the
469 : // transformation in MapSlotsToBitset.
470 0 : entry->stack = currentSlotsAreStack_;
471 0 : entry->slot = (((nextSlotChunkNumber_ - 1) * BitSet::BitsPerWord) + bit) * sizeof(intptr_t);
472 0 : return true;
473 : }
474 :
475 : bool
476 0 : SafepointReader::getGcSlot(SafepointSlotEntry* entry)
477 : {
478 0 : if (getSlotFromBitmap(entry))
479 0 : return true;
480 0 : advanceFromGcSlots();
481 0 : return false;
482 : }
483 :
484 : void
485 0 : SafepointReader::advanceFromGcSlots()
486 : {
487 : // No, reset the counter.
488 0 : currentSlotChunk_ = 0;
489 0 : nextSlotChunkNumber_ = 0;
490 0 : currentSlotsAreStack_ = true;
491 0 : }
492 :
493 : bool
494 0 : SafepointReader::getValueSlot(SafepointSlotEntry* entry)
495 : {
496 0 : if (getSlotFromBitmap(entry))
497 0 : return true;
498 0 : advanceFromValueSlots();
499 0 : return false;
500 : }
501 :
502 : void
503 0 : SafepointReader::advanceFromValueSlots()
504 : {
505 : #ifdef JS_NUNBOX32
506 : nunboxSlotsRemaining_ = stream_.readUnsigned();
507 : #else
508 0 : nunboxSlotsRemaining_ = 0;
509 0 : advanceFromNunboxSlots();
510 : #endif
511 0 : }
512 :
513 : static inline LAllocation
514 0 : PartFromStream(CompactBufferReader& stream, NunboxPartKind kind, uint32_t info)
515 : {
516 0 : if (kind == Part_Reg)
517 0 : return LGeneralReg(Register::FromCode(info));
518 :
519 0 : if (info == MAX_INFO_VALUE)
520 0 : info = stream.readUnsigned();
521 :
522 0 : if (kind == Part_Stack)
523 0 : return LStackSlot(info);
524 :
525 0 : MOZ_ASSERT(kind == Part_Arg);
526 0 : return LArgument(info);
527 : }
528 :
529 : bool
530 0 : SafepointReader::getNunboxSlot(LAllocation* type, LAllocation* payload)
531 : {
532 0 : if (!nunboxSlotsRemaining_--) {
533 0 : advanceFromNunboxSlots();
534 0 : return false;
535 : }
536 :
537 0 : uint16_t header = stream_.readFixedUint16_t();
538 0 : NunboxPartKind typeKind = (NunboxPartKind)((header >> TYPE_KIND_SHIFT) & PART_KIND_MASK);
539 0 : NunboxPartKind payloadKind = (NunboxPartKind)((header >> PAYLOAD_KIND_SHIFT) & PART_KIND_MASK);
540 0 : uint32_t typeInfo = (header >> TYPE_INFO_SHIFT) & PART_INFO_MASK;
541 0 : uint32_t payloadInfo = (header >> PAYLOAD_INFO_SHIFT) & PART_INFO_MASK;
542 :
543 0 : *type = PartFromStream(stream_, typeKind, typeInfo);
544 0 : *payload = PartFromStream(stream_, payloadKind, payloadInfo);
545 0 : return true;
546 : }
547 :
548 : void
549 0 : SafepointReader::advanceFromNunboxSlots()
550 : {
551 0 : slotsOrElementsSlotsRemaining_ = stream_.readUnsigned();
552 0 : }
553 :
554 : bool
555 0 : SafepointReader::getSlotsOrElementsSlot(SafepointSlotEntry* entry)
556 : {
557 0 : if (!slotsOrElementsSlotsRemaining_--)
558 0 : return false;
559 0 : entry->stack = true;
560 0 : entry->slot = stream_.readUnsigned();
561 0 : return true;
562 : }
|
