Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sw=4 et tw=78:
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
6 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
7 :
8 : #include "gc/Nursery-inl.h"
9 :
10 : #include "mozilla/DebugOnly.h"
11 : #include "mozilla/IntegerPrintfMacros.h"
12 : #include "mozilla/Move.h"
13 : #include "mozilla/Unused.h"
14 :
15 : #include "jscompartment.h"
16 : #include "jsfriendapi.h"
17 : #include "jsgc.h"
18 : #include "jsutil.h"
19 :
20 : #include "gc/GCInternals.h"
21 : #include "gc/Memory.h"
22 : #include "jit/JitFrames.h"
23 : #include "vm/ArrayObject.h"
24 : #include "vm/Debugger.h"
25 : #if defined(DEBUG)
26 : #include "vm/EnvironmentObject.h"
27 : #endif
28 : #include "vm/JSONPrinter.h"
29 : #include "vm/Time.h"
30 : #include "vm/TypedArrayObject.h"
31 : #include "vm/TypeInference.h"
32 :
33 : #include "jsobjinlines.h"
34 :
35 : #include "vm/NativeObject-inl.h"
36 :
37 : using namespace js;
38 : using namespace gc;
39 :
40 : using mozilla::ArrayLength;
41 : using mozilla::DebugOnly;
42 : using mozilla::PodCopy;
43 : using mozilla::TimeDuration;
44 : using mozilla::TimeStamp;
45 :
46 : constexpr uintptr_t CanaryMagicValue = 0xDEADB15D;
47 :
48 : struct js::Nursery::FreeMallocedBuffersTask : public GCParallelTask
49 : {
50 4 : explicit FreeMallocedBuffersTask(FreeOp* fop) : GCParallelTask(fop->runtime()), fop_(fop) {}
51 4 : bool init() { return buffers_.init(); }
52 : void transferBuffersToFree(MallocedBuffersSet& buffersToFree,
53 : const AutoLockHelperThreadState& lock);
54 0 : ~FreeMallocedBuffersTask() override { join(); }
55 :
56 : private:
57 : FreeOp* fop_;
58 : MallocedBuffersSet buffers_;
59 :
60 : virtual void run() override;
61 : };
62 :
63 : #ifdef JS_GC_ZEAL
64 : struct js::Nursery::Canary
65 : {
66 : uintptr_t magicValue;
67 : Canary* next;
68 : };
69 : #endif
70 :
71 : inline void
72 159 : js::Nursery::NurseryChunk::poisonAndInit(JSRuntime* rt, uint8_t poison)
73 : {
74 159 : JS_POISON(this, poison, ChunkSize);
75 159 : init(rt);
76 159 : }
77 :
78 : inline void
79 159 : js::Nursery::NurseryChunk::init(JSRuntime* rt)
80 : {
81 159 : new (&trailer) gc::ChunkTrailer(rt, &rt->gc.storeBuffer());
82 159 : }
83 :
84 : /* static */ inline js::Nursery::NurseryChunk*
85 11 : js::Nursery::NurseryChunk::fromChunk(Chunk* chunk)
86 : {
87 11 : return reinterpret_cast<NurseryChunk*>(chunk);
88 : }
89 :
90 : inline Chunk*
91 4 : js::Nursery::NurseryChunk::toChunk(JSRuntime* rt)
92 : {
93 4 : auto chunk = reinterpret_cast<Chunk*>(this);
94 4 : chunk->init(rt);
95 4 : return chunk;
96 : }
97 :
98 4 : js::Nursery::Nursery(JSRuntime* rt)
99 : : runtime_(rt)
100 : , position_(0)
101 : , currentStartChunk_(0)
102 : , currentStartPosition_(0)
103 : , currentEnd_(0)
104 : , currentChunk_(0)
105 : , maxNurseryChunks_(0)
106 : , previousPromotionRate_(0)
107 : , profileThreshold_(0)
108 : , enableProfiling_(false)
109 : , reportTenurings_(0)
110 : , minorGCTriggerReason_(JS::gcreason::NO_REASON)
111 : , minorGcCount_(0)
112 : , freeMallocedBuffersTask(nullptr)
113 : #ifdef JS_GC_ZEAL
114 4 : , lastCanary_(nullptr)
115 : #endif
116 4 : {}
117 :
118 : bool
119 4 : js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGC& lock)
120 : {
121 4 : if (!mallocedBuffers.init())
122 0 : return false;
123 :
124 4 : freeMallocedBuffersTask = js_new<FreeMallocedBuffersTask>(runtime()->defaultFreeOp());
125 4 : if (!freeMallocedBuffersTask || !freeMallocedBuffersTask->init())
126 0 : return false;
127 :
128 : /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */
129 4 : maxNurseryChunks_ = maxNurseryBytes >> ChunkShift;
130 :
131 : /* If no chunks are specified then the nursery is permanently disabled. */
132 4 : if (maxNurseryChunks_ == 0)
133 0 : return true;
134 :
135 8 : AutoMaybeStartBackgroundAllocation maybeBgAlloc;
136 4 : updateNumChunksLocked(1, maybeBgAlloc, lock);
137 4 : if (numChunks() == 0)
138 0 : return false;
139 :
140 4 : setCurrentChunk(0);
141 4 : setStartPosition();
142 :
143 4 : char* env = getenv("JS_GC_PROFILE_NURSERY");
144 4 : if (env) {
145 0 : if (0 == strcmp(env, "help")) {
146 : fprintf(stderr, "JS_GC_PROFILE_NURSERY=N\n"
147 0 : "\tReport minor GC's taking at least N microseconds.\n");
148 0 : exit(0);
149 : }
150 0 : enableProfiling_ = true;
151 0 : profileThreshold_ = TimeDuration::FromMicroseconds(atoi(env));
152 : }
153 :
154 4 : env = getenv("JS_GC_REPORT_TENURING");
155 4 : if (env) {
156 0 : if (0 == strcmp(env, "help")) {
157 : fprintf(stderr, "JS_GC_REPORT_TENURING=N\n"
158 0 : "\tAfter a minor GC, report any ObjectGroups with at least N instances tenured.\n");
159 0 : exit(0);
160 : }
161 0 : reportTenurings_ = atoi(env);
162 : }
163 :
164 4 : if (!runtime()->gc.storeBuffer().enable())
165 0 : return false;
166 :
167 4 : MOZ_ASSERT(isEnabled());
168 4 : return true;
169 : }
170 :
171 0 : js::Nursery::~Nursery()
172 : {
173 0 : disable();
174 0 : js_delete(freeMallocedBuffersTask);
175 0 : }
176 :
177 : void
178 3 : js::Nursery::enable()
179 : {
180 3 : MOZ_ASSERT(isEmpty());
181 3 : MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled());
182 3 : if (isEnabled() || !maxChunks())
183 0 : return;
184 :
185 3 : updateNumChunks(1);
186 3 : if (numChunks() == 0)
187 0 : return;
188 :
189 3 : setCurrentChunk(0);
190 3 : setStartPosition();
191 : #ifdef JS_GC_ZEAL
192 3 : if (runtime()->hasZealMode(ZealMode::GenerationalGC))
193 0 : enterZealMode();
194 : #endif
195 :
196 3 : MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer().enable());
197 3 : return;
198 : }
199 :
200 : void
201 12 : js::Nursery::disable()
202 : {
203 12 : MOZ_ASSERT(isEmpty());
204 12 : if (!isEnabled())
205 9 : return;
206 3 : updateNumChunks(0);
207 3 : currentEnd_ = 0;
208 3 : runtime()->gc.storeBuffer().disable();
209 : }
210 :
211 : bool
212 120 : js::Nursery::isEmpty() const
213 : {
214 120 : if (!isEnabled())
215 12 : return true;
216 :
217 108 : if (!runtime()->hasZealMode(ZealMode::GenerationalGC)) {
218 108 : MOZ_ASSERT(currentStartChunk_ == 0);
219 108 : MOZ_ASSERT(currentStartPosition_ == chunk(0).start());
220 : }
221 108 : return position() == currentStartPosition_;
222 : }
223 :
224 : #ifdef JS_GC_ZEAL
225 : void
226 0 : js::Nursery::enterZealMode() {
227 0 : if (isEnabled())
228 0 : updateNumChunks(maxNurseryChunks_);
229 0 : }
230 :
231 : void
232 0 : js::Nursery::leaveZealMode() {
233 0 : if (isEnabled()) {
234 0 : MOZ_ASSERT(isEmpty());
235 0 : setCurrentChunk(0);
236 0 : setStartPosition();
237 : }
238 0 : }
239 : #endif // JS_GC_ZEAL
240 :
241 : JSObject*
242 34752 : js::Nursery::allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp)
243 : {
244 : /* Ensure there's enough space to replace the contents with a RelocationOverlay. */
245 34752 : MOZ_ASSERT(size >= sizeof(RelocationOverlay));
246 :
247 : /* Sanity check the finalizer. */
248 34752 : MOZ_ASSERT_IF(clasp->hasFinalize(), CanNurseryAllocateFinalizedClass(clasp) ||
249 : clasp->isProxy());
250 :
251 : /* Make the object allocation. */
252 34752 : JSObject* obj = static_cast<JSObject*>(allocate(size));
253 34752 : if (!obj)
254 0 : return nullptr;
255 :
256 : /* If we want external slots, add them. */
257 34752 : HeapSlot* slots = nullptr;
258 34752 : if (numDynamic) {
259 62 : MOZ_ASSERT(clasp->isNative());
260 62 : slots = static_cast<HeapSlot*>(allocateBuffer(cx->zone(), numDynamic * sizeof(HeapSlot)));
261 62 : if (!slots) {
262 : /*
263 : * It is safe to leave the allocated object uninitialized, since we
264 : * do not visit unallocated things in the nursery.
265 : */
266 0 : return nullptr;
267 : }
268 : }
269 :
270 : /* Always initialize the slots field to match the JIT behavior. */
271 34752 : obj->setInitialSlotsMaybeNonNative(slots);
272 :
273 34752 : TraceNurseryAlloc(obj, size);
274 34752 : return obj;
275 : }
276 :
277 : void*
278 37648 : js::Nursery::allocate(size_t size)
279 : {
280 37648 : MOZ_ASSERT(isEnabled());
281 37648 : MOZ_ASSERT(!JS::CurrentThreadIsHeapBusy());
282 37648 : MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
283 37648 : MOZ_ASSERT_IF(currentChunk_ == currentStartChunk_, position() >= currentStartPosition_);
284 37648 : MOZ_ASSERT(position() % CellAlignBytes == 0);
285 37648 : MOZ_ASSERT(size % CellAlignBytes == 0);
286 :
287 : #ifdef JS_GC_ZEAL
288 : static const size_t CanarySize = (sizeof(Nursery::Canary) + CellAlignBytes - 1) & ~CellAlignMask;
289 37648 : if (runtime()->gc.hasZealMode(ZealMode::CheckNursery))
290 0 : size += CanarySize;
291 : #endif
292 :
293 37648 : if (currentEnd() < position() + size) {
294 0 : if (currentChunk_ + 1 == numChunks())
295 0 : return nullptr;
296 0 : setCurrentChunk(currentChunk_ + 1);
297 : }
298 :
299 37648 : void* thing = (void*)position();
300 37648 : position_ = position() + size;
301 :
302 37648 : JS_EXTRA_POISON(thing, JS_ALLOCATED_NURSERY_PATTERN, size);
303 :
304 : #ifdef JS_GC_ZEAL
305 37648 : if (runtime()->gc.hasZealMode(ZealMode::CheckNursery)) {
306 0 : auto canary = reinterpret_cast<Canary*>(position() - CanarySize);
307 0 : canary->magicValue = CanaryMagicValue;
308 0 : canary->next = nullptr;
309 0 : if (lastCanary_) {
310 0 : MOZ_ASSERT(!lastCanary_->next);
311 0 : lastCanary_->next = canary;
312 : }
313 0 : lastCanary_ = canary;
314 : }
315 : #endif
316 :
317 37648 : MemProfiler::SampleNursery(reinterpret_cast<void*>(thing), size);
318 37648 : return thing;
319 : }
320 :
321 : void*
322 2906 : js::Nursery::allocateBuffer(Zone* zone, size_t nbytes)
323 : {
324 2906 : MOZ_ASSERT(nbytes > 0);
325 :
326 2906 : if (nbytes <= MaxNurseryBufferSize) {
327 2896 : void* buffer = allocate(nbytes);
328 2896 : if (buffer)
329 2896 : return buffer;
330 : }
331 :
332 10 : void* buffer = zone->pod_malloc<uint8_t>(nbytes);
333 10 : if (buffer && !mallocedBuffers.putNew(buffer)) {
334 0 : js_free(buffer);
335 0 : return nullptr;
336 : }
337 10 : return buffer;
338 : }
339 :
340 : void*
341 10587 : js::Nursery::allocateBuffer(JSObject* obj, size_t nbytes)
342 : {
343 10587 : MOZ_ASSERT(obj);
344 10587 : MOZ_ASSERT(nbytes > 0);
345 :
346 10587 : if (!IsInsideNursery(obj))
347 8061 : return obj->zone()->pod_malloc<uint8_t>(nbytes);
348 2526 : return allocateBuffer(obj->zone(), nbytes);
349 : }
350 :
351 : void*
352 6037 : js::Nursery::reallocateBuffer(JSObject* obj, void* oldBuffer,
353 : size_t oldBytes, size_t newBytes)
354 : {
355 6037 : if (!IsInsideNursery(obj))
356 5785 : return obj->zone()->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes);
357 :
358 252 : if (!isInside(oldBuffer)) {
359 0 : void* newBuffer = obj->zone()->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes);
360 0 : if (newBuffer && oldBuffer != newBuffer)
361 0 : MOZ_ALWAYS_TRUE(mallocedBuffers.rekeyAs(oldBuffer, newBuffer, newBuffer));
362 0 : return newBuffer;
363 : }
364 :
365 : /* The nursery cannot make use of the returned slots data. */
366 252 : if (newBytes < oldBytes)
367 0 : return oldBuffer;
368 :
369 252 : void* newBuffer = allocateBuffer(obj->zone(), newBytes);
370 252 : if (newBuffer)
371 252 : PodCopy((uint8_t*)newBuffer, (uint8_t*)oldBuffer, oldBytes);
372 252 : return newBuffer;
373 : }
374 :
375 : void
376 0 : js::Nursery::freeBuffer(void* buffer)
377 : {
378 0 : if (!isInside(buffer)) {
379 0 : removeMallocedBuffer(buffer);
380 0 : js_free(buffer);
381 : }
382 0 : }
383 :
384 : void
385 2329 : Nursery::setForwardingPointer(void* oldData, void* newData, bool direct)
386 : {
387 2329 : MOZ_ASSERT(isInside(oldData));
388 :
389 : // Bug 1196210: If a zero-capacity header lands in the last 2 words of a
390 : // jemalloc chunk abutting the start of a nursery chunk, the (invalid)
391 : // newData pointer will appear to be "inside" the nursery.
392 2329 : MOZ_ASSERT(!isInside(newData) || (uintptr_t(newData) & ChunkMask) == 0);
393 :
394 2329 : if (direct) {
395 2328 : *reinterpret_cast<void**>(oldData) = newData;
396 : } else {
397 2 : AutoEnterOOMUnsafeRegion oomUnsafe;
398 1 : if (!forwardedBuffers.initialized() && !forwardedBuffers.init())
399 0 : oomUnsafe.crash("Nursery::setForwardingPointer");
400 : #ifdef DEBUG
401 1 : if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData))
402 0 : MOZ_ASSERT(p->value() == newData);
403 : #endif
404 1 : if (!forwardedBuffers.put(oldData, newData))
405 0 : oomUnsafe.crash("Nursery::setForwardingPointer");
406 : }
407 2329 : }
408 :
409 : void
410 1228 : Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots)
411 : {
412 : // Slot arrays always have enough space for a forwarding pointer, since the
413 : // number of slots is never zero.
414 1228 : MOZ_ASSERT(nslots > 0);
415 1228 : setForwardingPointer(oldSlots, newSlots, /* direct = */ true);
416 1228 : }
417 :
418 : void
419 1101 : Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
420 : uint32_t capacity)
421 : {
422 : // Only use a direct forwarding pointer if there is enough space for one.
423 1101 : setForwardingPointer(oldHeader->elements(), newHeader->elements(),
424 1101 : capacity > 0);
425 1101 : }
426 :
427 : #ifdef DEBUG
428 0 : static bool IsWriteableAddress(void* ptr)
429 : {
430 0 : volatile uint64_t* vPtr = reinterpret_cast<volatile uint64_t*>(ptr);
431 0 : *vPtr = *vPtr;
432 0 : return true;
433 : }
434 : #endif
435 :
436 : void
437 0 : js::Nursery::forwardBufferPointer(HeapSlot** pSlotsElems)
438 : {
439 0 : HeapSlot* old = *pSlotsElems;
440 :
441 0 : if (!isInside(old))
442 0 : return;
443 :
444 : // The new location for this buffer is either stored inline with it or in
445 : // the forwardedBuffers table.
446 : do {
447 0 : if (forwardedBuffers.initialized()) {
448 0 : if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(old)) {
449 0 : *pSlotsElems = reinterpret_cast<HeapSlot*>(p->value());
450 0 : break;
451 : }
452 : }
453 :
454 0 : *pSlotsElems = *reinterpret_cast<HeapSlot**>(old);
455 : } while (false);
456 :
457 0 : MOZ_ASSERT(!isInside(*pSlotsElems));
458 0 : MOZ_ASSERT(IsWriteableAddress(*pSlotsElems));
459 : }
460 :
461 21 : js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery)
462 : : JSTracer(rt, JSTracer::TracerKindTag::Tenuring, TraceWeakMapKeysValues)
463 : , nursery_(*nursery)
464 : , tenuredSize(0)
465 : , head(nullptr)
466 21 : , tail(&head)
467 : {
468 21 : }
469 :
470 : void
471 0 : js::Nursery::renderProfileJSON(JSONPrinter& json) const
472 : {
473 0 : if (!isEnabled()) {
474 0 : json.beginObject();
475 0 : json.property("status", "nursery disabled");
476 0 : json.endObject();
477 0 : return;
478 : }
479 :
480 0 : json.beginObject();
481 : #define EXTRACT_NAME(name, text) #name,
482 : static const char* names[] = {
483 : FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME)
484 : #undef EXTRACT_NAME
485 : "" };
486 :
487 0 : size_t i = 0;
488 0 : for (auto time : profileDurations_)
489 0 : json.property(names[i++], time, json.MICROSECONDS);
490 :
491 0 : json.endObject();
492 : }
493 :
494 : /* static */ void
495 0 : js::Nursery::printProfileHeader()
496 : {
497 0 : fprintf(stderr, "MinorGC: Reason PRate Size ");
498 : #define PRINT_HEADER(name, text) \
499 : fprintf(stderr, " %6s", text);
500 0 : FOR_EACH_NURSERY_PROFILE_TIME(PRINT_HEADER)
501 : #undef PRINT_HEADER
502 0 : fprintf(stderr, "\n");
503 0 : }
504 :
505 : /* static */ void
506 0 : js::Nursery::printProfileDurations(const ProfileDurations& times)
507 : {
508 0 : for (auto time : times)
509 0 : fprintf(stderr, " %6" PRIi64, static_cast<int64_t>(time.ToMicroseconds()));
510 0 : fprintf(stderr, "\n");
511 0 : }
512 :
513 : void
514 0 : js::Nursery::printTotalProfileTimes()
515 : {
516 0 : if (enableProfiling_) {
517 0 : fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ", minorGcCount_);
518 0 : printProfileDurations(totalDurations_);
519 : }
520 0 : }
521 :
522 : void
523 24 : js::Nursery::maybeClearProfileDurations()
524 : {
525 504 : for (auto& duration : profileDurations_)
526 480 : duration = mozilla::TimeDuration();
527 24 : }
528 :
529 : inline void
530 429 : js::Nursery::startProfile(ProfileKey key)
531 : {
532 429 : startTimes_[key] = TimeStamp::Now();
533 429 : }
534 :
535 : inline void
536 429 : js::Nursery::endProfile(ProfileKey key)
537 : {
538 429 : profileDurations_[key] = TimeStamp::Now() - startTimes_[key];
539 429 : totalDurations_[key] += profileDurations_[key];
540 429 : }
541 :
542 : void
543 24 : js::Nursery::collect(JS::gcreason::Reason reason)
544 : {
545 24 : MOZ_ASSERT(!TlsContext.get()->suppressGC);
546 :
547 24 : if (!isEnabled() || isEmpty()) {
548 : // Our barriers are not always exact, and there may be entries in the
549 : // storebuffer even when the nursery is disabled or empty. It's not safe
550 : // to keep these entries as they may refer to tenured cells which may be
551 : // freed after this point.
552 3 : runtime()->gc.storeBuffer().clear();
553 : }
554 :
555 24 : if (!isEnabled())
556 0 : return;
557 :
558 24 : JSRuntime* rt = runtime();
559 24 : rt->gc.incMinorGcNumber();
560 :
561 : #ifdef JS_GC_ZEAL
562 24 : if (rt->gc.hasZealMode(ZealMode::CheckNursery)) {
563 0 : for (auto canary = lastCanary_; canary; canary = canary->next)
564 0 : MOZ_ASSERT(canary->magicValue == CanaryMagicValue);
565 : }
566 24 : lastCanary_ = nullptr;
567 : #endif
568 :
569 24 : rt->gc.stats().beginNurseryCollection(reason);
570 24 : TraceMinorGCStart();
571 :
572 24 : maybeClearProfileDurations();
573 24 : startProfile(ProfileKey::Total);
574 :
575 : // The hazard analysis thinks doCollection can invalidate pointers in
576 : // tenureCounts below.
577 48 : JS::AutoSuppressGCAnalysis nogc;
578 :
579 24 : TenureCountCache tenureCounts;
580 24 : double promotionRate = 0;
581 24 : if (!isEmpty())
582 21 : promotionRate = doCollection(reason, tenureCounts);
583 :
584 : // Resize the nursery.
585 24 : startProfile(ProfileKey::Resize);
586 24 : maybeResizeNursery(reason, promotionRate);
587 24 : endProfile(ProfileKey::Resize);
588 :
589 : // If we are promoting the nursery, or exhausted the store buffer with
590 : // pointers to nursery things, which will force a collection well before
591 : // the nursery is full, look for object groups that are getting promoted
592 : // excessively and try to pretenure them.
593 24 : startProfile(ProfileKey::Pretenure);
594 24 : uint32_t pretenureCount = 0;
595 24 : if (promotionRate > 0.8 || reason == JS::gcreason::FULL_STORE_BUFFER) {
596 18 : JSContext* cx = TlsContext.get();
597 306 : for (auto& entry : tenureCounts.entries) {
598 288 : if (entry.count >= 3000) {
599 0 : ObjectGroup* group = entry.group;
600 0 : if (group->canPreTenure()) {
601 0 : AutoCompartment ac(cx, group);
602 0 : group->setShouldPreTenure(cx);
603 0 : pretenureCount++;
604 : }
605 : }
606 : }
607 : }
608 24 : endProfile(ProfileKey::Pretenure);
609 :
610 : // We ignore gcMaxBytes when allocating for minor collection. However, if we
611 : // overflowed, we disable the nursery. The next time we allocate, we'll fail
612 : // because gcBytes >= gcMaxBytes.
613 24 : if (rt->gc.usage.gcBytes() >= rt->gc.tunables.gcMaxBytes())
614 0 : disable();
615 :
616 24 : endProfile(ProfileKey::Total);
617 24 : minorGcCount_++;
618 :
619 24 : TimeDuration totalTime = profileDurations_[ProfileKey::Total];
620 24 : rt->addTelemetry(JS_TELEMETRY_GC_MINOR_US, totalTime.ToMicroseconds());
621 24 : rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON, reason);
622 24 : if (totalTime.ToMilliseconds() > 1.0)
623 21 : rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON_LONG, reason);
624 24 : rt->addTelemetry(JS_TELEMETRY_GC_NURSERY_BYTES, sizeOfHeapCommitted());
625 24 : rt->addTelemetry(JS_TELEMETRY_GC_PRETENURE_COUNT, pretenureCount);
626 :
627 24 : rt->gc.stats().endNurseryCollection(reason);
628 24 : TraceMinorGCEnd();
629 :
630 24 : if (enableProfiling_ && totalTime >= profileThreshold_) {
631 0 : rt->gc.stats().maybePrintProfileHeaders();
632 :
633 0 : fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ",
634 : JS::gcreason::ExplainReason(reason),
635 : promotionRate * 100,
636 0 : numChunks());
637 0 : printProfileDurations(profileDurations_);
638 :
639 0 : if (reportTenurings_) {
640 0 : for (auto& entry : tenureCounts.entries) {
641 0 : if (entry.count >= reportTenurings_) {
642 0 : fprintf(stderr, " %d x ", entry.count);
643 0 : entry.group->print();
644 : }
645 : }
646 : }
647 : }
648 : }
649 :
650 : double
651 21 : js::Nursery::doCollection(JS::gcreason::Reason reason,
652 : TenureCountCache& tenureCounts)
653 : {
654 21 : JSRuntime* rt = runtime();
655 42 : AutoTraceSession session(rt, JS::HeapState::MinorCollecting);
656 42 : AutoSetThreadIsPerformingGC performingGC;
657 42 : AutoStopVerifyingBarriers av(rt, false);
658 42 : AutoDisableProxyCheck disableStrictProxyChecking;
659 42 : mozilla::DebugOnly<AutoEnterOOMUnsafeRegion> oomUnsafeRegion;
660 :
661 21 : size_t initialNurserySize = spaceToEnd();
662 :
663 : // Move objects pointed to by roots from the nursery to the major heap.
664 21 : TenuringTracer mover(rt, this);
665 :
666 : // Mark the store buffer. This must happen first.
667 21 : StoreBuffer& sb = runtime()->gc.storeBuffer();
668 :
669 : // The MIR graph only contains nursery pointers if cancelIonCompilations()
670 : // is set on the store buffer, in which case we cancel all compilations.
671 21 : startProfile(ProfileKey::CancelIonCompilations);
672 21 : if (sb.cancelIonCompilations())
673 4 : js::CancelOffThreadIonCompile(rt);
674 21 : endProfile(ProfileKey::CancelIonCompilations);
675 :
676 21 : startProfile(ProfileKey::TraceValues);
677 21 : sb.traceValues(mover);
678 21 : endProfile(ProfileKey::TraceValues);
679 :
680 21 : startProfile(ProfileKey::TraceCells);
681 21 : sb.traceCells(mover);
682 21 : endProfile(ProfileKey::TraceCells);
683 :
684 21 : startProfile(ProfileKey::TraceSlots);
685 21 : sb.traceSlots(mover);
686 21 : endProfile(ProfileKey::TraceSlots);
687 :
688 21 : startProfile(ProfileKey::TraceWholeCells);
689 21 : sb.traceWholeCells(mover);
690 21 : endProfile(ProfileKey::TraceWholeCells);
691 :
692 21 : startProfile(ProfileKey::TraceGenericEntries);
693 21 : sb.traceGenericEntries(&mover);
694 21 : endProfile(ProfileKey::TraceGenericEntries);
695 :
696 21 : startProfile(ProfileKey::MarkRuntime);
697 21 : rt->gc.traceRuntimeForMinorGC(&mover, session.lock);
698 21 : endProfile(ProfileKey::MarkRuntime);
699 :
700 21 : startProfile(ProfileKey::MarkDebugger);
701 : {
702 42 : gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::MARK_ROOTS);
703 21 : Debugger::traceAllForMovingGC(&mover);
704 : }
705 21 : endProfile(ProfileKey::MarkDebugger);
706 :
707 21 : startProfile(ProfileKey::ClearNewObjectCache);
708 21 : rt->caches().newObjectCache.clearNurseryObjects(rt);
709 21 : endProfile(ProfileKey::ClearNewObjectCache);
710 :
711 : // Most of the work is done here. This loop iterates over objects that have
712 : // been moved to the major heap. If these objects have any outgoing pointers
713 : // to the nursery, then those nursery objects get moved as well, until no
714 : // objects are left to move. That is, we iterate to a fixed point.
715 21 : startProfile(ProfileKey::CollectToFP);
716 21 : collectToFixedPoint(mover, tenureCounts);
717 21 : endProfile(ProfileKey::CollectToFP);
718 :
719 : // Sweep compartments to update the array buffer object's view lists.
720 21 : startProfile(ProfileKey::SweepArrayBufferViewList);
721 2436 : for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next())
722 2415 : c->sweepAfterMinorGC(&mover);
723 21 : endProfile(ProfileKey::SweepArrayBufferViewList);
724 :
725 : // Update any slot or element pointers whose destination has been tenured.
726 21 : startProfile(ProfileKey::UpdateJitActivations);
727 21 : js::jit::UpdateJitActivationsForMinorGC(rt, &mover);
728 21 : forwardedBuffers.finish();
729 21 : endProfile(ProfileKey::UpdateJitActivations);
730 :
731 21 : startProfile(ProfileKey::ObjectsTenuredCallback);
732 21 : rt->gc.callObjectsTenuredCallback();
733 21 : endProfile(ProfileKey::ObjectsTenuredCallback);
734 :
735 : // Sweep.
736 21 : startProfile(ProfileKey::FreeMallocedBuffers);
737 21 : freeMallocedBuffers();
738 21 : endProfile(ProfileKey::FreeMallocedBuffers);
739 :
740 21 : startProfile(ProfileKey::Sweep);
741 21 : sweep();
742 21 : endProfile(ProfileKey::Sweep);
743 :
744 21 : startProfile(ProfileKey::ClearStoreBuffer);
745 21 : runtime()->gc.storeBuffer().clear();
746 21 : endProfile(ProfileKey::ClearStoreBuffer);
747 :
748 : // Make sure hashtables have been updated after the collection.
749 21 : startProfile(ProfileKey::CheckHashTables);
750 : #ifdef JS_GC_ZEAL
751 21 : if (rt->hasZealMode(ZealMode::CheckHashTablesOnMinorGC))
752 0 : CheckHashTablesAfterMovingGC(rt);
753 : #endif
754 21 : endProfile(ProfileKey::CheckHashTables);
755 :
756 : // Calculate and return the promotion rate.
757 42 : return mover.tenuredSize / double(initialNurserySize);
758 : }
759 :
760 : void
761 4 : js::Nursery::FreeMallocedBuffersTask::transferBuffersToFree(MallocedBuffersSet& buffersToFree,
762 : const AutoLockHelperThreadState& lock)
763 : {
764 : // Transfer the contents of the source set to the task's buffers_ member by
765 : // swapping the sets, which also clears the source.
766 4 : MOZ_ASSERT(!isRunningWithLockHeld(lock));
767 4 : MOZ_ASSERT(buffers_.empty());
768 4 : mozilla::Swap(buffers_, buffersToFree);
769 4 : }
770 :
771 : void
772 4 : js::Nursery::FreeMallocedBuffersTask::run()
773 : {
774 8 : for (MallocedBuffersSet::Range r = buffers_.all(); !r.empty(); r.popFront())
775 4 : fop_->free_(r.front());
776 4 : buffers_.clear();
777 4 : }
778 :
779 : void
780 21 : js::Nursery::freeMallocedBuffers()
781 : {
782 21 : if (mallocedBuffers.empty())
783 17 : return;
784 :
785 : bool started;
786 : {
787 8 : AutoLockHelperThreadState lock;
788 4 : freeMallocedBuffersTask->joinWithLockHeld(lock);
789 4 : freeMallocedBuffersTask->transferBuffersToFree(mallocedBuffers, lock);
790 4 : started = freeMallocedBuffersTask->startWithLockHeld(lock);
791 : }
792 :
793 4 : if (!started)
794 0 : freeMallocedBuffersTask->runFromActiveCooperatingThread(runtime());
795 :
796 4 : MOZ_ASSERT(mallocedBuffers.empty());
797 : }
798 :
799 : void
800 0 : js::Nursery::waitBackgroundFreeEnd()
801 : {
802 : // We may finishRoots before nursery init if runtime init fails.
803 0 : if (!isEnabled())
804 0 : return;
805 :
806 0 : MOZ_ASSERT(freeMallocedBuffersTask);
807 0 : freeMallocedBuffersTask->join();
808 : }
809 :
810 : void
811 21 : js::Nursery::sweep()
812 : {
813 : /* Sweep unique id's in all in-use chunks. */
814 443 : for (Cell* cell : cellsWithUid_) {
815 422 : JSObject* obj = static_cast<JSObject*>(cell);
816 422 : if (!IsForwarded(obj))
817 27 : obj->zone()->removeUniqueId(obj);
818 : else
819 395 : MOZ_ASSERT(Forwarded(obj)->zone()->hasUniqueId(Forwarded(obj)));
820 : }
821 21 : cellsWithUid_.clear();
822 :
823 21 : sweepDictionaryModeObjects();
824 :
825 : #ifdef JS_GC_ZEAL
826 : /* Poison the nursery contents so touching a freed object will crash. */
827 81 : for (unsigned i = 0; i < numChunks(); i++)
828 60 : chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
829 :
830 21 : if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
831 : /* Only reset the alloc point when we are close to the end. */
832 0 : if (currentChunk_ + 1 == numChunks())
833 0 : setCurrentChunk(0);
834 : } else
835 : #endif
836 : {
837 : #ifdef JS_CRASH_DIAGNOSTICS
838 81 : for (unsigned i = 0; i < numChunks(); ++i)
839 60 : chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
840 : #endif
841 21 : setCurrentChunk(0);
842 : }
843 :
844 : /* Set current start position for isEmpty checks. */
845 21 : setStartPosition();
846 21 : MemProfiler::SweepNursery(runtime());
847 21 : }
848 :
849 : size_t
850 21 : js::Nursery::spaceToEnd() const
851 : {
852 21 : unsigned lastChunk = numChunks() - 1;
853 :
854 21 : MOZ_ASSERT(lastChunk >= currentStartChunk_);
855 21 : MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize);
856 :
857 21 : size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
858 21 : ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize);
859 :
860 21 : MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize);
861 :
862 21 : return bytes;
863 : }
864 :
865 : MOZ_ALWAYS_INLINE void
866 28 : js::Nursery::setCurrentChunk(unsigned chunkno)
867 : {
868 28 : MOZ_ASSERT(chunkno < maxChunks());
869 28 : MOZ_ASSERT(chunkno < numChunks());
870 28 : currentChunk_ = chunkno;
871 28 : position_ = chunk(chunkno).start();
872 28 : currentEnd_ = chunk(chunkno).end();
873 28 : chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN);
874 28 : }
875 :
876 : MOZ_ALWAYS_INLINE void
877 28 : js::Nursery::setStartPosition()
878 : {
879 28 : currentStartChunk_ = currentChunk_;
880 28 : currentStartPosition_ = position();
881 28 : }
882 :
883 : void
884 24 : js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate)
885 : {
886 : static const double GrowThreshold = 0.05;
887 : static const double ShrinkThreshold = 0.01;
888 :
889 : // Shrink the nursery to its minimum size of we ran out of memory or
890 : // received a memory pressure event.
891 24 : if (gc::IsOOMReason(reason)) {
892 0 : minimizeAllocableSpace();
893 0 : return;
894 : }
895 :
896 24 : if (promotionRate > GrowThreshold)
897 3 : growAllocableSpace();
898 21 : else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold)
899 4 : shrinkAllocableSpace();
900 :
901 24 : previousPromotionRate_ = promotionRate;
902 : }
903 :
904 : void
905 3 : js::Nursery::growAllocableSpace()
906 : {
907 3 : updateNumChunks(Min(numChunks() * 2, maxNurseryChunks_));
908 3 : }
909 :
910 : void
911 4 : js::Nursery::shrinkAllocableSpace()
912 : {
913 : #ifdef JS_GC_ZEAL
914 4 : if (runtime()->hasZealMode(ZealMode::GenerationalGC))
915 0 : return;
916 : #endif
917 4 : updateNumChunks(Max(numChunks() - 1, 1u));
918 : }
919 :
920 : void
921 0 : js::Nursery::minimizeAllocableSpace()
922 : {
923 : #ifdef JS_GC_ZEAL
924 0 : if (runtime()->hasZealMode(ZealMode::GenerationalGC))
925 0 : return;
926 : #endif
927 0 : updateNumChunks(1);
928 : }
929 :
930 : void
931 13 : js::Nursery::updateNumChunks(unsigned newCount)
932 : {
933 13 : if (numChunks() != newCount) {
934 20 : AutoMaybeStartBackgroundAllocation maybeBgAlloc;
935 20 : AutoLockGC lock(runtime());
936 10 : updateNumChunksLocked(newCount, maybeBgAlloc, lock);
937 : }
938 13 : }
939 :
940 : void
941 14 : js::Nursery::updateNumChunksLocked(unsigned newCount,
942 : AutoMaybeStartBackgroundAllocation& maybeBgAlloc,
943 : AutoLockGC& lock)
944 : {
945 : // The GC nursery is an optimization and so if we fail to allocate nursery
946 : // chunks we do not report an error.
947 :
948 14 : MOZ_ASSERT(newCount <= maxChunks());
949 :
950 14 : unsigned priorCount = numChunks();
951 14 : MOZ_ASSERT(priorCount != newCount);
952 :
953 14 : if (newCount < priorCount) {
954 : // Shrink the nursery and free unused chunks.
955 8 : for (unsigned i = newCount; i < priorCount; i++)
956 4 : runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock);
957 4 : chunks_.shrinkTo(newCount);
958 4 : return;
959 : }
960 :
961 : // Grow the nursery and allocate new chunks.
962 10 : if (!chunks_.resize(newCount))
963 0 : return;
964 :
965 21 : for (unsigned i = priorCount; i < newCount; i++) {
966 11 : auto newChunk = runtime()->gc.getOrAllocChunk(lock, maybeBgAlloc);
967 11 : if (!newChunk) {
968 0 : chunks_.shrinkTo(i);
969 0 : return;
970 : }
971 :
972 11 : chunks_[i] = NurseryChunk::fromChunk(newChunk);
973 11 : chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN);
974 : }
975 : }
976 :
977 : bool
978 6 : js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj)
979 : {
980 6 : MOZ_ASSERT(IsInsideNursery(obj));
981 6 : return dictionaryModeObjects_.append(obj);
982 : }
983 :
984 : void
985 21 : js::Nursery::sweepDictionaryModeObjects()
986 : {
987 27 : for (auto obj : dictionaryModeObjects_) {
988 6 : if (!IsForwarded(obj))
989 0 : obj->sweepDictionaryListPointer();
990 : }
991 21 : dictionaryModeObjects_.clear();
992 21 : }
|
