Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sw=4 et tw=78:
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
6 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
7 :
8 : #ifndef gc_Nursery_h
9 : #define gc_Nursery_h
10 :
11 : #include "mozilla/EnumeratedArray.h"
12 : #include "mozilla/TimeStamp.h"
13 :
14 : #include "jsalloc.h"
15 : #include "jspubtd.h"
16 :
17 : #include "ds/BitArray.h"
18 : #include "gc/Heap.h"
19 : #include "gc/Memory.h"
20 : #include "js/Class.h"
21 : #include "js/GCAPI.h"
22 : #include "js/HashTable.h"
23 : #include "js/HeapAPI.h"
24 : #include "js/Value.h"
25 : #include "js/Vector.h"
26 : #include "vm/SharedMem.h"
27 :
28 : #define FOR_EACH_NURSERY_PROFILE_TIME(_) \
29 : /* Key Header text */ \
30 : _(Total, "total") \
31 : _(CancelIonCompilations, "canIon") \
32 : _(TraceValues, "mkVals") \
33 : _(TraceCells, "mkClls") \
34 : _(TraceSlots, "mkSlts") \
35 : _(TraceWholeCells, "mcWCll") \
36 : _(TraceGenericEntries, "mkGnrc") \
37 : _(CheckHashTables, "ckTbls") \
38 : _(MarkRuntime, "mkRntm") \
39 : _(MarkDebugger, "mkDbgr") \
40 : _(ClearNewObjectCache, "clrNOC") \
41 : _(CollectToFP, "collct") \
42 : _(ObjectsTenuredCallback, "tenCB") \
43 : _(SweepArrayBufferViewList, "swpABO") \
44 : _(UpdateJitActivations, "updtIn") \
45 : _(FreeMallocedBuffers, "frSlts") \
46 : _(ClearStoreBuffer, "clrSB") \
47 : _(Sweep, "sweep") \
48 : _(Resize, "resize") \
49 : _(Pretenure, "pretnr")
50 :
51 : namespace JS {
52 : struct Zone;
53 : } // namespace JS
54 :
55 : namespace js {
56 :
57 : class ObjectElements;
58 : class NativeObject;
59 : class Nursery;
60 : class HeapSlot;
61 : class ZoneGroup;
62 : class JSONPrinter;
63 :
64 : void SetGCZeal(JSRuntime*, uint8_t, uint32_t);
65 :
66 : namespace gc {
67 : class AutoMaybeStartBackgroundAllocation;
68 : struct Cell;
69 : class MinorCollectionTracer;
70 : class RelocationOverlay;
71 : struct TenureCountCache;
72 : } /* namespace gc */
73 :
74 : namespace jit {
75 : class MacroAssembler;
76 : } // namespace jit
77 :
78 : class TenuringTracer : public JSTracer
79 : {
80 : friend class Nursery;
81 : Nursery& nursery_;
82 :
83 : // Amount of data moved to the tenured generation during collection.
84 : size_t tenuredSize;
85 :
86 : // This list is threaded through the Nursery using the space from already
87 : // moved things. The list is used to fix up the moved things and to find
88 : // things held live by intra-Nursery pointers.
89 : gc::RelocationOverlay* head;
90 : gc::RelocationOverlay** tail;
91 :
92 : TenuringTracer(JSRuntime* rt, Nursery* nursery);
93 :
94 : public:
95 : const Nursery& nursery() const { return nursery_; }
96 :
97 : // Returns true if the pointer was updated.
98 : template <typename T> void traverse(T** thingp);
99 : template <typename T> void traverse(T* thingp);
100 :
101 : void insertIntoFixupList(gc::RelocationOverlay* entry);
102 :
103 : // The store buffers need to be able to call these directly.
104 : void traceObject(JSObject* src);
105 : void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length);
106 15 : void traceSlots(JS::Value* vp, uint32_t nslots) { traceSlots(vp, vp + nslots); }
107 :
108 : private:
109 112969 : Nursery& nursery() { return nursery_; }
110 :
111 : JSObject* moveToTenured(JSObject* src);
112 : size_t moveObjectToTenured(JSObject* dst, JSObject* src, gc::AllocKind dstKind);
113 : size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
114 : size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
115 :
116 : void traceSlots(JS::Value* vp, JS::Value* end);
117 : };
118 :
119 : /*
120 : * Classes with JSCLASS_SKIP_NURSERY_FINALIZE or Wrapper classes with
121 : * CROSS_COMPARTMENT flags will not have their finalizer called if they are
122 : * nursery allocated and not promoted to the tenured heap. The finalizers for
123 : * these classes must do nothing except free data which was allocated via
124 : * Nursery::allocateBuffer.
125 : */
126 : inline bool
127 21670 : CanNurseryAllocateFinalizedClass(const js::Class* const clasp)
128 : {
129 21670 : MOZ_ASSERT(clasp->hasFinalize());
130 21670 : return clasp->flags & JSCLASS_SKIP_NURSERY_FINALIZE;
131 : }
132 :
133 : class Nursery
134 : {
135 : public:
136 : static const size_t Alignment = gc::ChunkSize;
137 : static const size_t ChunkShift = gc::ChunkShift;
138 :
139 : explicit Nursery(JSRuntime* rt);
140 : ~Nursery();
141 :
142 : MOZ_MUST_USE bool init(uint32_t maxNurseryBytes, AutoLockGC& lock);
143 :
144 259 : unsigned maxChunks() const { return maxNurseryChunks_; }
145 199500 : unsigned numChunks() const { return chunks_.length(); }
146 :
147 214 : bool exists() const { return maxChunks() != 0; }
148 : size_t nurserySize() const { return maxChunks() << ChunkShift; }
149 :
150 : void enable();
151 : void disable();
152 199137 : bool isEnabled() const { return numChunks() != 0; }
153 :
154 : /* Return true if no allocations have been made since the last collection. */
155 : bool isEmpty() const;
156 :
157 : /*
158 : * Check whether an arbitrary pointer is within the nursery. This is
159 : * slower than IsInsideNursery(Cell*), but works on all types of pointers.
160 : */
161 : MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete;
162 78456 : MOZ_ALWAYS_INLINE bool isInside(const void* p) const {
163 266584 : for (auto chunk : chunks_) {
164 202636 : if (uintptr_t(p) - chunk->start() < gc::ChunkSize)
165 14508 : return true;
166 : }
167 63948 : return false;
168 : }
169 : template<typename T>
170 0 : bool isInside(const SharedMem<T>& p) const {
171 0 : return isInside(p.unwrap(/*safe - used for value in comparison above*/));
172 : }
173 :
174 : /*
175 : * Allocate and return a pointer to a new GC object with its |slots|
176 : * pointer pre-filled. Returns nullptr if the Nursery is full.
177 : */
178 : JSObject* allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp);
179 :
180 : /* Allocate a buffer for a given zone, using the nursery if possible. */
181 : void* allocateBuffer(JS::Zone* zone, size_t nbytes);
182 :
183 : /*
184 : * Allocate a buffer for a given object, using the nursery if possible and
185 : * obj is in the nursery.
186 : */
187 : void* allocateBuffer(JSObject* obj, size_t nbytes);
188 :
189 : /* Resize an existing object buffer. */
190 : void* reallocateBuffer(JSObject* obj, void* oldBuffer,
191 : size_t oldBytes, size_t newBytes);
192 :
193 : /* Free an object buffer. */
194 : void freeBuffer(void* buffer);
195 :
196 : /* The maximum number of bytes allowed to reside in nursery buffers. */
197 : static const size_t MaxNurseryBufferSize = 1024;
198 :
199 : /* Do a minor collection. */
200 : void collect(JS::gcreason::Reason reason);
201 :
202 : /*
203 : * Check if the thing at |*ref| in the Nursery has been forwarded. If so,
204 : * sets |*ref| to the new location of the object and returns true. Otherwise
205 : * returns false and leaves |*ref| unset.
206 : */
207 : MOZ_ALWAYS_INLINE MOZ_MUST_USE static bool getForwardedPointer(JSObject** ref);
208 :
209 : /* Forward a slots/elements pointer stored in an Ion frame. */
210 : void forwardBufferPointer(HeapSlot** pSlotsElems);
211 :
212 0 : void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct) {
213 0 : if (trc->isTenuringTracer() && isInside(oldData))
214 0 : setForwardingPointer(oldData, newData, direct);
215 0 : }
216 :
217 : /* Mark a malloced buffer as no longer needing to be freed. */
218 3 : void removeMallocedBuffer(void* buffer) {
219 3 : mallocedBuffers.remove(buffer);
220 3 : }
221 :
222 : void waitBackgroundFreeEnd();
223 :
224 508 : MOZ_MUST_USE bool addedUniqueIdToCell(gc::Cell* cell) {
225 508 : MOZ_ASSERT(IsInsideNursery(cell));
226 508 : MOZ_ASSERT(isEnabled());
227 508 : return cellsWithUid_.append(cell);
228 : }
229 :
230 : MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj);
231 :
232 24 : size_t sizeOfHeapCommitted() const {
233 24 : return numChunks() * gc::ChunkSize;
234 : }
235 0 : size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const {
236 0 : if (!mallocedBuffers.initialized())
237 0 : return 0;
238 0 : size_t total = 0;
239 0 : for (MallocedBuffersSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront())
240 0 : total += mallocSizeOf(r.front());
241 0 : total += mallocedBuffers.sizeOfExcludingThis(mallocSizeOf);
242 0 : return total;
243 : }
244 :
245 : // The number of bytes from the start position to the end of the nursery.
246 : size_t spaceToEnd() const;
247 :
248 : // Free space remaining, not counting chunk trailers.
249 66 : MOZ_ALWAYS_INLINE size_t freeSpace() const {
250 66 : MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize);
251 66 : return (currentEnd_ - position_) +
252 66 : (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize;
253 : }
254 :
255 : #ifdef JS_GC_ZEAL
256 : void enterZealMode();
257 : void leaveZealMode();
258 : #endif
259 :
260 : /* Write profile time JSON on JSONPrinter. */
261 : void renderProfileJSON(JSONPrinter& json) const;
262 :
263 : /* Print header line for profile times. */
264 : static void printProfileHeader();
265 :
266 : /* Print total profile times on shutdown. */
267 : void printTotalProfileTimes();
268 :
269 95 : void* addressOfCurrentEnd() const { return (void*)¤tEnd_; }
270 190 : void* addressOfPosition() const { return (void*)&position_; }
271 :
272 : void requestMinorGC(JS::gcreason::Reason reason) const;
273 :
274 54352 : bool minorGCRequested() const { return minorGCTriggerReason_ != JS::gcreason::NO_REASON; }
275 18 : JS::gcreason::Reason minorGCTriggerReason() const { return minorGCTriggerReason_; }
276 24 : void clearMinorGCRequest() { minorGCTriggerReason_ = JS::gcreason::NO_REASON; }
277 :
278 0 : bool enableProfiling() const { return enableProfiling_; }
279 :
280 : private:
281 : /* The amount of space in the mapped nursery available to allocations. */
282 : static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer);
283 :
284 : struct NurseryChunk {
285 : char data[NurseryChunkUsableSize];
286 : gc::ChunkTrailer trailer;
287 : static NurseryChunk* fromChunk(gc::Chunk* chunk);
288 : void init(JSRuntime* rt);
289 : void poisonAndInit(JSRuntime* rt, uint8_t poison);
290 202793 : uintptr_t start() const { return uintptr_t(&data); }
291 37697 : uintptr_t end() const { return uintptr_t(&trailer); }
292 : gc::Chunk* toChunk(JSRuntime* rt);
293 : };
294 : static_assert(sizeof(NurseryChunk) == gc::ChunkSize,
295 : "Nursery chunk size must match gc::Chunk size.");
296 :
297 : JSRuntime* runtime_;
298 :
299 : /* Vector of allocated chunks to allocate from. */
300 : Vector<NurseryChunk*, 0, SystemAllocPolicy> chunks_;
301 :
302 : /* Pointer to the first unallocated byte in the nursery. */
303 : uintptr_t position_;
304 :
305 : /* Pointer to the logical start of the Nursery. */
306 : unsigned currentStartChunk_;
307 : uintptr_t currentStartPosition_;
308 :
309 : /* Pointer to the last byte of space in the current chunk. */
310 : uintptr_t currentEnd_;
311 :
312 : /* The index of the chunk that is currently being allocated from. */
313 : unsigned currentChunk_;
314 :
315 : /* Maximum number of chunks to allocate for the nursery. */
316 : unsigned maxNurseryChunks_;
317 :
318 : /* Promotion rate for the previous minor collection. */
319 : double previousPromotionRate_;
320 :
321 : /* Report minor collections taking at least this long, if enabled. */
322 : mozilla::TimeDuration profileThreshold_;
323 : bool enableProfiling_;
324 :
325 : /* Report ObjectGroups with at lest this many instances tenured. */
326 : int64_t reportTenurings_;
327 :
328 : /*
329 : * Whether and why a collection of this nursery has been requested. This is
330 : * mutable as it is set by the store buffer, which otherwise cannot modify
331 : * anything in the nursery.
332 : */
333 : mutable JS::gcreason::Reason minorGCTriggerReason_;
334 :
335 : /* Profiling data. */
336 :
337 : enum class ProfileKey
338 : {
339 : #define DEFINE_TIME_KEY(name, text) \
340 : name,
341 : FOR_EACH_NURSERY_PROFILE_TIME(DEFINE_TIME_KEY)
342 : #undef DEFINE_TIME_KEY
343 : KeyCount
344 : };
345 :
346 : using ProfileTimes =
347 : mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount, mozilla::TimeStamp>;
348 : using ProfileDurations =
349 : mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount, mozilla::TimeDuration>;
350 :
351 : ProfileTimes startTimes_;
352 : ProfileDurations profileDurations_;
353 : ProfileDurations totalDurations_;
354 : uint64_t minorGcCount_;
355 :
356 : /*
357 : * The set of externally malloced buffers potentially kept live by objects
358 : * stored in the nursery. Any external buffers that do not belong to a
359 : * tenured thing at the end of a minor GC must be freed.
360 : */
361 : typedef HashSet<void*, PointerHasher<void*, 3>, SystemAllocPolicy> MallocedBuffersSet;
362 : MallocedBuffersSet mallocedBuffers;
363 :
364 : /* A task structure used to free the malloced bufers on a background thread. */
365 : struct FreeMallocedBuffersTask;
366 : FreeMallocedBuffersTask* freeMallocedBuffersTask;
367 :
368 : /*
369 : * During a collection most hoisted slot and element buffers indicate their
370 : * new location with a forwarding pointer at the base. This does not work
371 : * for buffers whose length is less than pointer width, or when different
372 : * buffers might overlap each other. For these, an entry in the following
373 : * table is used.
374 : */
375 : typedef HashMap<void*, void*, PointerHasher<void*, 1>, SystemAllocPolicy> ForwardedBufferMap;
376 : ForwardedBufferMap forwardedBuffers;
377 :
378 : /*
379 : * When we assign a unique id to cell in the nursery, that almost always
380 : * means that the cell will be in a hash table, and thus, held live,
381 : * automatically moving the uid from the nursery to its new home in
382 : * tenured. It is possible, if rare, for an object that acquired a uid to
383 : * be dead before the next collection, in which case we need to know to
384 : * remove it when we sweep.
385 : *
386 : * Note: we store the pointers as Cell* here, resulting in an ugly cast in
387 : * sweep. This is because this structure is used to help implement
388 : * stable object hashing and we have to break the cycle somehow.
389 : */
390 : using CellsWithUniqueIdVector = Vector<gc::Cell*, 8, SystemAllocPolicy>;
391 : CellsWithUniqueIdVector cellsWithUid_;
392 :
393 : using NativeObjectVector = Vector<NativeObject*, 0, SystemAllocPolicy>;
394 : NativeObjectVector dictionaryModeObjects_;
395 :
396 : #ifdef JS_GC_ZEAL
397 : struct Canary;
398 : Canary* lastCanary_;
399 : #endif
400 :
401 : NurseryChunk* allocChunk();
402 :
403 38017 : NurseryChunk& chunk(unsigned index) const {
404 38017 : return *chunks_[index];
405 : }
406 :
407 : void setCurrentChunk(unsigned chunkno);
408 : void setStartPosition();
409 :
410 : void updateNumChunks(unsigned newCount);
411 : void updateNumChunksLocked(unsigned newCount,
412 : gc::AutoMaybeStartBackgroundAllocation& maybeBgAlloc,
413 : AutoLockGC& lock);
414 :
415 : MOZ_ALWAYS_INLINE uintptr_t allocationEnd() const {
416 : MOZ_ASSERT(numChunks() > 0);
417 : return chunks_.back()->end();
418 : }
419 :
420 37648 : MOZ_ALWAYS_INLINE uintptr_t currentEnd() const {
421 37648 : MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end());
422 37648 : return currentEnd_;
423 : }
424 :
425 188376 : uintptr_t position() const { return position_; }
426 :
427 113414 : JSRuntime* runtime() const { return runtime_; }
428 :
429 : /* Allocates a new GC thing from the tenured generation during minor GC. */
430 : gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind);
431 :
432 : /* Common internal allocator function. */
433 : void* allocate(size_t size);
434 :
435 : double doCollection(JS::gcreason::Reason reason,
436 : gc::TenureCountCache& tenureCounts);
437 :
438 : /*
439 : * Move the object at |src| in the Nursery to an already-allocated cell
440 : * |dst| in Tenured.
441 : */
442 : void collectToFixedPoint(TenuringTracer& trc, gc::TenureCountCache& tenureCounts);
443 :
444 : /* Handle relocation of slots/elements pointers stored in Ion frames. */
445 : void setForwardingPointer(void* oldData, void* newData, bool direct);
446 :
447 : void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots);
448 : void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
449 : uint32_t capacity);
450 :
451 : /* Free malloced pointers owned by freed things in the nursery. */
452 : void freeMallocedBuffers();
453 :
454 : /*
455 : * Frees all non-live nursery-allocated things at the end of a minor
456 : * collection.
457 : */
458 : void sweep();
459 :
460 : void sweepDictionaryModeObjects();
461 :
462 : /* Change the allocable space provided by the nursery. */
463 : void maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate);
464 : void growAllocableSpace();
465 : void shrinkAllocableSpace();
466 : void minimizeAllocableSpace();
467 :
468 : /* Profile recording and printing. */
469 : void maybeClearProfileDurations();
470 : void startProfile(ProfileKey key);
471 : void endProfile(ProfileKey key);
472 : static void printProfileDurations(const ProfileDurations& times);
473 :
474 : friend class TenuringTracer;
475 : friend class gc::MinorCollectionTracer;
476 : friend class jit::MacroAssembler;
477 : };
478 :
479 : } /* namespace js */
480 :
481 : #endif /* gc_Nursery_h */
|
