Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sts=4 et sw=4 tw=99:
3 : * This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #ifndef jsgcinlines_h
8 : #define jsgcinlines_h
9 :
10 : #include "jsgc.h"
11 :
12 : #include "mozilla/DebugOnly.h"
13 : #include "mozilla/Maybe.h"
14 :
15 : #include "gc/GCTrace.h"
16 : #include "gc/Zone.h"
17 :
18 : namespace js {
19 : namespace gc {
20 :
21 : inline void
22 45 : MakeAccessibleAfterMovingGC(void* anyp) {}
23 :
24 : inline void
25 18820 : MakeAccessibleAfterMovingGC(JSObject* obj) {
26 18820 : if (obj->isNative())
27 18577 : obj->as<NativeObject>().updateShapeAfterMovingGC();
28 18820 : }
29 :
30 : static inline AllocKind
31 25940 : GetGCObjectKind(const Class* clasp)
32 : {
33 25940 : if (clasp == FunctionClassPtr)
34 0 : return AllocKind::FUNCTION;
35 :
36 25940 : MOZ_ASSERT(!clasp->isProxy(), "Proxies should use GetProxyGCObjectKind");
37 :
38 25940 : uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp);
39 25940 : if (clasp->flags & JSCLASS_HAS_PRIVATE)
40 11080 : nslots++;
41 25940 : return GetGCObjectKind(nslots);
42 : }
43 :
44 492 : class ArenaIter
45 : {
46 : Arena* arena;
47 : Arena* unsweptArena;
48 : Arena* sweptArena;
49 : mozilla::DebugOnly<bool> initialized;
50 :
51 : public:
52 57 : ArenaIter()
53 57 : : arena(nullptr), unsweptArena(nullptr), sweptArena(nullptr), initialized(false) {}
54 :
55 435 : ArenaIter(JS::Zone* zone, AllocKind kind) : initialized(false) { init(zone, kind); }
56 :
57 492 : void init(JS::Zone* zone, AllocKind kind) {
58 492 : MOZ_ASSERT(!initialized);
59 492 : MOZ_ASSERT(zone);
60 492 : initialized = true;
61 492 : arena = zone->arenas.getFirstArena(kind);
62 492 : unsweptArena = zone->arenas.getFirstArenaToSweep(kind);
63 492 : sweptArena = zone->arenas.getFirstSweptArena(kind);
64 492 : if (!unsweptArena) {
65 492 : unsweptArena = sweptArena;
66 492 : sweptArena = nullptr;
67 : }
68 492 : if (!arena) {
69 206 : arena = unsweptArena;
70 206 : unsweptArena = sweptArena;
71 206 : sweptArena = nullptr;
72 : }
73 492 : }
74 :
75 291491 : bool done() const {
76 291491 : MOZ_ASSERT(initialized);
77 291491 : return !arena;
78 : }
79 :
80 2846 : Arena* get() const {
81 2846 : MOZ_ASSERT(!done());
82 2846 : return arena;
83 : }
84 :
85 2846 : void next() {
86 2846 : MOZ_ASSERT(!done());
87 2846 : arena = arena->next;
88 2846 : if (!arena) {
89 286 : arena = unsweptArena;
90 286 : unsweptArena = sweptArena;
91 286 : sweptArena = nullptr;
92 : }
93 2846 : }
94 : };
95 :
96 : enum CellIterNeedsBarrier : uint8_t
97 : {
98 : CellIterDoesntNeedBarrier = 0,
99 : CellIterMayNeedBarrier = 1
100 : };
101 :
102 57 : class ArenaCellIterImpl
103 : {
104 : size_t firstThingOffset;
105 : size_t thingSize;
106 : Arena* arenaAddr;
107 : FreeSpan span;
108 : uint_fast16_t thing;
109 : JS::TraceKind traceKind;
110 : bool needsBarrier;
111 : mozilla::DebugOnly<bool> initialized;
112 :
113 : // Upon entry, |thing| points to any thing (free or used) and finds the
114 : // first used thing, which may be |thing|.
115 62266 : void moveForwardIfFree() {
116 62266 : MOZ_ASSERT(!done());
117 62266 : MOZ_ASSERT(thing);
118 : // Note: if |span| is empty, this test will fail, which is what we want
119 : // -- |span| being empty means that we're past the end of the last free
120 : // thing, all the remaining things in the arena are used, and we'll
121 : // never need to move forward.
122 62266 : if (thing == span.first) {
123 55 : thing = span.last + thingSize;
124 55 : span = *span.nextSpan(arenaAddr);
125 : }
126 62266 : }
127 :
128 : public:
129 57 : ArenaCellIterImpl()
130 57 : : firstThingOffset(0),
131 : thingSize(0),
132 : arenaAddr(nullptr),
133 : thing(0),
134 : traceKind(JS::TraceKind::Null),
135 : needsBarrier(false),
136 57 : initialized(false)
137 57 : {}
138 :
139 0 : explicit ArenaCellIterImpl(Arena* arena, CellIterNeedsBarrier mayNeedBarrier)
140 0 : : initialized(false)
141 : {
142 0 : init(arena, mayNeedBarrier);
143 0 : }
144 :
145 55 : void init(Arena* arena, CellIterNeedsBarrier mayNeedBarrier) {
146 55 : MOZ_ASSERT(!initialized);
147 55 : MOZ_ASSERT(arena);
148 55 : initialized = true;
149 55 : AllocKind kind = arena->getAllocKind();
150 55 : firstThingOffset = Arena::firstThingOffset(kind);
151 55 : thingSize = Arena::thingSize(kind);
152 55 : traceKind = MapAllocToTraceKind(kind);
153 55 : needsBarrier = mayNeedBarrier && !JS::CurrentThreadIsHeapCollecting();
154 55 : reset(arena);
155 55 : }
156 :
157 : // Use this to move from an Arena of a particular kind to another Arena of
158 : // the same kind.
159 2036 : void reset(Arena* arena) {
160 2036 : MOZ_ASSERT(initialized);
161 2036 : MOZ_ASSERT(arena);
162 2036 : arenaAddr = arena;
163 2036 : span = *arena->getFirstFreeSpan();
164 2036 : thing = firstThingOffset;
165 2036 : moveForwardIfFree();
166 2036 : }
167 :
168 498580 : bool done() const {
169 498580 : MOZ_ASSERT(initialized);
170 498580 : MOZ_ASSERT(thing <= ArenaSize);
171 498580 : return thing == ArenaSize;
172 : }
173 :
174 155946 : TenuredCell* getCell() const {
175 155946 : MOZ_ASSERT(!done());
176 155946 : TenuredCell* cell = reinterpret_cast<TenuredCell*>(uintptr_t(arenaAddr) + thing);
177 :
178 : // This can result in a a new reference being created to an object that
179 : // an ongoing incremental GC may find to be unreachable, so we may need
180 : // a barrier here.
181 155946 : if (needsBarrier)
182 51795 : ExposeGCThingToActiveJS(JS::GCCellPtr(cell, traceKind));
183 :
184 155946 : return cell;
185 : }
186 :
187 155946 : template<typename T> T* get() const {
188 155946 : MOZ_ASSERT(!done());
189 155946 : MOZ_ASSERT(JS::MapTypeToTraceKind<T>::kind == traceKind);
190 155946 : return static_cast<T*>(getCell());
191 : }
192 :
193 62211 : void next() {
194 62211 : MOZ_ASSERT(!done());
195 62211 : thing += thingSize;
196 62211 : if (thing < ArenaSize)
197 60230 : moveForwardIfFree();
198 62211 : }
199 : };
200 :
201 : template<>
202 : JSObject*
203 : ArenaCellIterImpl::get<JSObject>() const;
204 :
205 : class ArenaCellIter : public ArenaCellIterImpl
206 : {
207 : public:
208 : explicit ArenaCellIter(Arena* arena)
209 : : ArenaCellIterImpl(arena, CellIterMayNeedBarrier)
210 : {
211 : MOZ_ASSERT(JS::CurrentThreadIsHeapTracing());
212 : }
213 : };
214 :
215 0 : class ArenaCellIterUnderGC : public ArenaCellIterImpl
216 : {
217 : public:
218 0 : explicit ArenaCellIterUnderGC(Arena* arena)
219 0 : : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier)
220 : {
221 0 : MOZ_ASSERT(CurrentThreadIsPerformingGC());
222 0 : }
223 : };
224 :
225 0 : class ArenaCellIterUnderFinalize : public ArenaCellIterImpl
226 : {
227 : public:
228 0 : explicit ArenaCellIterUnderFinalize(Arena* arena)
229 0 : : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier)
230 : {
231 0 : MOZ_ASSERT(CurrentThreadIsGCSweeping());
232 0 : }
233 : };
234 :
235 0 : class ArenaCellIterUnbarriered : public ArenaCellIterImpl
236 : {
237 : public:
238 0 : explicit ArenaCellIterUnbarriered(Arena* arena)
239 0 : : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier)
240 0 : {}
241 : };
242 :
243 : template <typename T>
244 : class ZoneCellIter;
245 :
246 : template <>
247 57 : class ZoneCellIter<TenuredCell> {
248 : ArenaIter arenaIter;
249 : ArenaCellIterImpl cellIter;
250 : mozilla::Maybe<JS::AutoAssertNoGC> nogc;
251 :
252 : protected:
253 : // For use when a subclass wants to insert some setup before init().
254 57 : ZoneCellIter() {}
255 :
256 57 : void init(JS::Zone* zone, AllocKind kind) {
257 57 : MOZ_ASSERT_IF(IsNurseryAllocable(kind),
258 : zone->isAtomsZone() || zone->group()->nursery().isEmpty());
259 57 : initForTenuredIteration(zone, kind);
260 57 : }
261 :
262 57 : void initForTenuredIteration(JS::Zone* zone, AllocKind kind) {
263 57 : JSRuntime* rt = zone->runtimeFromAnyThread();
264 :
265 : // If called from outside a GC, ensure that the heap is in a state
266 : // that allows us to iterate.
267 57 : if (!JS::CurrentThreadIsHeapBusy()) {
268 : // Assert that no GCs can occur while a ZoneCellIter is live.
269 18 : nogc.emplace();
270 : }
271 :
272 : // We have a single-threaded runtime, so there's no need to protect
273 : // against other threads iterating or allocating. However, we do have
274 : // background finalization; we may have to wait for this to finish if
275 : // it's currently active.
276 57 : if (IsBackgroundFinalized(kind) && zone->arenas.needBackgroundFinalizeWait(kind))
277 0 : rt->gc.waitBackgroundSweepEnd();
278 57 : arenaIter.init(zone, kind);
279 57 : if (!arenaIter.done())
280 55 : cellIter.init(arenaIter.get(), CellIterMayNeedBarrier);
281 57 : }
282 :
283 : public:
284 0 : ZoneCellIter(JS::Zone* zone, AllocKind kind) {
285 : // If we are iterating a nursery-allocated kind then we need to
286 : // evict first so that we can see all things.
287 0 : if (IsNurseryAllocable(kind))
288 0 : zone->runtimeFromActiveCooperatingThread()->gc.evictNursery();
289 :
290 0 : init(zone, kind);
291 0 : }
292 :
293 0 : ZoneCellIter(JS::Zone* zone, AllocKind kind, const js::gc::AutoAssertEmptyNursery&) {
294 : // No need to evict the nursery. (This constructor is known statically
295 : // to not GC.)
296 0 : init(zone, kind);
297 0 : }
298 :
299 280425 : bool done() const {
300 280425 : return arenaIter.done();
301 : }
302 :
303 : template<typename T>
304 155946 : T* get() const {
305 155946 : MOZ_ASSERT(!done());
306 155946 : return cellIter.get<T>();
307 : }
308 :
309 0 : TenuredCell* getCell() const {
310 0 : MOZ_ASSERT(!done());
311 0 : return cellIter.getCell();
312 : }
313 :
314 62211 : void next() {
315 62211 : MOZ_ASSERT(!done());
316 62211 : cellIter.next();
317 62211 : if (cellIter.done()) {
318 2036 : MOZ_ASSERT(!arenaIter.done());
319 2036 : arenaIter.next();
320 2036 : if (!arenaIter.done())
321 1981 : cellIter.reset(arenaIter.get());
322 : }
323 62211 : }
324 : };
325 :
326 : // Iterator over the cells in a Zone, where the GC type (JSString, JSObject) is
327 : // known, for a single AllocKind. Example usages:
328 : //
329 : // for (auto obj = zone->cellIter<JSObject>(AllocKind::OBJECT0); !obj.done(); obj.next())
330 : // ...
331 : //
332 : // for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next())
333 : // f(script->code());
334 : //
335 : // As this code demonstrates, you can use 'script' as if it were a JSScript*.
336 : // Its actual type is ZoneCellIter<JSScript>, but for most purposes it will
337 : // autoconvert to JSScript*.
338 : //
339 : // Note that in the JSScript case, ZoneCellIter is able to infer the AllocKind
340 : // from the type 'JSScript', whereas in the JSObject case, the kind must be
341 : // given (because there are multiple AllocKinds for objects).
342 : //
343 : // Also, the static rooting hazard analysis knows that the JSScript case will
344 : // not GC during construction. The JSObject case needs to GC, or more precisely
345 : // to empty the nursery and clear out the store buffer, so that it can see all
346 : // objects to iterate over (the nursery is not iterable) and remove the
347 : // possibility of having pointers from the store buffer to data hanging off
348 : // stuff we're iterating over that we are going to delete. (The latter should
349 : // not be a problem, since such instances should be using RelocatablePtr do
350 : // remove themselves from the store buffer on deletion, but currently for
351 : // subtle reasons that isn't good enough.)
352 : //
353 : // If the iterator is used within a GC, then there is no need to evict the
354 : // nursery (again). You may select a variant that will skip the eviction either
355 : // by specializing on a GCType that is never allocated in the nursery, or
356 : // explicitly by passing in a trailing AutoAssertEmptyNursery argument.
357 : //
358 : template <typename GCType>
359 57 : class ZoneCellIter : public ZoneCellIter<TenuredCell> {
360 : public:
361 : // Non-nursery allocated (equivalent to having an entry in
362 : // MapTypeToFinalizeKind). The template declaration here is to discard this
363 : // constructor overload if MapTypeToFinalizeKind<GCType>::kind does not
364 : // exist. Note that there will be no remaining overloads that will work,
365 : // which makes sense given that you haven't specified which of the
366 : // AllocKinds to use for GCType.
367 : //
368 : // If we later add a nursery allocable GCType with a single AllocKind, we
369 : // will want to add an overload of this constructor that does the right
370 : // thing (ie, it empties the nursery before iterating.)
371 57 : explicit ZoneCellIter(JS::Zone* zone) : ZoneCellIter<TenuredCell>() {
372 57 : init(zone, MapTypeToFinalizeKind<GCType>::kind);
373 57 : }
374 :
375 : // Non-nursery allocated, nursery is known to be empty: same behavior as above.
376 0 : ZoneCellIter(JS::Zone* zone, const js::gc::AutoAssertEmptyNursery&) : ZoneCellIter(zone) {
377 0 : }
378 :
379 : // Arbitrary kind, which will be assumed to be nursery allocable (and
380 : // therefore the nursery will be emptied before iterating.)
381 0 : ZoneCellIter(JS::Zone* zone, AllocKind kind) : ZoneCellIter<TenuredCell>(zone, kind) {
382 0 : }
383 :
384 : // Arbitrary kind, which will be assumed to be nursery allocable, but the
385 : // nursery is known to be empty already: same behavior as non-nursery types.
386 0 : ZoneCellIter(JS::Zone* zone, AllocKind kind, const js::gc::AutoAssertEmptyNursery& empty)
387 0 : : ZoneCellIter<TenuredCell>(zone, kind, empty)
388 : {
389 0 : }
390 :
391 155946 : GCType* get() const { return ZoneCellIter<TenuredCell>::get<GCType>(); }
392 43005 : operator GCType*() const { return get(); }
393 112941 : GCType* operator ->() const { return get(); }
394 : };
395 :
396 0 : class GrayObjectIter : public ZoneCellIter<TenuredCell> {
397 : public:
398 0 : explicit GrayObjectIter(JS::Zone* zone, AllocKind kind) : ZoneCellIter<TenuredCell>() {
399 0 : initForTenuredIteration(zone, kind);
400 0 : }
401 :
402 0 : JSObject* get() const { return ZoneCellIter<TenuredCell>::get<JSObject>(); }
403 : operator JSObject*() const { return get(); }
404 0 : JSObject* operator ->() const { return get(); }
405 : };
406 :
407 11 : class GCZonesIter
408 : {
409 : private:
410 : ZonesIter zone;
411 :
412 : public:
413 11 : explicit GCZonesIter(JSRuntime* rt, ZoneSelector selector = WithAtoms) : zone(rt, selector) {
414 11 : MOZ_ASSERT(JS::CurrentThreadIsHeapBusy());
415 11 : if (!zone->isCollectingFromAnyThread())
416 1 : next();
417 11 : }
418 :
419 2158 : bool done() const { return zone.done(); }
420 :
421 161 : void next() {
422 161 : MOZ_ASSERT(!done());
423 176 : do {
424 176 : zone.next();
425 176 : } while (!zone.done() && !zone->isCollectingFromAnyThread());
426 161 : }
427 :
428 240 : JS::Zone* get() const {
429 240 : MOZ_ASSERT(!done());
430 240 : return zone;
431 : }
432 :
433 48 : operator JS::Zone*() const { return get(); }
434 192 : JS::Zone* operator->() const { return get(); }
435 : };
436 :
437 : typedef CompartmentsIterT<GCZonesIter> GCCompartmentsIter;
438 :
439 : /* Iterates over all zones in the current sweep group. */
440 : class GCSweepGroupIter {
441 : private:
442 : JS::Zone* current;
443 :
444 : public:
445 0 : explicit GCSweepGroupIter(JSRuntime* rt) {
446 0 : MOZ_ASSERT(CurrentThreadIsPerformingGC());
447 0 : current = rt->gc.getCurrentSweepGroup();
448 0 : }
449 :
450 0 : bool done() const { return !current; }
451 :
452 0 : void next() {
453 0 : MOZ_ASSERT(!done());
454 0 : current = current->nextNodeInGroup();
455 0 : }
456 :
457 0 : JS::Zone* get() const {
458 0 : MOZ_ASSERT(!done());
459 0 : return current;
460 : }
461 :
462 0 : operator JS::Zone*() const { return get(); }
463 0 : JS::Zone* operator->() const { return get(); }
464 : };
465 :
466 : typedef CompartmentsIterT<GCSweepGroupIter> GCCompartmentGroupIter;
467 :
468 : inline void
469 22681 : RelocationOverlay::forwardTo(Cell* cell)
470 : {
471 22681 : MOZ_ASSERT(!isForwarded());
472 : // The location of magic_ is important because it must never be valid to see
473 : // the value Relocated there in a GC thing that has not been moved.
474 : static_assert(offsetof(RelocationOverlay, magic_) == offsetof(JSObject, group_) &&
475 : offsetof(RelocationOverlay, magic_) == offsetof(js::Shape, base_) &&
476 : offsetof(RelocationOverlay, magic_) == offsetof(JSString, d.u1.flags),
477 : "RelocationOverlay::magic_ is in the wrong location");
478 22681 : magic_ = Relocated;
479 22681 : newLocation_ = cell;
480 22681 : }
481 :
482 : template <typename T>
483 : struct MightBeForwarded
484 : {
485 : static_assert(mozilla::IsBaseOf<Cell, T>::value,
486 : "T must derive from Cell");
487 : static_assert(!mozilla::IsSame<Cell, T>::value && !mozilla::IsSame<TenuredCell, T>::value,
488 : "T must not be Cell or TenuredCell");
489 :
490 : static const bool value = mozilla::IsBaseOf<JSObject, T>::value ||
491 : mozilla::IsBaseOf<Shape, T>::value ||
492 : mozilla::IsBaseOf<BaseShape, T>::value ||
493 : mozilla::IsBaseOf<JSString, T>::value ||
494 : mozilla::IsBaseOf<JSScript, T>::value ||
495 : mozilla::IsBaseOf<js::LazyScript, T>::value ||
496 : mozilla::IsBaseOf<js::Scope, T>::value ||
497 : mozilla::IsBaseOf<js::RegExpShared, T>::value;
498 : };
499 :
500 : template <typename T>
501 : inline bool
502 131350 : IsForwarded(T* t)
503 : {
504 131350 : RelocationOverlay* overlay = RelocationOverlay::fromCell(t);
505 : if (!MightBeForwarded<T>::value) {
506 14388 : MOZ_ASSERT(!overlay->isForwarded());
507 14388 : return false;
508 : }
509 :
510 116962 : return overlay->isForwarded();
511 : }
512 :
513 : struct IsForwardedFunctor : public BoolDefaultAdaptor<Value, false> {
514 : template <typename T> bool operator()(T* t) { return IsForwarded(t); }
515 : };
516 :
517 : inline bool
518 : IsForwarded(const JS::Value& value)
519 : {
520 : return DispatchTyped(IsForwardedFunctor(), value);
521 : }
522 :
523 : template <typename T>
524 : inline T*
525 1309 : Forwarded(T* t)
526 : {
527 1309 : RelocationOverlay* overlay = RelocationOverlay::fromCell(t);
528 1309 : MOZ_ASSERT(overlay->isForwarded());
529 1309 : return reinterpret_cast<T*>(overlay->forwardingAddress());
530 : }
531 :
532 : struct ForwardedFunctor : public IdentityDefaultAdaptor<Value> {
533 : template <typename T> inline Value operator()(T* t) {
534 : return js::gc::RewrapTaggedPointer<Value, T>::wrap(Forwarded(t));
535 : }
536 : };
537 :
538 : inline Value
539 : Forwarded(const JS::Value& value)
540 : {
541 : return DispatchTyped(ForwardedFunctor(), value);
542 : }
543 :
544 : template <typename T>
545 : inline T
546 18865 : MaybeForwarded(T t)
547 : {
548 18865 : if (IsForwarded(t))
549 519 : t = Forwarded(t);
550 18865 : MakeAccessibleAfterMovingGC(t);
551 18865 : return t;
552 : }
553 :
554 : #ifdef JSGC_HASH_TABLE_CHECKS
555 :
556 : template <typename T>
557 : inline bool
558 0 : IsGCThingValidAfterMovingGC(T* t)
559 : {
560 0 : return !IsInsideNursery(t) && !RelocationOverlay::isCellForwarded(t);
561 : }
562 :
563 : template <typename T>
564 : inline void
565 0 : CheckGCThingAfterMovingGC(T* t)
566 : {
567 0 : if (t)
568 0 : MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t));
569 0 : }
570 :
571 : template <typename T>
572 : inline void
573 0 : CheckGCThingAfterMovingGC(const ReadBarriered<T*>& t)
574 : {
575 0 : CheckGCThingAfterMovingGC(t.unbarrieredGet());
576 0 : }
577 :
578 : struct CheckValueAfterMovingGCFunctor : public VoidDefaultAdaptor<Value> {
579 : template <typename T> void operator()(T* t) { CheckGCThingAfterMovingGC(t); }
580 : };
581 :
582 : inline void
583 : CheckValueAfterMovingGC(const JS::Value& value)
584 : {
585 : DispatchTyped(CheckValueAfterMovingGCFunctor(), value);
586 : }
587 :
588 : #endif // JSGC_HASH_TABLE_CHECKS
589 :
590 : } /* namespace gc */
591 : } /* namespace js */
592 :
593 : #endif /* jsgcinlines_h */
|
