Line data Source code
1 : /* This Source Code Form is subject to the terms of the Mozilla Public
2 : * License, v. 2.0. If a copy of the MPL was not distributed with this
3 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4 :
5 : #include "CacheIndex.h"
6 :
7 : #include "CacheLog.h"
8 : #include "CacheFileIOManager.h"
9 : #include "CacheFileMetadata.h"
10 : #include "CacheIndexIterator.h"
11 : #include "CacheIndexContextIterator.h"
12 : #include "nsThreadUtils.h"
13 : #include "nsISimpleEnumerator.h"
14 : #include "nsIDirectoryEnumerator.h"
15 : #include "nsISizeOf.h"
16 : #include "nsPrintfCString.h"
17 : #include "mozilla/DebugOnly.h"
18 : #include "prinrval.h"
19 : #include "nsIFile.h"
20 : #include "nsITimer.h"
21 : #include "mozilla/AutoRestore.h"
22 : #include <algorithm>
23 : #include "mozilla/Telemetry.h"
24 : #include "mozilla/Unused.h"
25 :
26 :
27 : #define kMinUnwrittenChanges 300
28 : #define kMinDumpInterval 20000 // in milliseconds
29 : #define kMaxBufSize 16384
30 : #define kIndexVersion 0x00000005
31 : #define kUpdateIndexStartDelay 50000 // in milliseconds
32 :
33 : #define INDEX_NAME "index"
34 : #define TEMP_INDEX_NAME "index.tmp"
35 : #define JOURNAL_NAME "index.log"
36 :
37 : namespace mozilla {
38 : namespace net {
39 :
40 : namespace {
41 :
42 : class FrecencyComparator
43 : {
44 : public:
45 15 : bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
46 15 : if (!a || !b) {
47 8 : return false;
48 : }
49 :
50 7 : return a->mFrecency == b->mFrecency;
51 : }
52 29 : bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
53 : // Removed (=null) entries must be at the end of the array.
54 29 : if (!a) {
55 8 : return false;
56 : }
57 21 : if (!b) {
58 6 : return true;
59 : }
60 :
61 : // Place entries with frecency 0 at the end of the non-removed entries.
62 15 : if (a->mFrecency == 0) {
63 0 : return false;
64 : }
65 15 : if (b->mFrecency == 0) {
66 0 : return true;
67 : }
68 :
69 15 : return a->mFrecency < b->mFrecency;
70 : }
71 : };
72 :
73 : } // namespace
74 :
75 : /**
76 : * This helper class is responsible for keeping CacheIndex::mIndexStats and
77 : * CacheIndex::mFrecencyArray up to date.
78 : */
79 : class CacheIndexEntryAutoManage
80 : {
81 : public:
82 32 : CacheIndexEntryAutoManage(const SHA1Sum::Hash *aHash, CacheIndex *aIndex)
83 32 : : mIndex(aIndex)
84 : , mOldRecord(nullptr)
85 : , mOldFrecency(0)
86 : , mDoNotSearchInIndex(false)
87 32 : , mDoNotSearchInUpdates(false)
88 : {
89 32 : CacheIndex::sLock.AssertCurrentThreadOwns();
90 :
91 32 : mHash = aHash;
92 32 : const CacheIndexEntry *entry = FindEntry();
93 32 : mIndex->mIndexStats.BeforeChange(entry);
94 32 : if (entry && entry->IsInitialized() && !entry->IsRemoved()) {
95 22 : mOldRecord = entry->mRec;
96 22 : mOldFrecency = entry->mRec->mFrecency;
97 : }
98 32 : }
99 :
100 32 : ~CacheIndexEntryAutoManage()
101 32 : {
102 32 : CacheIndex::sLock.AssertCurrentThreadOwns();
103 :
104 32 : const CacheIndexEntry *entry = FindEntry();
105 32 : mIndex->mIndexStats.AfterChange(entry);
106 32 : if (!entry || !entry->IsInitialized() || entry->IsRemoved()) {
107 5 : entry = nullptr;
108 : }
109 :
110 32 : if (entry && !mOldRecord) {
111 5 : mIndex->mFrecencyArray.AppendRecord(entry->mRec);
112 5 : mIndex->AddRecordToIterators(entry->mRec);
113 27 : } else if (!entry && mOldRecord) {
114 0 : mIndex->mFrecencyArray.RemoveRecord(mOldRecord);
115 0 : mIndex->RemoveRecordFromIterators(mOldRecord);
116 27 : } else if (entry && mOldRecord) {
117 22 : if (entry->mRec != mOldRecord) {
118 : // record has a different address, we have to replace it
119 0 : mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec);
120 :
121 0 : if (entry->mRec->mFrecency == mOldFrecency) {
122 : // If frecency hasn't changed simply replace the pointer
123 0 : mIndex->mFrecencyArray.ReplaceRecord(mOldRecord, entry->mRec);
124 : } else {
125 : // Remove old pointer and insert the new one at the end of the array
126 0 : mIndex->mFrecencyArray.RemoveRecord(mOldRecord);
127 0 : mIndex->mFrecencyArray.AppendRecord(entry->mRec);
128 : }
129 22 : } else if (entry->mRec->mFrecency != mOldFrecency) {
130 : // Move the element at the end of the array
131 9 : mIndex->mFrecencyArray.RemoveRecord(entry->mRec);
132 9 : mIndex->mFrecencyArray.AppendRecord(entry->mRec);
133 : }
134 : } else {
135 : // both entries were removed or not initialized, do nothing
136 : }
137 32 : }
138 :
139 : // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries
140 : // while iterating. Destructor is called before the entry is removed. Caller
141 : // must call one of following methods to skip lookup in the hashtable.
142 0 : void DoNotSearchInIndex() { mDoNotSearchInIndex = true; }
143 0 : void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; }
144 :
145 : private:
146 64 : const CacheIndexEntry * FindEntry()
147 : {
148 64 : const CacheIndexEntry *entry = nullptr;
149 :
150 64 : switch (mIndex->mState) {
151 : case CacheIndex::READING:
152 : case CacheIndex::WRITING:
153 0 : if (!mDoNotSearchInUpdates) {
154 0 : entry = mIndex->mPendingUpdates.GetEntry(*mHash);
155 : }
156 : MOZ_FALLTHROUGH;
157 : case CacheIndex::BUILDING:
158 : case CacheIndex::UPDATING:
159 : case CacheIndex::READY:
160 64 : if (!entry && !mDoNotSearchInIndex) {
161 64 : entry = mIndex->mIndex.GetEntry(*mHash);
162 : }
163 64 : break;
164 : case CacheIndex::INITIAL:
165 : case CacheIndex::SHUTDOWN:
166 : default:
167 0 : MOZ_ASSERT(false, "Unexpected state!");
168 : }
169 :
170 64 : return entry;
171 : }
172 :
173 : const SHA1Sum::Hash *mHash;
174 : RefPtr<CacheIndex> mIndex;
175 : CacheIndexRecord *mOldRecord;
176 : uint32_t mOldFrecency;
177 : bool mDoNotSearchInIndex;
178 : bool mDoNotSearchInUpdates;
179 : };
180 :
181 : class FileOpenHelper : public CacheFileIOListener
182 : {
183 : public:
184 : NS_DECL_THREADSAFE_ISUPPORTS
185 :
186 3 : explicit FileOpenHelper(CacheIndex* aIndex)
187 3 : : mIndex(aIndex)
188 3 : , mCanceled(false)
189 3 : {}
190 :
191 2 : void Cancel() {
192 2 : CacheIndex::sLock.AssertCurrentThreadOwns();
193 2 : mCanceled = true;
194 2 : }
195 :
196 : private:
197 9 : virtual ~FileOpenHelper() {}
198 :
199 : NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) override;
200 0 : NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
201 : nsresult aResult) override {
202 0 : MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
203 : return NS_ERROR_UNEXPECTED;
204 : }
205 0 : NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf,
206 : nsresult aResult) override {
207 0 : MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
208 : return NS_ERROR_UNEXPECTED;
209 : }
210 0 : NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) override {
211 0 : MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
212 : return NS_ERROR_UNEXPECTED;
213 : }
214 0 : NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) override {
215 0 : MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
216 : return NS_ERROR_UNEXPECTED;
217 : }
218 0 : NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) override {
219 0 : MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
220 : return NS_ERROR_UNEXPECTED;
221 : }
222 :
223 : RefPtr<CacheIndex> mIndex;
224 : bool mCanceled;
225 : };
226 :
227 3 : NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle *aHandle,
228 : nsresult aResult)
229 : {
230 6 : StaticMutexAutoLock lock(CacheIndex::sLock);
231 :
232 3 : if (mCanceled) {
233 2 : if (aHandle) {
234 0 : CacheFileIOManager::DoomFile(aHandle, nullptr);
235 : }
236 :
237 2 : return NS_OK;
238 : }
239 :
240 1 : mIndex->OnFileOpenedInternal(this, aHandle, aResult);
241 :
242 1 : return NS_OK;
243 : }
244 :
245 21 : NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener);
246 :
247 :
248 3 : StaticRefPtr<CacheIndex> CacheIndex::gInstance;
249 3 : StaticMutex CacheIndex::sLock;
250 :
251 :
252 88 : NS_IMPL_ADDREF(CacheIndex)
253 87 : NS_IMPL_RELEASE(CacheIndex)
254 :
255 0 : NS_INTERFACE_MAP_BEGIN(CacheIndex)
256 0 : NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener)
257 0 : NS_INTERFACE_MAP_ENTRY(nsIRunnable)
258 0 : NS_INTERFACE_MAP_END_THREADSAFE
259 :
260 :
261 1 : CacheIndex::CacheIndex()
262 : : mState(INITIAL)
263 : , mShuttingDown(false)
264 : , mIndexNeedsUpdate(false)
265 : , mRemovingAll(false)
266 : , mIndexOnDiskIsValid(false)
267 : , mDontMarkIndexClean(false)
268 : , mIndexTimeStamp(0)
269 : , mUpdateEventPending(false)
270 : , mSkipEntries(0)
271 : , mProcessEntries(0)
272 : , mRWBuf(nullptr)
273 : , mRWBufSize(0)
274 : , mRWBufPos(0)
275 : , mRWPending(false)
276 : , mJournalReadSuccessfully(false)
277 1 : , mAsyncGetDiskConsumptionBlocked(false)
278 : {
279 1 : sLock.AssertCurrentThreadOwns();
280 1 : LOG(("CacheIndex::CacheIndex [this=%p]", this));
281 1 : MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!");
282 1 : }
283 :
284 0 : CacheIndex::~CacheIndex()
285 : {
286 0 : sLock.AssertCurrentThreadOwns();
287 0 : LOG(("CacheIndex::~CacheIndex [this=%p]", this));
288 :
289 0 : ReleaseBuffer();
290 0 : }
291 :
292 : // static
293 : nsresult
294 1 : CacheIndex::Init(nsIFile *aCacheDirectory)
295 : {
296 1 : LOG(("CacheIndex::Init()"));
297 :
298 1 : MOZ_ASSERT(NS_IsMainThread());
299 :
300 2 : StaticMutexAutoLock lock(sLock);
301 :
302 1 : if (gInstance) {
303 0 : return NS_ERROR_ALREADY_INITIALIZED;
304 : }
305 :
306 2 : RefPtr<CacheIndex> idx = new CacheIndex();
307 :
308 1 : nsresult rv = idx->InitInternal(aCacheDirectory);
309 1 : NS_ENSURE_SUCCESS(rv, rv);
310 :
311 1 : gInstance = idx.forget();
312 1 : return NS_OK;
313 : }
314 :
315 : nsresult
316 1 : CacheIndex::InitInternal(nsIFile *aCacheDirectory)
317 : {
318 : nsresult rv;
319 :
320 1 : rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory));
321 1 : NS_ENSURE_SUCCESS(rv, rv);
322 :
323 1 : mStartTime = TimeStamp::NowLoRes();
324 :
325 1 : ReadIndexFromDisk();
326 :
327 1 : return NS_OK;
328 : }
329 :
330 : // static
331 : nsresult
332 0 : CacheIndex::PreShutdown()
333 : {
334 0 : MOZ_ASSERT(NS_IsMainThread());
335 :
336 0 : StaticMutexAutoLock lock(sLock);
337 :
338 0 : LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance.get()));
339 :
340 : nsresult rv;
341 0 : RefPtr<CacheIndex> index = gInstance;
342 :
343 0 : if (!index) {
344 0 : return NS_ERROR_NOT_INITIALIZED;
345 : }
346 :
347 0 : LOG(("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
348 : "dontMarkIndexClean=%d]", index->mState, index->mIndexOnDiskIsValid,
349 : index->mDontMarkIndexClean));
350 :
351 0 : LOG(("CacheIndex::PreShutdown() - Closing iterators."));
352 0 : for (uint32_t i = 0; i < index->mIterators.Length(); ) {
353 0 : rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE);
354 0 : if (NS_FAILED(rv)) {
355 : // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
356 : // it returns success.
357 0 : LOG(("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
358 : "[rv=0x%08" PRIx32 "]", index->mIterators[i], static_cast<uint32_t>(rv)));
359 0 : i++;
360 : }
361 : }
362 :
363 0 : index->mShuttingDown = true;
364 :
365 0 : if (index->mState == READY) {
366 0 : return NS_OK; // nothing to do
367 : }
368 :
369 0 : nsCOMPtr<nsIRunnable> event;
370 0 : event = NewRunnableMethod("net::CacheIndex::PreShutdownInternal",
371 : index,
372 0 : &CacheIndex::PreShutdownInternal);
373 :
374 0 : nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
375 0 : MOZ_ASSERT(ioTarget);
376 :
377 : // PreShutdownInternal() will be executed before any queued event on INDEX
378 : // level. That's OK since we don't want to wait for any operation in progess.
379 0 : rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
380 0 : if (NS_FAILED(rv)) {
381 0 : NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
382 0 : LOG(("CacheIndex::PreShutdown() - Can't dispatch event" ));
383 0 : return rv;
384 : }
385 :
386 0 : return NS_OK;
387 : }
388 :
389 : void
390 0 : CacheIndex::PreShutdownInternal()
391 : {
392 0 : StaticMutexAutoLock lock(sLock);
393 :
394 0 : LOG(("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
395 : "dontMarkIndexClean=%d]", mState, mIndexOnDiskIsValid,
396 : mDontMarkIndexClean));
397 :
398 0 : MOZ_ASSERT(mShuttingDown);
399 :
400 0 : if (mUpdateTimer) {
401 0 : mUpdateTimer->Cancel();
402 0 : mUpdateTimer = nullptr;
403 : }
404 :
405 0 : switch (mState) {
406 : case WRITING:
407 0 : FinishWrite(false);
408 0 : break;
409 : case READY:
410 : // nothing to do, write the journal in Shutdown()
411 0 : break;
412 : case READING:
413 0 : FinishRead(false);
414 0 : break;
415 : case BUILDING:
416 : case UPDATING:
417 0 : FinishUpdate(false);
418 0 : break;
419 : default:
420 0 : MOZ_ASSERT(false, "Implement me!");
421 : }
422 :
423 : // We should end up in READY state
424 0 : MOZ_ASSERT(mState == READY);
425 0 : }
426 :
427 : // static
428 : nsresult
429 0 : CacheIndex::Shutdown()
430 : {
431 0 : MOZ_ASSERT(NS_IsMainThread());
432 :
433 0 : StaticMutexAutoLock lock(sLock);
434 :
435 0 : LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance.get()));
436 :
437 0 : RefPtr<CacheIndex> index = gInstance.forget();
438 :
439 0 : if (!index) {
440 0 : return NS_ERROR_NOT_INITIALIZED;
441 : }
442 :
443 0 : bool sanitize = CacheObserver::ClearCacheOnShutdown();
444 :
445 0 : LOG(("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
446 : "dontMarkIndexClean=%d, sanitize=%d]", index->mState,
447 : index->mIndexOnDiskIsValid, index->mDontMarkIndexClean, sanitize));
448 :
449 0 : MOZ_ASSERT(index->mShuttingDown);
450 :
451 0 : EState oldState = index->mState;
452 0 : index->ChangeState(SHUTDOWN);
453 :
454 0 : if (oldState != READY) {
455 0 : LOG(("CacheIndex::Shutdown() - Unexpected state. Did posting of "
456 : "PreShutdownInternal() fail?"));
457 : }
458 :
459 0 : switch (oldState) {
460 : case WRITING:
461 0 : index->FinishWrite(false);
462 : MOZ_FALLTHROUGH;
463 : case READY:
464 0 : if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) {
465 0 : if (!sanitize && NS_FAILED(index->WriteLogToDisk())) {
466 0 : index->RemoveJournalAndTempFile();
467 : }
468 : } else {
469 0 : index->RemoveJournalAndTempFile();
470 : }
471 0 : break;
472 : case READING:
473 0 : index->FinishRead(false);
474 0 : break;
475 : case BUILDING:
476 : case UPDATING:
477 0 : index->FinishUpdate(false);
478 0 : break;
479 : default:
480 0 : MOZ_ASSERT(false, "Unexpected state!");
481 : }
482 :
483 0 : if (sanitize) {
484 0 : index->RemoveAllIndexFiles();
485 : }
486 :
487 0 : return NS_OK;
488 : }
489 :
490 : // static
491 : nsresult
492 2 : CacheIndex::AddEntry(const SHA1Sum::Hash *aHash)
493 : {
494 2 : LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash)));
495 :
496 2 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
497 :
498 4 : StaticMutexAutoLock lock(sLock);
499 :
500 4 : RefPtr<CacheIndex> index = gInstance;
501 :
502 2 : if (!index) {
503 0 : return NS_ERROR_NOT_INITIALIZED;
504 : }
505 :
506 2 : if (!index->IsIndexUsable()) {
507 0 : return NS_ERROR_NOT_AVAILABLE;
508 : }
509 :
510 : // Getters in CacheIndexStats assert when mStateLogged is true since the
511 : // information is incomplete between calls to BeforeChange() and AfterChange()
512 : // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
513 : // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
514 2 : bool updateIfNonFreshEntriesExist = false;
515 :
516 : {
517 4 : CacheIndexEntryAutoManage entryMng(aHash, index);
518 :
519 2 : CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
520 2 : bool entryRemoved = entry && entry->IsRemoved();
521 2 : CacheIndexEntryUpdate *updated = nullptr;
522 :
523 4 : if (index->mState == READY || index->mState == UPDATING ||
524 2 : index->mState == BUILDING) {
525 2 : MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
526 :
527 2 : if (entry && !entryRemoved) {
528 : // Found entry in index that shouldn't exist.
529 :
530 0 : if (entry->IsFresh()) {
531 : // Someone removed the file on disk while FF is running. Update
532 : // process can fix only non-fresh entries (i.e. entries that were not
533 : // added within this session). Start update only if we have such
534 : // entries.
535 : //
536 : // TODO: This should be very rare problem. If it turns out not to be
537 : // true, change the update process so that it also iterates all
538 : // initialized non-empty entries and checks whether the file exists.
539 :
540 0 : LOG(("CacheIndex::AddEntry() - Cache file was removed outside FF "
541 : "process!"));
542 :
543 0 : updateIfNonFreshEntriesExist = true;
544 0 : } else if (index->mState == READY) {
545 : // Index is outdated, update it.
546 0 : LOG(("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
547 : "update is needed"));
548 0 : index->mIndexNeedsUpdate = true;
549 : } else {
550 : // We cannot be here when building index since all entries are fresh
551 : // during building.
552 0 : MOZ_ASSERT(index->mState == UPDATING);
553 : }
554 : }
555 :
556 2 : if (!entry) {
557 2 : entry = index->mIndex.PutEntry(*aHash);
558 : }
559 : } else { // WRITING, READING
560 0 : updated = index->mPendingUpdates.GetEntry(*aHash);
561 0 : bool updatedRemoved = updated && updated->IsRemoved();
562 :
563 0 : if ((updated && !updatedRemoved) ||
564 0 : (!updated && entry && !entryRemoved && entry->IsFresh())) {
565 : // Fresh entry found, so the file was removed outside FF
566 0 : LOG(("CacheIndex::AddEntry() - Cache file was removed outside FF "
567 : "process!"));
568 :
569 0 : updateIfNonFreshEntriesExist = true;
570 0 : } else if (!updated && entry && !entryRemoved) {
571 0 : if (index->mState == WRITING) {
572 0 : LOG(("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
573 : "update is needed"));
574 0 : index->mIndexNeedsUpdate = true;
575 : }
576 : // Ignore if state is READING since the index information is partial
577 : }
578 :
579 0 : updated = index->mPendingUpdates.PutEntry(*aHash);
580 : }
581 :
582 2 : if (updated) {
583 0 : updated->InitNew();
584 0 : updated->MarkDirty();
585 0 : updated->MarkFresh();
586 : } else {
587 2 : entry->InitNew();
588 2 : entry->MarkDirty();
589 2 : entry->MarkFresh();
590 : }
591 : }
592 :
593 2 : if (updateIfNonFreshEntriesExist &&
594 0 : index->mIndexStats.Count() != index->mIndexStats.Fresh()) {
595 0 : index->mIndexNeedsUpdate = true;
596 : }
597 :
598 2 : index->StartUpdatingIndexIfNeeded();
599 2 : index->WriteIndexToDiskIfNeeded();
600 :
601 2 : return NS_OK;
602 : }
603 :
604 : // static
605 : nsresult
606 3 : CacheIndex::EnsureEntryExists(const SHA1Sum::Hash *aHash)
607 : {
608 3 : LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
609 : LOGSHA1(aHash)));
610 :
611 3 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
612 :
613 6 : StaticMutexAutoLock lock(sLock);
614 :
615 6 : RefPtr<CacheIndex> index = gInstance;
616 :
617 3 : if (!index) {
618 0 : return NS_ERROR_NOT_INITIALIZED;
619 : }
620 :
621 3 : if (!index->IsIndexUsable()) {
622 0 : return NS_ERROR_NOT_AVAILABLE;
623 : }
624 :
625 : {
626 6 : CacheIndexEntryAutoManage entryMng(aHash, index);
627 :
628 3 : CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
629 3 : bool entryRemoved = entry && entry->IsRemoved();
630 :
631 6 : if (index->mState == READY || index->mState == UPDATING ||
632 3 : index->mState == BUILDING) {
633 3 : MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
634 :
635 3 : if (!entry || entryRemoved) {
636 3 : if (entryRemoved && entry->IsFresh()) {
637 : // This could happen only if somebody copies files to the entries
638 : // directory while FF is running.
639 0 : LOG(("CacheIndex::EnsureEntryExists() - Cache file was added outside "
640 : "FF process! Update is needed."));
641 0 : index->mIndexNeedsUpdate = true;
642 3 : } else if (index->mState == READY ||
643 0 : (entryRemoved && !entry->IsFresh())) {
644 : // Removed non-fresh entries can be present as a result of
645 : // MergeJournal()
646 0 : LOG(("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
647 : " exist, update is needed"));
648 0 : index->mIndexNeedsUpdate = true;
649 : }
650 :
651 3 : if (!entry) {
652 3 : entry = index->mIndex.PutEntry(*aHash);
653 : }
654 3 : entry->InitNew();
655 3 : entry->MarkDirty();
656 : }
657 3 : entry->MarkFresh();
658 : } else { // WRITING, READING
659 0 : CacheIndexEntryUpdate *updated = index->mPendingUpdates.GetEntry(*aHash);
660 0 : bool updatedRemoved = updated && updated->IsRemoved();
661 :
662 0 : if (updatedRemoved ||
663 0 : (!updated && entryRemoved && entry->IsFresh())) {
664 : // Fresh information about missing entry found. This could happen only
665 : // if somebody copies files to the entries directory while FF is running.
666 0 : LOG(("CacheIndex::EnsureEntryExists() - Cache file was added outside "
667 : "FF process! Update is needed."));
668 0 : index->mIndexNeedsUpdate = true;
669 0 : } else if (!updated && (!entry || entryRemoved)) {
670 0 : if (index->mState == WRITING) {
671 0 : LOG(("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
672 : " exist, update is needed"));
673 0 : index->mIndexNeedsUpdate = true;
674 : }
675 : // Ignore if state is READING since the index information is partial
676 : }
677 :
678 : // We don't need entryRemoved and updatedRemoved info anymore
679 0 : if (entryRemoved) entry = nullptr;
680 0 : if (updatedRemoved) updated = nullptr;
681 :
682 0 : if (updated) {
683 0 : updated->MarkFresh();
684 : } else {
685 0 : if (!entry) {
686 : // Create a new entry
687 0 : updated = index->mPendingUpdates.PutEntry(*aHash);
688 0 : updated->InitNew();
689 0 : updated->MarkFresh();
690 0 : updated->MarkDirty();
691 : } else {
692 0 : if (!entry->IsFresh()) {
693 : // To mark the entry fresh we must make a copy of index entry
694 : // since the index is read-only.
695 0 : updated = index->mPendingUpdates.PutEntry(*aHash);
696 0 : *updated = *entry;
697 0 : updated->MarkFresh();
698 : }
699 : }
700 : }
701 : }
702 : }
703 :
704 3 : index->StartUpdatingIndexIfNeeded();
705 3 : index->WriteIndexToDiskIfNeeded();
706 :
707 3 : return NS_OK;
708 : }
709 :
710 : // static
711 : nsresult
712 5 : CacheIndex::InitEntry(const SHA1Sum::Hash *aHash,
713 : OriginAttrsHash aOriginAttrsHash,
714 : bool aAnonymous,
715 : bool aPinned)
716 : {
717 5 : LOG(("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, "
718 : "originAttrsHash=%" PRIx64 ", anonymous=%d, pinned=%d]", LOGSHA1(aHash),
719 : aOriginAttrsHash, aAnonymous, aPinned));
720 :
721 5 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
722 :
723 10 : StaticMutexAutoLock lock(sLock);
724 :
725 10 : RefPtr<CacheIndex> index = gInstance;
726 :
727 5 : if (!index) {
728 0 : return NS_ERROR_NOT_INITIALIZED;
729 : }
730 :
731 5 : if (!index->IsIndexUsable()) {
732 0 : return NS_ERROR_NOT_AVAILABLE;
733 : }
734 :
735 : {
736 10 : CacheIndexEntryAutoManage entryMng(aHash, index);
737 :
738 5 : CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
739 5 : CacheIndexEntryUpdate *updated = nullptr;
740 5 : bool reinitEntry = false;
741 :
742 5 : if (entry && entry->IsRemoved()) {
743 0 : entry = nullptr;
744 : }
745 :
746 10 : if (index->mState == READY || index->mState == UPDATING ||
747 5 : index->mState == BUILDING) {
748 5 : MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
749 5 : MOZ_ASSERT(entry);
750 5 : MOZ_ASSERT(entry->IsFresh());
751 :
752 5 : if (!entry) {
753 0 : LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
754 0 : NS_WARNING(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
755 0 : return NS_ERROR_UNEXPECTED;
756 : }
757 :
758 5 : if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
759 0 : index->mIndexNeedsUpdate = true; // TODO Does this really help in case of collision?
760 0 : reinitEntry = true;
761 : } else {
762 5 : if (entry->IsInitialized()) {
763 0 : return NS_OK;
764 : }
765 : }
766 : } else {
767 0 : updated = index->mPendingUpdates.GetEntry(*aHash);
768 0 : DebugOnly<bool> removed = updated && updated->IsRemoved();
769 :
770 0 : MOZ_ASSERT(updated || !removed);
771 0 : MOZ_ASSERT(updated || entry);
772 :
773 0 : if (!updated && !entry) {
774 0 : LOG(("CacheIndex::InitEntry() - Entry was found neither in mIndex nor "
775 : "in mPendingUpdates!"));
776 : NS_WARNING(("CacheIndex::InitEntry() - Entry was found neither in "
777 0 : "mIndex nor in mPendingUpdates!"));
778 0 : return NS_ERROR_UNEXPECTED;
779 : }
780 :
781 0 : if (updated) {
782 0 : MOZ_ASSERT(updated->IsFresh());
783 :
784 0 : if (IsCollision(updated, aOriginAttrsHash, aAnonymous)) {
785 0 : index->mIndexNeedsUpdate = true;
786 0 : reinitEntry = true;
787 : } else {
788 0 : if (updated->IsInitialized()) {
789 0 : return NS_OK;
790 : }
791 : }
792 : } else {
793 0 : MOZ_ASSERT(entry->IsFresh());
794 :
795 0 : if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
796 0 : index->mIndexNeedsUpdate = true;
797 0 : reinitEntry = true;
798 : } else {
799 0 : if (entry->IsInitialized()) {
800 0 : return NS_OK;
801 : }
802 : }
803 :
804 : // make a copy of a read-only entry
805 0 : updated = index->mPendingUpdates.PutEntry(*aHash);
806 0 : *updated = *entry;
807 : }
808 : }
809 :
810 5 : if (reinitEntry) {
811 : // There is a collision and we are going to rewrite this entry. Initialize
812 : // it as a new entry.
813 0 : if (updated) {
814 0 : updated->InitNew();
815 0 : updated->MarkFresh();
816 : } else {
817 0 : entry->InitNew();
818 0 : entry->MarkFresh();
819 : }
820 : }
821 :
822 5 : if (updated) {
823 0 : updated->Init(aOriginAttrsHash, aAnonymous, aPinned);
824 0 : updated->MarkDirty();
825 : } else {
826 5 : entry->Init(aOriginAttrsHash, aAnonymous, aPinned);
827 5 : entry->MarkDirty();
828 : }
829 : }
830 :
831 5 : index->StartUpdatingIndexIfNeeded();
832 5 : index->WriteIndexToDiskIfNeeded();
833 :
834 5 : return NS_OK;
835 : }
836 :
837 : // static
838 : nsresult
839 0 : CacheIndex::RemoveEntry(const SHA1Sum::Hash *aHash)
840 : {
841 0 : LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
842 : LOGSHA1(aHash)));
843 :
844 0 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
845 :
846 0 : StaticMutexAutoLock lock(sLock);
847 :
848 0 : RefPtr<CacheIndex> index = gInstance;
849 :
850 0 : if (!index) {
851 0 : return NS_ERROR_NOT_INITIALIZED;
852 : }
853 :
854 0 : if (!index->IsIndexUsable()) {
855 0 : return NS_ERROR_NOT_AVAILABLE;
856 : }
857 :
858 : {
859 0 : CacheIndexEntryAutoManage entryMng(aHash, index);
860 :
861 0 : CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
862 0 : bool entryRemoved = entry && entry->IsRemoved();
863 :
864 0 : if (index->mState == READY || index->mState == UPDATING ||
865 0 : index->mState == BUILDING) {
866 0 : MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
867 :
868 0 : if (!entry || entryRemoved) {
869 0 : if (entryRemoved && entry->IsFresh()) {
870 : // This could happen only if somebody copies files to the entries
871 : // directory while FF is running.
872 0 : LOG(("CacheIndex::RemoveEntry() - Cache file was added outside FF "
873 : "process! Update is needed."));
874 0 : index->mIndexNeedsUpdate = true;
875 0 : } else if (index->mState == READY ||
876 0 : (entryRemoved && !entry->IsFresh())) {
877 : // Removed non-fresh entries can be present as a result of
878 : // MergeJournal()
879 0 : LOG(("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
880 : ", update is needed"));
881 0 : index->mIndexNeedsUpdate = true;
882 : }
883 : } else {
884 0 : if (entry) {
885 0 : if (!entry->IsDirty() && entry->IsFileEmpty()) {
886 0 : index->mIndex.RemoveEntry(entry);
887 0 : entry = nullptr;
888 : } else {
889 0 : entry->MarkRemoved();
890 0 : entry->MarkDirty();
891 0 : entry->MarkFresh();
892 : }
893 : }
894 : }
895 : } else { // WRITING, READING
896 0 : CacheIndexEntryUpdate *updated = index->mPendingUpdates.GetEntry(*aHash);
897 0 : bool updatedRemoved = updated && updated->IsRemoved();
898 :
899 0 : if (updatedRemoved ||
900 0 : (!updated && entryRemoved && entry->IsFresh())) {
901 : // Fresh information about missing entry found. This could happen only
902 : // if somebody copies files to the entries directory while FF is running.
903 0 : LOG(("CacheIndex::RemoveEntry() - Cache file was added outside FF "
904 : "process! Update is needed."));
905 0 : index->mIndexNeedsUpdate = true;
906 0 : } else if (!updated && (!entry || entryRemoved)) {
907 0 : if (index->mState == WRITING) {
908 0 : LOG(("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
909 : ", update is needed"));
910 0 : index->mIndexNeedsUpdate = true;
911 : }
912 : // Ignore if state is READING since the index information is partial
913 : }
914 :
915 0 : if (!updated) {
916 0 : updated = index->mPendingUpdates.PutEntry(*aHash);
917 0 : updated->InitNew();
918 : }
919 :
920 0 : updated->MarkRemoved();
921 0 : updated->MarkDirty();
922 0 : updated->MarkFresh();
923 : }
924 : }
925 :
926 0 : index->StartUpdatingIndexIfNeeded();
927 0 : index->WriteIndexToDiskIfNeeded();
928 :
929 0 : return NS_OK;
930 : }
931 :
932 : // static
933 : nsresult
934 22 : CacheIndex::UpdateEntry(const SHA1Sum::Hash *aHash,
935 : const uint32_t *aFrecency,
936 : const uint32_t *aExpirationTime,
937 : const bool *aHasAltData,
938 : const uint16_t *aOnStartTime,
939 : const uint16_t *aOnStopTime,
940 : const uint32_t *aSize)
941 : {
942 22 : LOG(("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
943 : "frecency=%s, expirationTime=%s, hasAltData=%s, onStartTime=%s, "
944 : "onStopTime=%s, size=%s]", LOGSHA1(aHash),
945 : aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "",
946 : aExpirationTime ? nsPrintfCString("%u", *aExpirationTime).get() : "",
947 : aHasAltData ? (*aHasAltData ? "true" : "false") : "",
948 : aOnStartTime ? nsPrintfCString("%u", *aOnStartTime).get() : "",
949 : aOnStopTime ? nsPrintfCString("%u", *aOnStopTime).get() : "",
950 : aSize ? nsPrintfCString("%u", *aSize).get() : ""));
951 :
952 22 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
953 :
954 44 : StaticMutexAutoLock lock(sLock);
955 :
956 44 : RefPtr<CacheIndex> index = gInstance;
957 :
958 22 : if (!index) {
959 0 : return NS_ERROR_NOT_INITIALIZED;
960 : }
961 :
962 22 : if (!index->IsIndexUsable()) {
963 0 : return NS_ERROR_NOT_AVAILABLE;
964 : }
965 :
966 : {
967 40 : CacheIndexEntryAutoManage entryMng(aHash, index);
968 :
969 22 : CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
970 :
971 22 : if (entry && entry->IsRemoved()) {
972 0 : entry = nullptr;
973 : }
974 :
975 44 : if (index->mState == READY || index->mState == UPDATING ||
976 22 : index->mState == BUILDING) {
977 22 : MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
978 22 : MOZ_ASSERT(entry);
979 :
980 22 : if (!entry) {
981 0 : LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
982 0 : NS_WARNING(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
983 0 : return NS_ERROR_UNEXPECTED;
984 : }
985 :
986 22 : if (!HasEntryChanged(entry, aFrecency, aExpirationTime, aHasAltData,
987 : aOnStartTime, aOnStopTime, aSize)) {
988 4 : return NS_OK;
989 : }
990 :
991 18 : MOZ_ASSERT(entry->IsFresh());
992 18 : MOZ_ASSERT(entry->IsInitialized());
993 18 : entry->MarkDirty();
994 :
995 18 : if (aFrecency) {
996 9 : entry->SetFrecency(*aFrecency);
997 : }
998 :
999 18 : if (aExpirationTime) {
1000 5 : entry->SetExpirationTime(*aExpirationTime);
1001 : }
1002 :
1003 18 : if (aHasAltData) {
1004 3 : entry->SetHasAltData(*aHasAltData);
1005 : }
1006 :
1007 18 : if (aOnStartTime) {
1008 4 : entry->SetOnStartTime(*aOnStartTime);
1009 : }
1010 :
1011 18 : if (aOnStopTime) {
1012 4 : entry->SetOnStopTime(*aOnStopTime);
1013 : }
1014 :
1015 18 : if (aSize) {
1016 6 : entry->SetFileSize(*aSize);
1017 : }
1018 : } else {
1019 0 : CacheIndexEntryUpdate *updated = index->mPendingUpdates.GetEntry(*aHash);
1020 0 : DebugOnly<bool> removed = updated && updated->IsRemoved();
1021 :
1022 0 : MOZ_ASSERT(updated || !removed);
1023 0 : MOZ_ASSERT(updated || entry);
1024 :
1025 0 : if (!updated) {
1026 0 : if (!entry) {
1027 0 : LOG(("CacheIndex::UpdateEntry() - Entry was found neither in mIndex "
1028 : "nor in mPendingUpdates!"));
1029 : NS_WARNING(("CacheIndex::UpdateEntry() - Entry was found neither in "
1030 0 : "mIndex nor in mPendingUpdates!"));
1031 0 : return NS_ERROR_UNEXPECTED;
1032 : }
1033 :
1034 : // make a copy of a read-only entry
1035 0 : updated = index->mPendingUpdates.PutEntry(*aHash);
1036 0 : *updated = *entry;
1037 : }
1038 :
1039 0 : MOZ_ASSERT(updated->IsFresh());
1040 0 : MOZ_ASSERT(updated->IsInitialized());
1041 0 : updated->MarkDirty();
1042 :
1043 0 : if (aFrecency) {
1044 0 : updated->SetFrecency(*aFrecency);
1045 : }
1046 :
1047 0 : if (aExpirationTime) {
1048 0 : updated->SetExpirationTime(*aExpirationTime);
1049 : }
1050 :
1051 0 : if (aHasAltData) {
1052 0 : updated->SetHasAltData(*aHasAltData);
1053 : }
1054 :
1055 0 : if (aOnStartTime) {
1056 0 : updated->SetOnStartTime(*aOnStartTime);
1057 : }
1058 :
1059 0 : if (aOnStopTime) {
1060 0 : updated->SetOnStopTime(*aOnStopTime);
1061 : }
1062 :
1063 0 : if (aSize) {
1064 0 : updated->SetFileSize(*aSize);
1065 : }
1066 : }
1067 : }
1068 :
1069 18 : index->WriteIndexToDiskIfNeeded();
1070 :
1071 18 : return NS_OK;
1072 : }
1073 :
1074 : // static
1075 : nsresult
1076 0 : CacheIndex::RemoveAll()
1077 : {
1078 0 : LOG(("CacheIndex::RemoveAll()"));
1079 :
1080 0 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1081 :
1082 0 : nsCOMPtr<nsIFile> file;
1083 :
1084 : {
1085 0 : StaticMutexAutoLock lock(sLock);
1086 :
1087 0 : RefPtr<CacheIndex> index = gInstance;
1088 :
1089 0 : if (!index) {
1090 0 : return NS_ERROR_NOT_INITIALIZED;
1091 : }
1092 :
1093 0 : MOZ_ASSERT(!index->mRemovingAll);
1094 :
1095 0 : if (!index->IsIndexUsable()) {
1096 0 : return NS_ERROR_NOT_AVAILABLE;
1097 : }
1098 :
1099 0 : AutoRestore<bool> saveRemovingAll(index->mRemovingAll);
1100 0 : index->mRemovingAll = true;
1101 :
1102 : // Doom index and journal handles but don't null them out since this will be
1103 : // done in FinishWrite/FinishRead methods.
1104 0 : if (index->mIndexHandle) {
1105 0 : CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr);
1106 : } else {
1107 : // We don't have a handle to index file, so get the file here, but delete
1108 : // it outside the lock. Ignore the result since this is not fatal.
1109 0 : index->GetFile(NS_LITERAL_CSTRING(INDEX_NAME), getter_AddRefs(file));
1110 : }
1111 :
1112 0 : if (index->mJournalHandle) {
1113 0 : CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr);
1114 : }
1115 :
1116 0 : switch (index->mState) {
1117 : case WRITING:
1118 0 : index->FinishWrite(false);
1119 0 : break;
1120 : case READY:
1121 : // nothing to do
1122 0 : break;
1123 : case READING:
1124 0 : index->FinishRead(false);
1125 0 : break;
1126 : case BUILDING:
1127 : case UPDATING:
1128 0 : index->FinishUpdate(false);
1129 0 : break;
1130 : default:
1131 0 : MOZ_ASSERT(false, "Unexpected state!");
1132 : }
1133 :
1134 : // We should end up in READY state
1135 0 : MOZ_ASSERT(index->mState == READY);
1136 :
1137 : // There should not be any handle
1138 0 : MOZ_ASSERT(!index->mIndexHandle);
1139 0 : MOZ_ASSERT(!index->mJournalHandle);
1140 :
1141 0 : index->mIndexOnDiskIsValid = false;
1142 0 : index->mIndexNeedsUpdate = false;
1143 :
1144 0 : index->mIndexStats.Clear();
1145 0 : index->mFrecencyArray.Clear();
1146 0 : index->mIndex.Clear();
1147 :
1148 0 : for (uint32_t i = 0; i < index->mIterators.Length(); ) {
1149 0 : nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE);
1150 0 : if (NS_FAILED(rv)) {
1151 : // CacheIndexIterator::CloseInternal() removes itself from mIterators
1152 : // iff it returns success.
1153 0 : LOG(("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
1154 : "[rv=0x%08" PRIx32 "]", index->mIterators[i], static_cast<uint32_t>(rv)));
1155 0 : i++;
1156 : }
1157 : }
1158 : }
1159 :
1160 0 : if (file) {
1161 : // Ignore the result. The file might not exist and the failure is not fatal.
1162 0 : file->Remove(false);
1163 : }
1164 :
1165 0 : return NS_OK;
1166 : }
1167 :
1168 : // static
1169 : nsresult
1170 10 : CacheIndex::HasEntry(const nsACString &aKey, EntryStatus *_retval,
1171 : const std::function<void(const CacheIndexEntry*)> &aCB)
1172 : {
1173 10 : LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get()));
1174 :
1175 10 : SHA1Sum sum;
1176 : SHA1Sum::Hash hash;
1177 10 : sum.update(aKey.BeginReading(), aKey.Length());
1178 10 : sum.finish(hash);
1179 :
1180 10 : return HasEntry(hash, _retval, aCB);
1181 : }
1182 :
1183 : // static
1184 : nsresult
1185 10 : CacheIndex::HasEntry(const SHA1Sum::Hash &hash, EntryStatus *_retval,
1186 : const std::function<void(const CacheIndexEntry*)> &aCB)
1187 : {
1188 20 : StaticMutexAutoLock lock(sLock);
1189 :
1190 20 : RefPtr<CacheIndex> index = gInstance;
1191 :
1192 10 : if (!index) {
1193 0 : return NS_ERROR_NOT_INITIALIZED;
1194 : }
1195 :
1196 10 : if (!index->IsIndexUsable()) {
1197 0 : return NS_ERROR_NOT_AVAILABLE;
1198 : }
1199 :
1200 10 : const CacheIndexEntry *entry = nullptr;
1201 :
1202 10 : switch (index->mState) {
1203 : case READING:
1204 : case WRITING:
1205 0 : entry = index->mPendingUpdates.GetEntry(hash);
1206 : MOZ_FALLTHROUGH;
1207 : case BUILDING:
1208 : case UPDATING:
1209 : case READY:
1210 10 : if (!entry) {
1211 10 : entry = index->mIndex.GetEntry(hash);
1212 : }
1213 10 : break;
1214 : case INITIAL:
1215 : case SHUTDOWN:
1216 0 : MOZ_ASSERT(false, "Unexpected state!");
1217 : }
1218 :
1219 10 : if (!entry) {
1220 8 : if (index->mState == READY || index->mState == WRITING) {
1221 0 : *_retval = DOES_NOT_EXIST;
1222 : } else {
1223 8 : *_retval = DO_NOT_KNOW;
1224 : }
1225 : } else {
1226 2 : if (entry->IsRemoved()) {
1227 0 : if (entry->IsFresh()) {
1228 0 : *_retval = DOES_NOT_EXIST;
1229 : } else {
1230 0 : *_retval = DO_NOT_KNOW;
1231 : }
1232 : } else {
1233 2 : *_retval = EXISTS;
1234 2 : if (aCB) {
1235 2 : aCB(entry);
1236 : }
1237 : }
1238 : }
1239 :
1240 10 : LOG(("CacheIndex::HasEntry() - result is %u", *_retval));
1241 10 : return NS_OK;
1242 : }
1243 :
1244 : // static
1245 : nsresult
1246 0 : CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries, SHA1Sum::Hash *aHash, uint32_t *aCnt)
1247 : {
1248 0 : LOG(("CacheIndex::GetEntryForEviction()"));
1249 :
1250 0 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1251 :
1252 0 : StaticMutexAutoLock lock(sLock);
1253 :
1254 0 : RefPtr<CacheIndex> index = gInstance;
1255 :
1256 0 : if (!index)
1257 0 : return NS_ERROR_NOT_INITIALIZED;
1258 :
1259 0 : if (!index->IsIndexUsable()) {
1260 0 : return NS_ERROR_NOT_AVAILABLE;
1261 : }
1262 :
1263 : SHA1Sum::Hash hash;
1264 0 : CacheIndexRecord *foundRecord = nullptr;
1265 0 : uint32_t skipped = 0;
1266 :
1267 : // find first non-forced valid and unpinned entry with the lowest frecency
1268 0 : index->mFrecencyArray.SortIfNeeded();
1269 :
1270 0 : for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1271 0 : CacheIndexRecord *rec = iter.Get();
1272 :
1273 0 : memcpy(&hash, rec->mHash, sizeof(SHA1Sum::Hash));
1274 :
1275 0 : ++skipped;
1276 :
1277 0 : if (IsForcedValidEntry(&hash)) {
1278 0 : continue;
1279 : }
1280 :
1281 0 : if (CacheIndexEntry::IsPinned(rec)) {
1282 0 : continue;
1283 : }
1284 :
1285 0 : if (aIgnoreEmptyEntries && !CacheIndexEntry::GetFileSize(rec)) {
1286 0 : continue;
1287 : }
1288 :
1289 0 : --skipped;
1290 0 : foundRecord = rec;
1291 0 : break;
1292 : }
1293 :
1294 0 : if (!foundRecord)
1295 0 : return NS_ERROR_NOT_AVAILABLE;
1296 :
1297 0 : *aCnt = skipped;
1298 :
1299 0 : LOG(("CacheIndex::GetEntryForEviction() - returning entry from frecency "
1300 : "array [hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u]",
1301 : LOGSHA1(&hash), *aCnt, foundRecord->mFrecency));
1302 :
1303 0 : memcpy(aHash, &hash, sizeof(SHA1Sum::Hash));
1304 :
1305 0 : return NS_OK;
1306 : }
1307 :
1308 :
1309 : // static
1310 0 : bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash *aHash)
1311 : {
1312 0 : RefPtr<CacheFileHandle> handle;
1313 :
1314 0 : CacheFileIOManager::gInstance->mHandles.GetHandle(
1315 0 : aHash, getter_AddRefs(handle));
1316 :
1317 0 : if (!handle)
1318 0 : return false;
1319 :
1320 0 : nsCString hashKey = handle->Key();
1321 0 : return CacheStorageService::Self()->IsForcedValidEntry(hashKey);
1322 : }
1323 :
1324 :
1325 : // static
1326 : nsresult
1327 4 : CacheIndex::GetCacheSize(uint32_t *_retval)
1328 : {
1329 4 : LOG(("CacheIndex::GetCacheSize()"));
1330 :
1331 8 : StaticMutexAutoLock lock(sLock);
1332 :
1333 8 : RefPtr<CacheIndex> index = gInstance;
1334 :
1335 4 : if (!index)
1336 0 : return NS_ERROR_NOT_INITIALIZED;
1337 :
1338 4 : if (!index->IsIndexUsable()) {
1339 0 : return NS_ERROR_NOT_AVAILABLE;
1340 : }
1341 :
1342 4 : *_retval = index->mIndexStats.Size();
1343 4 : LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval));
1344 4 : return NS_OK;
1345 : }
1346 :
1347 : // static
1348 : nsresult
1349 0 : CacheIndex::GetEntryFileCount(uint32_t *_retval)
1350 : {
1351 0 : LOG(("CacheIndex::GetEntryFileCount()"));
1352 :
1353 0 : StaticMutexAutoLock lock(sLock);
1354 :
1355 0 : RefPtr<CacheIndex> index = gInstance;
1356 :
1357 0 : if (!index) {
1358 0 : return NS_ERROR_NOT_INITIALIZED;
1359 : }
1360 :
1361 0 : if (!index->IsIndexUsable()) {
1362 0 : return NS_ERROR_NOT_AVAILABLE;
1363 : }
1364 :
1365 0 : *_retval = index->mIndexStats.ActiveEntriesCount();
1366 0 : LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval));
1367 0 : return NS_OK;
1368 : }
1369 :
1370 : // static
1371 : nsresult
1372 0 : CacheIndex::GetCacheStats(nsILoadContextInfo *aInfo, uint32_t *aSize, uint32_t *aCount)
1373 : {
1374 0 : LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo));
1375 :
1376 0 : StaticMutexAutoLock lock(sLock);
1377 :
1378 0 : RefPtr<CacheIndex> index = gInstance;
1379 :
1380 0 : if (!index) {
1381 0 : return NS_ERROR_NOT_INITIALIZED;
1382 : }
1383 :
1384 0 : if (!index->IsIndexUsable()) {
1385 0 : return NS_ERROR_NOT_AVAILABLE;
1386 : }
1387 :
1388 0 : *aSize = 0;
1389 0 : *aCount = 0;
1390 :
1391 0 : for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1392 0 : CacheIndexRecord *record = iter.Get();
1393 0 : if (aInfo && !CacheIndexEntry::RecordMatchesLoadContextInfo(record, aInfo))
1394 0 : continue;
1395 :
1396 0 : *aSize += CacheIndexEntry::GetFileSize(record);
1397 0 : ++*aCount;
1398 : }
1399 :
1400 0 : return NS_OK;
1401 : }
1402 :
1403 : // static
1404 : nsresult
1405 0 : CacheIndex::AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver)
1406 : {
1407 0 : LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1408 :
1409 0 : StaticMutexAutoLock lock(sLock);
1410 :
1411 0 : RefPtr<CacheIndex> index = gInstance;
1412 :
1413 0 : if (!index) {
1414 0 : return NS_ERROR_NOT_INITIALIZED;
1415 : }
1416 :
1417 0 : if (!index->IsIndexUsable()) {
1418 0 : return NS_ERROR_NOT_AVAILABLE;
1419 : }
1420 :
1421 : RefPtr<DiskConsumptionObserver> observer =
1422 0 : DiskConsumptionObserver::Init(aObserver);
1423 :
1424 0 : NS_ENSURE_ARG(observer);
1425 :
1426 0 : if ((index->mState == READY || index->mState == WRITING) &&
1427 0 : !index->mAsyncGetDiskConsumptionBlocked) {
1428 0 : LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1429 : // Safe to call the callback under the lock,
1430 : // we always post to the main thread.
1431 0 : observer->OnDiskConsumption(index->mIndexStats.Size() << 10);
1432 0 : return NS_OK;
1433 : }
1434 :
1435 0 : LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1436 : // Will be called when the index get to the READY state.
1437 0 : index->mDiskConsumptionObservers.AppendElement(observer);
1438 :
1439 : // Move forward with index re/building if it is pending
1440 0 : RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
1441 0 : if (ioThread) {
1442 0 : ioThread->Dispatch(
1443 0 : NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption",
1444 0 : []() -> void {
1445 0 : StaticMutexAutoLock lock(sLock);
1446 :
1447 0 : RefPtr<CacheIndex> index = gInstance;
1448 0 : if (index && index->mUpdateTimer) {
1449 0 : index->mUpdateTimer->Cancel();
1450 0 : index->DelayedUpdateLocked();
1451 : }
1452 0 : }),
1453 0 : CacheIOThread::INDEX);
1454 : }
1455 :
1456 0 : return NS_OK;
1457 : }
1458 :
1459 : // static
1460 : nsresult
1461 0 : CacheIndex::GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,
1462 : CacheIndexIterator **_retval)
1463 : {
1464 0 : LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew));
1465 :
1466 0 : StaticMutexAutoLock lock(sLock);
1467 :
1468 0 : RefPtr<CacheIndex> index = gInstance;
1469 :
1470 0 : if (!index) {
1471 0 : return NS_ERROR_NOT_INITIALIZED;
1472 : }
1473 :
1474 0 : if (!index->IsIndexUsable()) {
1475 0 : return NS_ERROR_NOT_AVAILABLE;
1476 : }
1477 :
1478 0 : RefPtr<CacheIndexIterator> idxIter;
1479 0 : if (aInfo) {
1480 0 : idxIter = new CacheIndexContextIterator(index, aAddNew, aInfo);
1481 : } else {
1482 0 : idxIter = new CacheIndexIterator(index, aAddNew);
1483 : }
1484 :
1485 0 : index->mFrecencyArray.SortIfNeeded();
1486 :
1487 0 : for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1488 0 : idxIter->AddRecord(iter.Get());
1489 : }
1490 :
1491 0 : index->mIterators.AppendElement(idxIter);
1492 0 : idxIter.swap(*_retval);
1493 0 : return NS_OK;
1494 : }
1495 :
1496 : // static
1497 : nsresult
1498 6 : CacheIndex::IsUpToDate(bool *_retval)
1499 : {
1500 6 : LOG(("CacheIndex::IsUpToDate()"));
1501 :
1502 12 : StaticMutexAutoLock lock(sLock);
1503 :
1504 12 : RefPtr<CacheIndex> index = gInstance;
1505 :
1506 6 : if (!index) {
1507 0 : return NS_ERROR_NOT_INITIALIZED;
1508 : }
1509 :
1510 6 : if (!index->IsIndexUsable()) {
1511 0 : return NS_ERROR_NOT_AVAILABLE;
1512 : }
1513 :
1514 18 : *_retval = (index->mState == READY || index->mState == WRITING) &&
1515 6 : !index->mIndexNeedsUpdate && !index->mShuttingDown;
1516 :
1517 6 : LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval));
1518 6 : return NS_OK;
1519 : }
1520 :
1521 : bool
1522 53 : CacheIndex::IsIndexUsable()
1523 : {
1524 53 : MOZ_ASSERT(mState != INITIAL);
1525 :
1526 53 : switch (mState) {
1527 : case INITIAL:
1528 : case SHUTDOWN:
1529 0 : return false;
1530 :
1531 : case READING:
1532 : case WRITING:
1533 : case BUILDING:
1534 : case UPDATING:
1535 : case READY:
1536 53 : break;
1537 : }
1538 :
1539 53 : return true;
1540 : }
1541 :
1542 : // static
1543 : bool
1544 5 : CacheIndex::IsCollision(CacheIndexEntry *aEntry,
1545 : OriginAttrsHash aOriginAttrsHash,
1546 : bool aAnonymous)
1547 : {
1548 5 : if (!aEntry->IsInitialized()) {
1549 5 : return false;
1550 : }
1551 :
1552 0 : if (aEntry->Anonymous() != aAnonymous ||
1553 0 : aEntry->OriginAttrsHash() != aOriginAttrsHash) {
1554 0 : LOG(("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1555 : "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64 ", "
1556 : "anonymous=%d; actual values: originAttrsHash=%" PRIu64 ", anonymous=%d]",
1557 : LOGSHA1(aEntry->Hash()), aOriginAttrsHash, aAnonymous,
1558 : aEntry->OriginAttrsHash(), aEntry->Anonymous()));
1559 0 : return true;
1560 : }
1561 :
1562 0 : return false;
1563 : }
1564 :
1565 : // static
1566 : bool
1567 22 : CacheIndex::HasEntryChanged(CacheIndexEntry *aEntry,
1568 : const uint32_t *aFrecency,
1569 : const uint32_t *aExpirationTime,
1570 : const bool *aHasAltData,
1571 : const uint16_t *aOnStartTime,
1572 : const uint16_t *aOnStopTime,
1573 : const uint32_t *aSize)
1574 : {
1575 22 : if (aFrecency && *aFrecency != aEntry->GetFrecency()) {
1576 9 : return true;
1577 : }
1578 :
1579 13 : if (aExpirationTime && *aExpirationTime != aEntry->GetExpirationTime()) {
1580 2 : return true;
1581 : }
1582 :
1583 11 : if (aHasAltData && *aHasAltData != aEntry->GetHasAltData()) {
1584 0 : return true;
1585 : }
1586 :
1587 11 : if (aOnStartTime && *aOnStartTime != aEntry->GetOnStartTime()) {
1588 1 : return true;
1589 : }
1590 :
1591 10 : if (aOnStopTime && *aOnStopTime != aEntry->GetOnStopTime()) {
1592 0 : return true;
1593 : }
1594 :
1595 18 : if (aSize &&
1596 8 : (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) {
1597 6 : return true;
1598 : }
1599 :
1600 4 : return false;
1601 : }
1602 :
1603 : void
1604 1 : CacheIndex::ProcessPendingOperations()
1605 : {
1606 1 : LOG(("CacheIndex::ProcessPendingOperations()"));
1607 :
1608 1 : sLock.AssertCurrentThreadOwns();
1609 :
1610 1 : for (auto iter = mPendingUpdates.Iter(); !iter.Done(); iter.Next()) {
1611 0 : CacheIndexEntryUpdate* update = iter.Get();
1612 :
1613 0 : LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]",
1614 : LOGSHA1(update->Hash())));
1615 :
1616 0 : MOZ_ASSERT(update->IsFresh());
1617 :
1618 0 : CacheIndexEntry* entry = mIndex.GetEntry(*update->Hash());
1619 :
1620 : {
1621 0 : CacheIndexEntryAutoManage emng(update->Hash(), this);
1622 0 : emng.DoNotSearchInUpdates();
1623 :
1624 0 : if (update->IsRemoved()) {
1625 0 : if (entry) {
1626 0 : if (entry->IsRemoved()) {
1627 0 : MOZ_ASSERT(entry->IsFresh());
1628 0 : MOZ_ASSERT(entry->IsDirty());
1629 0 : } else if (!entry->IsDirty() && entry->IsFileEmpty()) {
1630 : // Entries with empty file are not stored in index on disk. Just
1631 : // remove the entry, but only in case the entry is not dirty, i.e.
1632 : // the entry file was empty when we wrote the index.
1633 0 : mIndex.RemoveEntry(entry);
1634 0 : entry = nullptr;
1635 : } else {
1636 0 : entry->MarkRemoved();
1637 0 : entry->MarkDirty();
1638 0 : entry->MarkFresh();
1639 : }
1640 : }
1641 0 : } else if (entry) {
1642 : // Some information in mIndex can be newer than in mPendingUpdates (see
1643 : // bug 1074832). This will copy just those values that were really
1644 : // updated.
1645 0 : update->ApplyUpdate(entry);
1646 : } else {
1647 : // There is no entry in mIndex, copy all information from
1648 : // mPendingUpdates to mIndex.
1649 0 : entry = mIndex.PutEntry(*update->Hash());
1650 0 : *entry = *update;
1651 : }
1652 : }
1653 :
1654 0 : iter.Remove();
1655 : }
1656 :
1657 1 : MOZ_ASSERT(mPendingUpdates.Count() == 0);
1658 :
1659 1 : EnsureCorrectStats();
1660 1 : }
1661 :
1662 : bool
1663 28 : CacheIndex::WriteIndexToDiskIfNeeded()
1664 : {
1665 28 : if (mState != READY || mShuttingDown || mRWPending) {
1666 28 : return false;
1667 : }
1668 :
1669 0 : if (!mLastDumpTime.IsNull() &&
1670 0 : (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() <
1671 : kMinDumpInterval) {
1672 0 : return false;
1673 : }
1674 :
1675 0 : if (mIndexStats.Dirty() < kMinUnwrittenChanges) {
1676 0 : return false;
1677 : }
1678 :
1679 0 : WriteIndexToDisk();
1680 0 : return true;
1681 : }
1682 :
1683 : void
1684 0 : CacheIndex::WriteIndexToDisk()
1685 : {
1686 0 : LOG(("CacheIndex::WriteIndexToDisk()"));
1687 0 : mIndexStats.Log();
1688 :
1689 : nsresult rv;
1690 :
1691 0 : sLock.AssertCurrentThreadOwns();
1692 0 : MOZ_ASSERT(mState == READY);
1693 0 : MOZ_ASSERT(!mRWBuf);
1694 0 : MOZ_ASSERT(!mRWHash);
1695 0 : MOZ_ASSERT(!mRWPending);
1696 :
1697 0 : ChangeState(WRITING);
1698 :
1699 0 : mProcessEntries = mIndexStats.ActiveEntriesCount();
1700 :
1701 0 : mIndexFileOpener = new FileOpenHelper(this);
1702 0 : rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(TEMP_INDEX_NAME),
1703 : CacheFileIOManager::SPECIAL_FILE |
1704 : CacheFileIOManager::CREATE,
1705 0 : mIndexFileOpener);
1706 0 : if (NS_FAILED(rv)) {
1707 0 : LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32 "]",
1708 : static_cast<uint32_t>(rv)));
1709 0 : FinishWrite(false);
1710 0 : return;
1711 : }
1712 :
1713 : // Write index header to a buffer, it will be written to disk together with
1714 : // records in WriteRecords() once we open the file successfully.
1715 0 : AllocBuffer();
1716 0 : mRWHash = new CacheHash();
1717 :
1718 0 : mRWBufPos = 0;
1719 : // index version
1720 0 : NetworkEndian::writeUint32(mRWBuf + mRWBufPos, kIndexVersion);
1721 0 : mRWBufPos += sizeof(uint32_t);
1722 : // timestamp
1723 0 : NetworkEndian::writeUint32(mRWBuf + mRWBufPos,
1724 0 : static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC));
1725 0 : mRWBufPos += sizeof(uint32_t);
1726 : // dirty flag
1727 0 : NetworkEndian::writeUint32(mRWBuf + mRWBufPos, 1);
1728 0 : mRWBufPos += sizeof(uint32_t);
1729 :
1730 0 : mSkipEntries = 0;
1731 : }
1732 :
1733 : void
1734 0 : CacheIndex::WriteRecords()
1735 : {
1736 0 : LOG(("CacheIndex::WriteRecords()"));
1737 :
1738 : nsresult rv;
1739 :
1740 0 : sLock.AssertCurrentThreadOwns();
1741 0 : MOZ_ASSERT(mState == WRITING);
1742 0 : MOZ_ASSERT(!mRWPending);
1743 :
1744 : int64_t fileOffset;
1745 :
1746 0 : if (mSkipEntries) {
1747 0 : MOZ_ASSERT(mRWBufPos == 0);
1748 0 : fileOffset = sizeof(CacheIndexHeader);
1749 0 : fileOffset += sizeof(CacheIndexRecord) * mSkipEntries;
1750 : } else {
1751 0 : MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader));
1752 0 : fileOffset = 0;
1753 : }
1754 0 : uint32_t hashOffset = mRWBufPos;
1755 :
1756 0 : char* buf = mRWBuf + mRWBufPos;
1757 0 : uint32_t skip = mSkipEntries;
1758 0 : uint32_t processMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord);
1759 0 : MOZ_ASSERT(processMax != 0 || mProcessEntries == 0); // TODO make sure we can write an empty index
1760 0 : uint32_t processed = 0;
1761 : #ifdef DEBUG
1762 0 : bool hasMore = false;
1763 : #endif
1764 0 : for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1765 0 : CacheIndexEntry* entry = iter.Get();
1766 0 : if (entry->IsRemoved() ||
1767 0 : !entry->IsInitialized() ||
1768 0 : entry->IsFileEmpty()) {
1769 0 : continue;
1770 : }
1771 :
1772 0 : if (skip) {
1773 0 : skip--;
1774 0 : continue;
1775 : }
1776 :
1777 0 : if (processed == processMax) {
1778 : #ifdef DEBUG
1779 0 : hasMore = true;
1780 : #endif
1781 0 : break;
1782 : }
1783 :
1784 0 : entry->WriteToBuf(buf);
1785 0 : buf += sizeof(CacheIndexRecord);
1786 0 : processed++;
1787 : }
1788 :
1789 0 : MOZ_ASSERT(mRWBufPos != static_cast<uint32_t>(buf - mRWBuf) ||
1790 : mProcessEntries == 0);
1791 0 : mRWBufPos = buf - mRWBuf;
1792 0 : mSkipEntries += processed;
1793 0 : MOZ_ASSERT(mSkipEntries <= mProcessEntries);
1794 :
1795 0 : mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset);
1796 :
1797 0 : if (mSkipEntries == mProcessEntries) {
1798 0 : MOZ_ASSERT(!hasMore);
1799 :
1800 : // We've processed all records
1801 0 : if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) {
1802 : // realloc buffer to spare another write cycle
1803 0 : mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t);
1804 0 : mRWBuf = static_cast<char *>(moz_xrealloc(mRWBuf, mRWBufSize));
1805 : }
1806 :
1807 0 : NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash());
1808 0 : mRWBufPos += sizeof(CacheHash::Hash32_t);
1809 : } else {
1810 0 : MOZ_ASSERT(hasMore);
1811 : }
1812 :
1813 0 : rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos,
1814 0 : mSkipEntries == mProcessEntries, false, this);
1815 0 : if (NS_FAILED(rv)) {
1816 0 : LOG(("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1817 : "synchronously [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(rv)));
1818 0 : FinishWrite(false);
1819 : } else {
1820 0 : mRWPending = true;
1821 : }
1822 :
1823 0 : mRWBufPos = 0;
1824 0 : }
1825 :
1826 : void
1827 0 : CacheIndex::FinishWrite(bool aSucceeded)
1828 : {
1829 0 : LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded));
1830 :
1831 0 : MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING);
1832 :
1833 0 : sLock.AssertCurrentThreadOwns();
1834 :
1835 : // If there is write operation pending we must be cancelling writing of the
1836 : // index when shutting down or removing the whole index.
1837 0 : MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
1838 :
1839 0 : mIndexHandle = nullptr;
1840 0 : mRWHash = nullptr;
1841 0 : ReleaseBuffer();
1842 :
1843 0 : if (aSucceeded) {
1844 : // Opening of the file must not be in progress if writing succeeded.
1845 0 : MOZ_ASSERT(!mIndexFileOpener);
1846 :
1847 0 : for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1848 0 : CacheIndexEntry* entry = iter.Get();
1849 :
1850 0 : bool remove = false;
1851 : {
1852 0 : CacheIndexEntryAutoManage emng(entry->Hash(), this);
1853 :
1854 0 : if (entry->IsRemoved()) {
1855 0 : emng.DoNotSearchInIndex();
1856 0 : remove = true;
1857 0 : } else if (entry->IsDirty()) {
1858 0 : entry->ClearDirty();
1859 : }
1860 : }
1861 0 : if (remove) {
1862 0 : iter.Remove();
1863 : }
1864 : }
1865 :
1866 0 : mIndexOnDiskIsValid = true;
1867 : } else {
1868 0 : if (mIndexFileOpener) {
1869 : // If opening of the file is still in progress (e.g. WRITE process was
1870 : // canceled by RemoveAll()) then we need to cancel the opener to make sure
1871 : // that OnFileOpenedInternal() won't be called.
1872 0 : mIndexFileOpener->Cancel();
1873 0 : mIndexFileOpener = nullptr;
1874 : }
1875 : }
1876 :
1877 0 : ProcessPendingOperations();
1878 0 : mIndexStats.Log();
1879 :
1880 0 : if (mState == WRITING) {
1881 0 : ChangeState(READY);
1882 0 : mLastDumpTime = TimeStamp::NowLoRes();
1883 : }
1884 0 : }
1885 :
1886 : nsresult
1887 0 : CacheIndex::GetFile(const nsACString &aName, nsIFile **_retval)
1888 : {
1889 : nsresult rv;
1890 :
1891 0 : nsCOMPtr<nsIFile> file;
1892 0 : rv = mCacheDirectory->Clone(getter_AddRefs(file));
1893 0 : NS_ENSURE_SUCCESS(rv, rv);
1894 :
1895 0 : rv = file->AppendNative(aName);
1896 0 : NS_ENSURE_SUCCESS(rv, rv);
1897 :
1898 0 : file.swap(*_retval);
1899 0 : return NS_OK;
1900 : }
1901 :
1902 : nsresult
1903 0 : CacheIndex::RemoveFile(const nsACString &aName)
1904 : {
1905 0 : MOZ_ASSERT(mState == SHUTDOWN);
1906 :
1907 : nsresult rv;
1908 :
1909 0 : nsCOMPtr<nsIFile> file;
1910 0 : rv = GetFile(aName, getter_AddRefs(file));
1911 0 : NS_ENSURE_SUCCESS(rv, rv);
1912 :
1913 : bool exists;
1914 0 : rv = file->Exists(&exists);
1915 0 : NS_ENSURE_SUCCESS(rv, rv);
1916 :
1917 0 : if (exists) {
1918 0 : rv = file->Remove(false);
1919 0 : if (NS_FAILED(rv)) {
1920 0 : LOG(("CacheIndex::RemoveFile() - Cannot remove old entry file from disk."
1921 : "[name=%s]", PromiseFlatCString(aName).get()));
1922 0 : NS_WARNING("Cannot remove old entry file from the disk");
1923 0 : return rv;
1924 : }
1925 : }
1926 :
1927 0 : return NS_OK;
1928 : }
1929 :
1930 : void
1931 0 : CacheIndex::RemoveAllIndexFiles()
1932 : {
1933 0 : LOG(("CacheIndex::RemoveAllIndexFiles()"));
1934 0 : RemoveFile(NS_LITERAL_CSTRING(INDEX_NAME));
1935 0 : RemoveJournalAndTempFile();
1936 0 : }
1937 :
1938 : void
1939 0 : CacheIndex::RemoveJournalAndTempFile()
1940 : {
1941 0 : LOG(("CacheIndex::RemoveJournalAndTempFile()"));
1942 0 : RemoveFile(NS_LITERAL_CSTRING(TEMP_INDEX_NAME));
1943 0 : RemoveFile(NS_LITERAL_CSTRING(JOURNAL_NAME));
1944 0 : }
1945 :
1946 : class WriteLogHelper
1947 : {
1948 : public:
1949 0 : explicit WriteLogHelper(PRFileDesc *aFD)
1950 0 : : mFD(aFD)
1951 : , mBufSize(kMaxBufSize)
1952 0 : , mBufPos(0)
1953 : {
1954 0 : mHash = new CacheHash();
1955 0 : mBuf = static_cast<char *>(moz_xmalloc(mBufSize));
1956 0 : }
1957 :
1958 0 : ~WriteLogHelper() {
1959 0 : free(mBuf);
1960 0 : }
1961 :
1962 : nsresult AddEntry(CacheIndexEntry *aEntry);
1963 : nsresult Finish();
1964 :
1965 : private:
1966 :
1967 : nsresult FlushBuffer();
1968 :
1969 : PRFileDesc *mFD;
1970 : char *mBuf;
1971 : uint32_t mBufSize;
1972 : int32_t mBufPos;
1973 : RefPtr<CacheHash> mHash;
1974 : };
1975 :
1976 : nsresult
1977 0 : WriteLogHelper::AddEntry(CacheIndexEntry *aEntry)
1978 : {
1979 : nsresult rv;
1980 :
1981 0 : if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) {
1982 0 : mHash->Update(mBuf, mBufPos);
1983 :
1984 0 : rv = FlushBuffer();
1985 0 : NS_ENSURE_SUCCESS(rv, rv);
1986 0 : MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize);
1987 : }
1988 :
1989 0 : aEntry->WriteToBuf(mBuf + mBufPos);
1990 0 : mBufPos += sizeof(CacheIndexRecord);
1991 :
1992 0 : return NS_OK;
1993 : }
1994 :
1995 : nsresult
1996 0 : WriteLogHelper::Finish()
1997 : {
1998 : nsresult rv;
1999 :
2000 0 : mHash->Update(mBuf, mBufPos);
2001 0 : if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) {
2002 0 : rv = FlushBuffer();
2003 0 : NS_ENSURE_SUCCESS(rv, rv);
2004 0 : MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize);
2005 : }
2006 :
2007 0 : NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash());
2008 0 : mBufPos += sizeof(CacheHash::Hash32_t);
2009 :
2010 0 : rv = FlushBuffer();
2011 0 : NS_ENSURE_SUCCESS(rv, rv);
2012 :
2013 0 : return NS_OK;
2014 : }
2015 :
2016 : nsresult
2017 0 : WriteLogHelper::FlushBuffer()
2018 : {
2019 0 : if (CacheObserver::IsPastShutdownIOLag()) {
2020 0 : LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal."));
2021 0 : return NS_ERROR_FAILURE;
2022 : }
2023 :
2024 0 : int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos);
2025 :
2026 0 : if (bytesWritten != mBufPos) {
2027 0 : return NS_ERROR_FAILURE;
2028 : }
2029 :
2030 0 : mBufPos = 0;
2031 0 : return NS_OK;
2032 : }
2033 :
2034 : nsresult
2035 0 : CacheIndex::WriteLogToDisk()
2036 : {
2037 0 : LOG(("CacheIndex::WriteLogToDisk()"));
2038 :
2039 : nsresult rv;
2040 :
2041 0 : MOZ_ASSERT(mPendingUpdates.Count() == 0);
2042 0 : MOZ_ASSERT(mState == SHUTDOWN);
2043 :
2044 0 : if (CacheObserver::IsPastShutdownIOLag()) {
2045 0 : LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal."));
2046 0 : return NS_ERROR_FAILURE;
2047 : }
2048 :
2049 0 : RemoveFile(NS_LITERAL_CSTRING(TEMP_INDEX_NAME));
2050 :
2051 0 : nsCOMPtr<nsIFile> indexFile;
2052 0 : rv = GetFile(NS_LITERAL_CSTRING(INDEX_NAME), getter_AddRefs(indexFile));
2053 0 : NS_ENSURE_SUCCESS(rv, rv);
2054 :
2055 0 : nsCOMPtr<nsIFile> logFile;
2056 0 : rv = GetFile(NS_LITERAL_CSTRING(JOURNAL_NAME), getter_AddRefs(logFile));
2057 0 : NS_ENSURE_SUCCESS(rv, rv);
2058 :
2059 0 : mIndexStats.Log();
2060 :
2061 0 : PRFileDesc *fd = nullptr;
2062 0 : rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE,
2063 0 : 0600, &fd);
2064 0 : NS_ENSURE_SUCCESS(rv, rv);
2065 :
2066 0 : WriteLogHelper wlh(fd);
2067 0 : for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2068 0 : CacheIndexEntry* entry = iter.Get();
2069 0 : if (entry->IsRemoved() || entry->IsDirty()) {
2070 0 : rv = wlh.AddEntry(entry);
2071 0 : if (NS_WARN_IF(NS_FAILED(rv))) {
2072 0 : return rv;
2073 : }
2074 : }
2075 : }
2076 :
2077 0 : rv = wlh.Finish();
2078 0 : PR_Close(fd);
2079 0 : NS_ENSURE_SUCCESS(rv, rv);
2080 :
2081 0 : rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd);
2082 0 : NS_ENSURE_SUCCESS(rv, rv);
2083 :
2084 : // Seek to dirty flag in the index header and clear it.
2085 : static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader, mIsDirty),
2086 : "Unexpected offset of CacheIndexHeader::mIsDirty");
2087 0 : int64_t offset = PR_Seek64(fd, 2 * sizeof(uint32_t), PR_SEEK_SET);
2088 0 : if (offset == -1) {
2089 0 : PR_Close(fd);
2090 0 : return NS_ERROR_FAILURE;
2091 : }
2092 :
2093 0 : uint32_t isDirty = 0;
2094 0 : int32_t bytesWritten = PR_Write(fd, &isDirty, sizeof(isDirty));
2095 0 : PR_Close(fd);
2096 0 : if (bytesWritten != sizeof(isDirty)) {
2097 0 : return NS_ERROR_FAILURE;
2098 : }
2099 :
2100 0 : return NS_OK;
2101 : }
2102 :
2103 : void
2104 1 : CacheIndex::ReadIndexFromDisk()
2105 : {
2106 1 : LOG(("CacheIndex::ReadIndexFromDisk()"));
2107 :
2108 : nsresult rv;
2109 :
2110 1 : sLock.AssertCurrentThreadOwns();
2111 1 : MOZ_ASSERT(mState == INITIAL);
2112 :
2113 1 : ChangeState(READING);
2114 :
2115 1 : mIndexFileOpener = new FileOpenHelper(this);
2116 3 : rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(INDEX_NAME),
2117 : CacheFileIOManager::SPECIAL_FILE |
2118 : CacheFileIOManager::OPEN,
2119 3 : mIndexFileOpener);
2120 1 : if (NS_FAILED(rv)) {
2121 0 : LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2122 : "failed [rv=0x%08" PRIx32 ", file=%s]", static_cast<uint32_t>(rv), INDEX_NAME));
2123 0 : FinishRead(false);
2124 0 : return;
2125 : }
2126 :
2127 1 : mJournalFileOpener = new FileOpenHelper(this);
2128 3 : rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(JOURNAL_NAME),
2129 : CacheFileIOManager::SPECIAL_FILE |
2130 : CacheFileIOManager::OPEN,
2131 3 : mJournalFileOpener);
2132 1 : if (NS_FAILED(rv)) {
2133 0 : LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2134 : "failed [rv=0x%08" PRIx32 ", file=%s]", static_cast<uint32_t>(rv), JOURNAL_NAME));
2135 0 : FinishRead(false);
2136 : }
2137 :
2138 1 : mTmpFileOpener = new FileOpenHelper(this);
2139 3 : rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(TEMP_INDEX_NAME),
2140 : CacheFileIOManager::SPECIAL_FILE |
2141 : CacheFileIOManager::OPEN,
2142 3 : mTmpFileOpener);
2143 1 : if (NS_FAILED(rv)) {
2144 0 : LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2145 : "failed [rv=0x%08" PRIx32 ", file=%s]", static_cast<uint32_t>(rv),
2146 : TEMP_INDEX_NAME));
2147 0 : FinishRead(false);
2148 : }
2149 : }
2150 :
2151 : void
2152 0 : CacheIndex::StartReadingIndex()
2153 : {
2154 0 : LOG(("CacheIndex::StartReadingIndex()"));
2155 :
2156 : nsresult rv;
2157 :
2158 0 : sLock.AssertCurrentThreadOwns();
2159 :
2160 0 : MOZ_ASSERT(mIndexHandle);
2161 0 : MOZ_ASSERT(mState == READING);
2162 0 : MOZ_ASSERT(!mIndexOnDiskIsValid);
2163 0 : MOZ_ASSERT(!mDontMarkIndexClean);
2164 0 : MOZ_ASSERT(!mJournalReadSuccessfully);
2165 0 : MOZ_ASSERT(mIndexHandle->FileSize() >= 0);
2166 0 : MOZ_ASSERT(!mRWPending);
2167 :
2168 0 : int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2169 0 : sizeof(CacheHash::Hash32_t);
2170 :
2171 0 : if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2172 0 : LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2173 0 : FinishRead(false);
2174 0 : return;
2175 : }
2176 :
2177 0 : AllocBuffer();
2178 0 : mSkipEntries = 0;
2179 0 : mRWHash = new CacheHash();
2180 :
2181 0 : mRWBufPos = std::min(mRWBufSize,
2182 0 : static_cast<uint32_t>(mIndexHandle->FileSize()));
2183 :
2184 0 : rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, this);
2185 0 : if (NS_FAILED(rv)) {
2186 0 : LOG(("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2187 : "synchronously [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(rv)));
2188 0 : FinishRead(false);
2189 : } else {
2190 0 : mRWPending = true;
2191 : }
2192 : }
2193 :
2194 : void
2195 0 : CacheIndex::ParseRecords()
2196 : {
2197 0 : LOG(("CacheIndex::ParseRecords()"));
2198 :
2199 : nsresult rv;
2200 :
2201 0 : sLock.AssertCurrentThreadOwns();
2202 :
2203 0 : MOZ_ASSERT(!mRWPending);
2204 :
2205 0 : uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2206 0 : sizeof(CacheHash::Hash32_t)) / sizeof(CacheIndexRecord);
2207 0 : uint32_t pos = 0;
2208 :
2209 0 : if (!mSkipEntries) {
2210 0 : if (NetworkEndian::readUint32(mRWBuf + pos) != kIndexVersion) {
2211 0 : FinishRead(false);
2212 0 : return;
2213 : }
2214 0 : pos += sizeof(uint32_t);
2215 :
2216 0 : mIndexTimeStamp = NetworkEndian::readUint32(mRWBuf + pos);
2217 0 : pos += sizeof(uint32_t);
2218 :
2219 0 : if (NetworkEndian::readUint32(mRWBuf + pos)) {
2220 0 : if (mJournalHandle) {
2221 0 : CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2222 0 : mJournalHandle = nullptr;
2223 : }
2224 : } else {
2225 : uint32_t * isDirty = reinterpret_cast<uint32_t *>(
2226 0 : moz_xmalloc(sizeof(uint32_t)));
2227 0 : NetworkEndian::writeUint32(isDirty, 1);
2228 :
2229 : // Mark index dirty. The buffer is freed by CacheFileIOManager when
2230 : // nullptr is passed as the listener and the call doesn't fail
2231 : // synchronously.
2232 0 : rv = CacheFileIOManager::Write(mIndexHandle, 2 * sizeof(uint32_t),
2233 : reinterpret_cast<char *>(isDirty),
2234 0 : sizeof(uint32_t), true, false, nullptr);
2235 0 : if (NS_FAILED(rv)) {
2236 : // This is not fatal, just free the memory
2237 0 : free(isDirty);
2238 : }
2239 : }
2240 0 : pos += sizeof(uint32_t);
2241 : }
2242 :
2243 0 : uint32_t hashOffset = pos;
2244 :
2245 0 : while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2246 0 : mSkipEntries != entryCnt) {
2247 0 : CacheIndexRecord *rec = reinterpret_cast<CacheIndexRecord *>(mRWBuf + pos);
2248 0 : CacheIndexEntry tmpEntry(&rec->mHash);
2249 0 : tmpEntry.ReadFromBuf(mRWBuf + pos);
2250 :
2251 0 : if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() ||
2252 0 : tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) {
2253 0 : LOG(("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2254 : " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2255 : "removed=%d]", tmpEntry.IsDirty(), tmpEntry.IsInitialized(),
2256 : tmpEntry.IsFileEmpty(), tmpEntry.IsFresh(), tmpEntry.IsRemoved()));
2257 0 : FinishRead(false);
2258 0 : return;
2259 : }
2260 :
2261 0 : CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this);
2262 :
2263 0 : CacheIndexEntry *entry = mIndex.PutEntry(*tmpEntry.Hash());
2264 0 : *entry = tmpEntry;
2265 :
2266 0 : pos += sizeof(CacheIndexRecord);
2267 0 : mSkipEntries++;
2268 : }
2269 :
2270 0 : mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset);
2271 :
2272 0 : if (pos != mRWBufPos) {
2273 0 : memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2274 : }
2275 :
2276 0 : mRWBufPos -= pos;
2277 0 : pos = 0;
2278 :
2279 : int64_t fileOffset = sizeof(CacheIndexHeader) +
2280 0 : mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2281 :
2282 0 : MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize());
2283 0 : if (fileOffset == mIndexHandle->FileSize()) {
2284 0 : uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2285 0 : if (mRWHash->GetHash() != expectedHash) {
2286 0 : LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2287 : mRWHash->GetHash(), expectedHash));
2288 0 : FinishRead(false);
2289 0 : return;
2290 : }
2291 :
2292 0 : mIndexOnDiskIsValid = true;
2293 0 : mJournalReadSuccessfully = false;
2294 :
2295 0 : if (mJournalHandle) {
2296 0 : StartReadingJournal();
2297 : } else {
2298 0 : FinishRead(false);
2299 : }
2300 :
2301 0 : return;
2302 : }
2303 :
2304 0 : pos = mRWBufPos;
2305 0 : uint32_t toRead = std::min(mRWBufSize - pos,
2306 0 : static_cast<uint32_t>(mIndexHandle->FileSize() -
2307 0 : fileOffset));
2308 0 : mRWBufPos = pos + toRead;
2309 :
2310 0 : rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead,
2311 0 : this);
2312 0 : if (NS_FAILED(rv)) {
2313 0 : LOG(("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2314 : "synchronously [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(rv)));
2315 0 : FinishRead(false);
2316 0 : return;
2317 : } else {
2318 0 : mRWPending = true;
2319 : }
2320 : }
2321 :
2322 : void
2323 0 : CacheIndex::StartReadingJournal()
2324 : {
2325 0 : LOG(("CacheIndex::StartReadingJournal()"));
2326 :
2327 : nsresult rv;
2328 :
2329 0 : sLock.AssertCurrentThreadOwns();
2330 :
2331 0 : MOZ_ASSERT(mJournalHandle);
2332 0 : MOZ_ASSERT(mIndexOnDiskIsValid);
2333 0 : MOZ_ASSERT(mTmpJournal.Count() == 0);
2334 0 : MOZ_ASSERT(mJournalHandle->FileSize() >= 0);
2335 0 : MOZ_ASSERT(!mRWPending);
2336 :
2337 0 : int64_t entriesSize = mJournalHandle->FileSize() -
2338 0 : sizeof(CacheHash::Hash32_t);
2339 :
2340 0 : if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2341 0 : LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2342 0 : FinishRead(false);
2343 0 : return;
2344 : }
2345 :
2346 0 : mSkipEntries = 0;
2347 0 : mRWHash = new CacheHash();
2348 :
2349 0 : mRWBufPos = std::min(mRWBufSize,
2350 0 : static_cast<uint32_t>(mJournalHandle->FileSize()));
2351 :
2352 0 : rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, this);
2353 0 : if (NS_FAILED(rv)) {
2354 0 : LOG(("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2355 : " synchronously [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(rv)));
2356 0 : FinishRead(false);
2357 : } else {
2358 0 : mRWPending = true;
2359 : }
2360 : }
2361 :
2362 : void
2363 0 : CacheIndex::ParseJournal()
2364 : {
2365 0 : LOG(("CacheIndex::ParseJournal()"));
2366 :
2367 : nsresult rv;
2368 :
2369 0 : sLock.AssertCurrentThreadOwns();
2370 :
2371 0 : MOZ_ASSERT(!mRWPending);
2372 :
2373 0 : uint32_t entryCnt = (mJournalHandle->FileSize() -
2374 0 : sizeof(CacheHash::Hash32_t)) / sizeof(CacheIndexRecord);
2375 :
2376 0 : uint32_t pos = 0;
2377 :
2378 0 : while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2379 0 : mSkipEntries != entryCnt) {
2380 0 : CacheIndexEntry tmpEntry(reinterpret_cast<SHA1Sum::Hash *>(mRWBuf + pos));
2381 0 : tmpEntry.ReadFromBuf(mRWBuf + pos);
2382 :
2383 0 : CacheIndexEntry *entry = mTmpJournal.PutEntry(*tmpEntry.Hash());
2384 0 : *entry = tmpEntry;
2385 :
2386 0 : if (entry->IsDirty() || entry->IsFresh()) {
2387 0 : LOG(("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2388 : "ignoring whole journal [dirty=%d, fresh=%d]", entry->IsDirty(),
2389 : entry->IsFresh()));
2390 0 : FinishRead(false);
2391 0 : return;
2392 : }
2393 :
2394 0 : pos += sizeof(CacheIndexRecord);
2395 0 : mSkipEntries++;
2396 : }
2397 :
2398 0 : mRWHash->Update(mRWBuf, pos);
2399 :
2400 0 : if (pos != mRWBufPos) {
2401 0 : memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2402 : }
2403 :
2404 0 : mRWBufPos -= pos;
2405 0 : pos = 0;
2406 :
2407 0 : int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2408 :
2409 0 : MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize());
2410 0 : if (fileOffset == mJournalHandle->FileSize()) {
2411 0 : uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2412 0 : if (mRWHash->GetHash() != expectedHash) {
2413 0 : LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2414 : mRWHash->GetHash(), expectedHash));
2415 0 : FinishRead(false);
2416 0 : return;
2417 : }
2418 :
2419 0 : mJournalReadSuccessfully = true;
2420 0 : FinishRead(true);
2421 0 : return;
2422 : }
2423 :
2424 0 : pos = mRWBufPos;
2425 0 : uint32_t toRead = std::min(mRWBufSize - pos,
2426 0 : static_cast<uint32_t>(mJournalHandle->FileSize() -
2427 0 : fileOffset));
2428 0 : mRWBufPos = pos + toRead;
2429 :
2430 0 : rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos,
2431 0 : toRead, this);
2432 0 : if (NS_FAILED(rv)) {
2433 0 : LOG(("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2434 : "synchronously [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(rv)));
2435 0 : FinishRead(false);
2436 0 : return;
2437 : } else {
2438 0 : mRWPending = true;
2439 : }
2440 : }
2441 :
2442 : void
2443 0 : CacheIndex::MergeJournal()
2444 : {
2445 0 : LOG(("CacheIndex::MergeJournal()"));
2446 :
2447 0 : sLock.AssertCurrentThreadOwns();
2448 :
2449 0 : for (auto iter = mTmpJournal.Iter(); !iter.Done(); iter.Next()) {
2450 0 : CacheIndexEntry* entry = iter.Get();
2451 :
2452 0 : LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]",
2453 : LOGSHA1(entry->Hash())));
2454 :
2455 0 : CacheIndexEntry* entry2 = mIndex.GetEntry(*entry->Hash());
2456 : {
2457 0 : CacheIndexEntryAutoManage emng(entry->Hash(), this);
2458 0 : if (entry->IsRemoved()) {
2459 0 : if (entry2) {
2460 0 : entry2->MarkRemoved();
2461 0 : entry2->MarkDirty();
2462 : }
2463 : } else {
2464 0 : if (!entry2) {
2465 0 : entry2 = mIndex.PutEntry(*entry->Hash());
2466 : }
2467 :
2468 0 : *entry2 = *entry;
2469 0 : entry2->MarkDirty();
2470 : }
2471 : }
2472 0 : iter.Remove();
2473 : }
2474 :
2475 0 : MOZ_ASSERT(mTmpJournal.Count() == 0);
2476 0 : }
2477 :
2478 : void
2479 1 : CacheIndex::EnsureNoFreshEntry()
2480 : {
2481 : #ifdef DEBUG_STATS
2482 1 : CacheIndexStats debugStats;
2483 1 : debugStats.DisableLogging();
2484 1 : for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2485 0 : debugStats.BeforeChange(nullptr);
2486 0 : debugStats.AfterChange(iter.Get());
2487 : }
2488 1 : MOZ_ASSERT(debugStats.Fresh() == 0);
2489 : #endif
2490 1 : }
2491 :
2492 : void
2493 1 : CacheIndex::EnsureCorrectStats()
2494 : {
2495 : #ifdef DEBUG_STATS
2496 1 : MOZ_ASSERT(mPendingUpdates.Count() == 0);
2497 1 : CacheIndexStats debugStats;
2498 1 : debugStats.DisableLogging();
2499 1 : for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2500 0 : debugStats.BeforeChange(nullptr);
2501 0 : debugStats.AfterChange(iter.Get());
2502 : }
2503 1 : MOZ_ASSERT(debugStats == mIndexStats);
2504 : #endif
2505 1 : }
2506 :
2507 : void
2508 1 : CacheIndex::FinishRead(bool aSucceeded)
2509 : {
2510 1 : LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded));
2511 1 : sLock.AssertCurrentThreadOwns();
2512 :
2513 1 : MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING);
2514 :
2515 1 : MOZ_ASSERT(
2516 : // -> rebuild
2517 : (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2518 : // -> update
2519 : (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2520 : // -> ready
2521 : (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully));
2522 :
2523 : // If there is read operation pending we must be cancelling reading of the
2524 : // index when shutting down or removing the whole index.
2525 1 : MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
2526 :
2527 1 : if (mState == SHUTDOWN) {
2528 0 : RemoveFile(NS_LITERAL_CSTRING(TEMP_INDEX_NAME));
2529 0 : RemoveFile(NS_LITERAL_CSTRING(JOURNAL_NAME));
2530 : } else {
2531 1 : if (mIndexHandle && !mIndexOnDiskIsValid) {
2532 0 : CacheFileIOManager::DoomFile(mIndexHandle, nullptr);
2533 : }
2534 :
2535 1 : if (mJournalHandle) {
2536 0 : CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2537 : }
2538 : }
2539 :
2540 1 : if (mIndexFileOpener) {
2541 0 : mIndexFileOpener->Cancel();
2542 0 : mIndexFileOpener = nullptr;
2543 : }
2544 1 : if (mJournalFileOpener) {
2545 1 : mJournalFileOpener->Cancel();
2546 1 : mJournalFileOpener = nullptr;
2547 : }
2548 1 : if (mTmpFileOpener) {
2549 1 : mTmpFileOpener->Cancel();
2550 1 : mTmpFileOpener = nullptr;
2551 : }
2552 :
2553 1 : mIndexHandle = nullptr;
2554 1 : mJournalHandle = nullptr;
2555 1 : mRWHash = nullptr;
2556 1 : ReleaseBuffer();
2557 :
2558 1 : if (mState == SHUTDOWN) {
2559 0 : return;
2560 : }
2561 :
2562 1 : if (!mIndexOnDiskIsValid) {
2563 1 : MOZ_ASSERT(mTmpJournal.Count() == 0);
2564 1 : EnsureNoFreshEntry();
2565 1 : ProcessPendingOperations();
2566 : // Remove all entries that we haven't seen during this session
2567 1 : RemoveNonFreshEntries();
2568 1 : StartUpdatingIndex(true);
2569 1 : return;
2570 : }
2571 :
2572 0 : if (!mJournalReadSuccessfully) {
2573 0 : mTmpJournal.Clear();
2574 0 : EnsureNoFreshEntry();
2575 0 : ProcessPendingOperations();
2576 0 : StartUpdatingIndex(false);
2577 0 : return;
2578 : }
2579 :
2580 0 : MergeJournal();
2581 0 : EnsureNoFreshEntry();
2582 0 : ProcessPendingOperations();
2583 0 : mIndexStats.Log();
2584 :
2585 0 : ChangeState(READY);
2586 0 : mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately
2587 : }
2588 :
2589 : // static
2590 : void
2591 0 : CacheIndex::DelayedUpdate(nsITimer *aTimer, void *aClosure)
2592 : {
2593 0 : LOG(("CacheIndex::DelayedUpdate()"));
2594 :
2595 0 : StaticMutexAutoLock lock(sLock);
2596 0 : RefPtr<CacheIndex> index = gInstance;
2597 :
2598 0 : if (!index) {
2599 0 : return;
2600 : }
2601 :
2602 0 : index->DelayedUpdateLocked();
2603 : }
2604 :
2605 : // static
2606 : void
2607 0 : CacheIndex::DelayedUpdateLocked()
2608 : {
2609 0 : LOG(("CacheIndex::DelayedUpdateLocked()"));
2610 :
2611 0 : sLock.AssertCurrentThreadOwns();
2612 :
2613 : nsresult rv;
2614 :
2615 0 : mUpdateTimer = nullptr;
2616 :
2617 0 : if (!IsIndexUsable()) {
2618 0 : return;
2619 : }
2620 :
2621 0 : if (mState == READY && mShuttingDown) {
2622 0 : return;
2623 : }
2624 :
2625 : // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2626 : // schedule timer if it is true.
2627 0 : MOZ_ASSERT(!mUpdateEventPending);
2628 0 : if (mState != BUILDING && mState != UPDATING) {
2629 0 : LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled"));
2630 0 : return;
2631 : }
2632 :
2633 : // We need to redispatch to run with lower priority
2634 0 : RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2635 0 : MOZ_ASSERT(ioThread);
2636 :
2637 0 : mUpdateEventPending = true;
2638 0 : rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2639 0 : if (NS_FAILED(rv)) {
2640 0 : mUpdateEventPending = false;
2641 0 : NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event");
2642 0 : LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event" ));
2643 0 : FinishUpdate(false);
2644 : }
2645 : }
2646 :
2647 : nsresult
2648 1 : CacheIndex::ScheduleUpdateTimer(uint32_t aDelay)
2649 : {
2650 1 : LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay));
2651 :
2652 1 : MOZ_ASSERT(!mUpdateTimer);
2653 :
2654 : nsresult rv;
2655 :
2656 2 : nsCOMPtr<nsITimer> timer = do_CreateInstance("@mozilla.org/timer;1", &rv);
2657 1 : NS_ENSURE_SUCCESS(rv, rv);
2658 :
2659 2 : nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
2660 1 : MOZ_ASSERT(ioTarget);
2661 :
2662 1 : rv = timer->SetTarget(ioTarget);
2663 1 : NS_ENSURE_SUCCESS(rv, rv);
2664 :
2665 2 : rv = timer->InitWithNamedFuncCallback(CacheIndex::DelayedUpdate,
2666 : nullptr,
2667 : aDelay,
2668 : nsITimer::TYPE_ONE_SHOT,
2669 1 : "net::CacheIndex::ScheduleUpdateTimer");
2670 1 : NS_ENSURE_SUCCESS(rv, rv);
2671 :
2672 1 : mUpdateTimer.swap(timer);
2673 1 : return NS_OK;
2674 : }
2675 :
2676 : nsresult
2677 0 : CacheIndex::SetupDirectoryEnumerator()
2678 : {
2679 0 : MOZ_ASSERT(!NS_IsMainThread());
2680 0 : MOZ_ASSERT(!mDirEnumerator);
2681 :
2682 : nsresult rv;
2683 0 : nsCOMPtr<nsIFile> file;
2684 :
2685 0 : rv = mCacheDirectory->Clone(getter_AddRefs(file));
2686 0 : NS_ENSURE_SUCCESS(rv, rv);
2687 :
2688 0 : rv = file->AppendNative(NS_LITERAL_CSTRING(ENTRIES_DIR));
2689 0 : NS_ENSURE_SUCCESS(rv, rv);
2690 :
2691 : bool exists;
2692 0 : rv = file->Exists(&exists);
2693 0 : NS_ENSURE_SUCCESS(rv, rv);
2694 :
2695 0 : if (!exists) {
2696 : NS_WARNING("CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2697 0 : "doesn't exist!");
2698 0 : LOG(("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2699 : "exist!" ));
2700 0 : return NS_ERROR_UNEXPECTED;
2701 : }
2702 :
2703 0 : nsCOMPtr<nsISimpleEnumerator> enumerator;
2704 0 : rv = file->GetDirectoryEntries(getter_AddRefs(enumerator));
2705 0 : NS_ENSURE_SUCCESS(rv, rv);
2706 :
2707 0 : mDirEnumerator = do_QueryInterface(enumerator, &rv);
2708 0 : NS_ENSURE_SUCCESS(rv, rv);
2709 :
2710 0 : return NS_OK;
2711 : }
2712 :
2713 : nsresult
2714 0 : CacheIndex::InitEntryFromDiskData(CacheIndexEntry *aEntry,
2715 : CacheFileMetadata *aMetaData,
2716 : int64_t aFileSize)
2717 : {
2718 0 : aEntry->InitNew();
2719 0 : aEntry->MarkDirty();
2720 0 : aEntry->MarkFresh();
2721 :
2722 0 : aEntry->Init(GetOriginAttrsHash(aMetaData->OriginAttributes()),
2723 0 : aMetaData->IsAnonymous(),
2724 0 : aMetaData->Pinned());
2725 :
2726 : uint32_t expirationTime;
2727 0 : aMetaData->GetExpirationTime(&expirationTime);
2728 0 : aEntry->SetExpirationTime(expirationTime);
2729 :
2730 : uint32_t frecency;
2731 0 : aMetaData->GetFrecency(&frecency);
2732 0 : aEntry->SetFrecency(frecency);
2733 :
2734 0 : const char *altData = aMetaData->GetElement(CacheFileUtils::kAltDataKey);
2735 0 : bool hasAltData = altData ? true : false;
2736 0 : if (hasAltData &&
2737 0 : NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo(altData, nullptr, nullptr))) {
2738 0 : return NS_ERROR_FAILURE;
2739 : }
2740 0 : aEntry->SetHasAltData(hasAltData);
2741 :
2742 0 : static auto getUint16MetaData = [&aMetaData](const char *key) -> uint16_t {
2743 0 : const char* s64 = aMetaData->GetElement(key);
2744 0 : if (s64) {
2745 : nsresult rv;
2746 0 : uint64_t n64 = nsCString(s64).ToInteger64(&rv);
2747 0 : MOZ_ASSERT(NS_SUCCEEDED(rv));
2748 0 : return n64 <= kIndexTimeOutOfBound ? n64 : kIndexTimeOutOfBound;
2749 : }
2750 0 : return kIndexTimeNotAvailable;
2751 0 : };
2752 :
2753 0 : aEntry->SetOnStartTime(getUint16MetaData("net-response-time-onstart"));
2754 0 : aEntry->SetOnStopTime(getUint16MetaData("net-response-time-onstop"));
2755 :
2756 0 : aEntry->SetFileSize(static_cast<uint32_t>(
2757 0 : std::min(static_cast<int64_t>(PR_UINT32_MAX),
2758 0 : (aFileSize + 0x3FF) >> 10)));
2759 0 : return NS_OK;
2760 : }
2761 :
2762 : bool
2763 1 : CacheIndex::IsUpdatePending()
2764 : {
2765 1 : sLock.AssertCurrentThreadOwns();
2766 :
2767 1 : if (mUpdateTimer || mUpdateEventPending) {
2768 0 : return true;
2769 : }
2770 :
2771 1 : return false;
2772 : }
2773 :
2774 : void
2775 0 : CacheIndex::BuildIndex()
2776 : {
2777 0 : LOG(("CacheIndex::BuildIndex()"));
2778 :
2779 0 : sLock.AssertCurrentThreadOwns();
2780 :
2781 0 : MOZ_ASSERT(mPendingUpdates.Count() == 0);
2782 :
2783 : nsresult rv;
2784 :
2785 0 : if (!mDirEnumerator) {
2786 : {
2787 : // Do not do IO under the lock.
2788 0 : StaticMutexAutoUnlock unlock(sLock);
2789 0 : rv = SetupDirectoryEnumerator();
2790 : }
2791 0 : if (mState == SHUTDOWN) {
2792 : // The index was shut down while we released the lock. FinishUpdate() was
2793 : // already called from Shutdown(), so just simply return here.
2794 0 : return;
2795 : }
2796 :
2797 0 : if (NS_FAILED(rv)) {
2798 0 : FinishUpdate(false);
2799 0 : return;
2800 : }
2801 : }
2802 :
2803 : while (true) {
2804 0 : if (CacheIOThread::YieldAndRerun()) {
2805 0 : LOG(("CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2806 0 : mUpdateEventPending = true;
2807 0 : return;
2808 : }
2809 :
2810 0 : nsCOMPtr<nsIFile> file;
2811 : {
2812 : // Do not do IO under the lock.
2813 0 : StaticMutexAutoUnlock unlock(sLock);
2814 0 : rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2815 : }
2816 0 : if (mState == SHUTDOWN) {
2817 0 : return;
2818 : }
2819 0 : if (!file) {
2820 0 : FinishUpdate(NS_SUCCEEDED(rv));
2821 0 : return;
2822 : }
2823 :
2824 0 : nsAutoCString leaf;
2825 0 : rv = file->GetNativeLeafName(leaf);
2826 0 : if (NS_FAILED(rv)) {
2827 0 : LOG(("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2828 : "file."));
2829 0 : mDontMarkIndexClean = true;
2830 0 : continue;
2831 : }
2832 :
2833 : SHA1Sum::Hash hash;
2834 0 : rv = CacheFileIOManager::StrToHash(leaf, &hash);
2835 0 : if (NS_FAILED(rv)) {
2836 0 : LOG(("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2837 : "[name=%s]", leaf.get()));
2838 0 : file->Remove(false);
2839 0 : continue;
2840 : }
2841 :
2842 0 : CacheIndexEntry *entry = mIndex.GetEntry(hash);
2843 0 : if (entry && entry->IsRemoved()) {
2844 0 : LOG(("CacheIndex::BuildIndex() - Found file that should not exist. "
2845 : "[name=%s]", leaf.get()));
2846 0 : entry->Log();
2847 0 : MOZ_ASSERT(entry->IsFresh());
2848 0 : entry = nullptr;
2849 : }
2850 :
2851 : #ifdef DEBUG
2852 0 : RefPtr<CacheFileHandle> handle;
2853 0 : CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
2854 0 : getter_AddRefs(handle));
2855 : #endif
2856 :
2857 0 : if (entry) {
2858 : // the entry is up to date
2859 0 : LOG(("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2860 : " date. [name=%s]", leaf.get()));
2861 0 : entry->Log();
2862 0 : MOZ_ASSERT(entry->IsFresh()); // The entry must be from this session
2863 : // there must be an active CacheFile if the entry is not initialized
2864 0 : MOZ_ASSERT(entry->IsInitialized() || handle);
2865 0 : continue;
2866 : }
2867 :
2868 0 : MOZ_ASSERT(!handle);
2869 :
2870 0 : RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
2871 0 : int64_t size = 0;
2872 :
2873 : {
2874 : // Do not do IO under the lock.
2875 0 : StaticMutexAutoUnlock unlock(sLock);
2876 0 : rv = meta->SyncReadMetadata(file);
2877 :
2878 0 : if (NS_SUCCEEDED(rv)) {
2879 0 : rv = file->GetFileSize(&size);
2880 0 : if (NS_FAILED(rv)) {
2881 0 : LOG(("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2882 : " successfully parsed. [name=%s]", leaf.get()));
2883 : }
2884 : }
2885 : }
2886 0 : if (mState == SHUTDOWN) {
2887 0 : return;
2888 : }
2889 :
2890 : // Nobody could add the entry while the lock was released since we modify
2891 : // the index only on IO thread and this loop is executed on IO thread too.
2892 0 : entry = mIndex.GetEntry(hash);
2893 0 : MOZ_ASSERT(!entry || entry->IsRemoved());
2894 :
2895 0 : if (NS_FAILED(rv)) {
2896 0 : LOG(("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2897 : "failed, removing file. [name=%s]", leaf.get()));
2898 0 : file->Remove(false);
2899 : } else {
2900 0 : CacheIndexEntryAutoManage entryMng(&hash, this);
2901 0 : entry = mIndex.PutEntry(hash);
2902 0 : if (NS_FAILED(InitEntryFromDiskData(entry, meta, size))) {
2903 0 : LOG(("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() "
2904 : "failed, removing file. [name=%s]", leaf.get()));
2905 0 : file->Remove(false);
2906 : } else {
2907 0 : LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]",
2908 : leaf.get()));
2909 0 : entry->Log();
2910 : }
2911 : }
2912 0 : }
2913 :
2914 : NS_NOTREACHED("We should never get here");
2915 : }
2916 :
2917 : bool
2918 10 : CacheIndex::StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState)
2919 : {
2920 : // Start updating process when we are in or we are switching to READY state
2921 : // and index needs update, but not during shutdown or when removing all
2922 : // entries.
2923 10 : if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate &&
2924 0 : !mShuttingDown && !mRemovingAll) {
2925 0 : LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2926 0 : mIndexNeedsUpdate = false;
2927 0 : StartUpdatingIndex(false);
2928 0 : return true;
2929 : }
2930 :
2931 10 : return false;
2932 : }
2933 :
2934 : void
2935 1 : CacheIndex::StartUpdatingIndex(bool aRebuild)
2936 : {
2937 1 : LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild));
2938 :
2939 1 : sLock.AssertCurrentThreadOwns();
2940 :
2941 : nsresult rv;
2942 :
2943 1 : mIndexStats.Log();
2944 :
2945 1 : ChangeState(aRebuild ? BUILDING : UPDATING);
2946 1 : mDontMarkIndexClean = false;
2947 :
2948 1 : if (mShuttingDown || mRemovingAll) {
2949 0 : FinishUpdate(false);
2950 0 : return;
2951 : }
2952 :
2953 1 : if (IsUpdatePending()) {
2954 0 : LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2955 0 : return;
2956 : }
2957 :
2958 1 : uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds();
2959 1 : if (elapsed < kUpdateIndexStartDelay) {
2960 1 : LOG(("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2961 : "scheduling timer to fire in %u ms.", elapsed,
2962 : kUpdateIndexStartDelay - elapsed));
2963 1 : rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed);
2964 1 : if (NS_SUCCEEDED(rv)) {
2965 1 : return;
2966 : }
2967 :
2968 0 : LOG(("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2969 : "Starting update immediately."));
2970 : } else {
2971 0 : LOG(("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2972 : "starting update now.", elapsed));
2973 : }
2974 :
2975 0 : RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2976 0 : MOZ_ASSERT(ioThread);
2977 :
2978 : // We need to dispatch an event even if we are on IO thread since we need to
2979 : // update the index with the correct priority.
2980 0 : mUpdateEventPending = true;
2981 0 : rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2982 0 : if (NS_FAILED(rv)) {
2983 0 : mUpdateEventPending = false;
2984 0 : NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2985 0 : LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event" ));
2986 0 : FinishUpdate(false);
2987 : }
2988 : }
2989 :
2990 : void
2991 0 : CacheIndex::UpdateIndex()
2992 : {
2993 0 : LOG(("CacheIndex::UpdateIndex()"));
2994 :
2995 0 : sLock.AssertCurrentThreadOwns();
2996 :
2997 0 : MOZ_ASSERT(mPendingUpdates.Count() == 0);
2998 :
2999 : nsresult rv;
3000 :
3001 0 : if (!mDirEnumerator) {
3002 : {
3003 : // Do not do IO under the lock.
3004 0 : StaticMutexAutoUnlock unlock(sLock);
3005 0 : rv = SetupDirectoryEnumerator();
3006 : }
3007 0 : if (mState == SHUTDOWN) {
3008 : // The index was shut down while we released the lock. FinishUpdate() was
3009 : // already called from Shutdown(), so just simply return here.
3010 0 : return;
3011 : }
3012 :
3013 0 : if (NS_FAILED(rv)) {
3014 0 : FinishUpdate(false);
3015 0 : return;
3016 : }
3017 : }
3018 :
3019 : while (true) {
3020 0 : if (CacheIOThread::YieldAndRerun()) {
3021 0 : LOG(("CacheIndex::UpdateIndex() - Breaking loop for higher level "
3022 : "events."));
3023 0 : mUpdateEventPending = true;
3024 0 : return;
3025 : }
3026 :
3027 0 : nsCOMPtr<nsIFile> file;
3028 : {
3029 : // Do not do IO under the lock.
3030 0 : StaticMutexAutoUnlock unlock(sLock);
3031 0 : rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
3032 : }
3033 0 : if (mState == SHUTDOWN) {
3034 0 : return;
3035 : }
3036 0 : if (!file) {
3037 0 : FinishUpdate(NS_SUCCEEDED(rv));
3038 0 : return;
3039 : }
3040 :
3041 0 : nsAutoCString leaf;
3042 0 : rv = file->GetNativeLeafName(leaf);
3043 0 : if (NS_FAILED(rv)) {
3044 0 : LOG(("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
3045 : "file."));
3046 0 : mDontMarkIndexClean = true;
3047 0 : continue;
3048 : }
3049 :
3050 : SHA1Sum::Hash hash;
3051 0 : rv = CacheFileIOManager::StrToHash(leaf, &hash);
3052 0 : if (NS_FAILED(rv)) {
3053 0 : LOG(("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
3054 : "[name=%s]", leaf.get()));
3055 0 : file->Remove(false);
3056 0 : continue;
3057 : }
3058 :
3059 0 : CacheIndexEntry *entry = mIndex.GetEntry(hash);
3060 0 : if (entry && entry->IsRemoved()) {
3061 0 : if (entry->IsFresh()) {
3062 0 : LOG(("CacheIndex::UpdateIndex() - Found file that should not exist. "
3063 : "[name=%s]", leaf.get()));
3064 0 : entry->Log();
3065 : }
3066 0 : entry = nullptr;
3067 : }
3068 :
3069 : #ifdef DEBUG
3070 0 : RefPtr<CacheFileHandle> handle;
3071 0 : CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
3072 0 : getter_AddRefs(handle));
3073 : #endif
3074 :
3075 0 : if (entry && entry->IsFresh()) {
3076 : // the entry is up to date
3077 0 : LOG(("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
3078 : " to date. [name=%s]", leaf.get()));
3079 0 : entry->Log();
3080 : // there must be an active CacheFile if the entry is not initialized
3081 0 : MOZ_ASSERT(entry->IsInitialized() || handle);
3082 0 : continue;
3083 : }
3084 :
3085 0 : MOZ_ASSERT(!handle);
3086 :
3087 0 : if (entry) {
3088 : PRTime lastModifiedTime;
3089 : {
3090 : // Do not do IO under the lock.
3091 0 : StaticMutexAutoUnlock unlock(sLock);
3092 0 : rv = file->GetLastModifiedTime(&lastModifiedTime);
3093 : }
3094 0 : if (mState == SHUTDOWN) {
3095 0 : return;
3096 : }
3097 0 : if (NS_FAILED(rv)) {
3098 0 : LOG(("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
3099 : "[name=%s]", leaf.get()));
3100 : // Assume the file is newer than index
3101 : } else {
3102 0 : if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) {
3103 0 : LOG(("CacheIndex::UpdateIndex() - Skipping file because of last "
3104 : "modified time. [name=%s, indexTimeStamp=%" PRIu32 ", "
3105 : "lastModifiedTime=%" PRId64 "]", leaf.get(), mIndexTimeStamp,
3106 : lastModifiedTime / PR_MSEC_PER_SEC));
3107 :
3108 0 : CacheIndexEntryAutoManage entryMng(&hash, this);
3109 0 : entry->MarkFresh();
3110 0 : continue;
3111 : }
3112 : }
3113 : }
3114 :
3115 0 : RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
3116 0 : int64_t size = 0;
3117 :
3118 : {
3119 : // Do not do IO under the lock.
3120 0 : StaticMutexAutoUnlock unlock(sLock);
3121 0 : rv = meta->SyncReadMetadata(file);
3122 :
3123 0 : if (NS_SUCCEEDED(rv)) {
3124 0 : rv = file->GetFileSize(&size);
3125 0 : if (NS_FAILED(rv)) {
3126 0 : LOG(("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
3127 : "was successfully parsed. [name=%s]", leaf.get()));
3128 : }
3129 : }
3130 : }
3131 0 : if (mState == SHUTDOWN) {
3132 0 : return;
3133 : }
3134 :
3135 : // Nobody could add the entry while the lock was released since we modify
3136 : // the index only on IO thread and this loop is executed on IO thread too.
3137 0 : entry = mIndex.GetEntry(hash);
3138 0 : MOZ_ASSERT(!entry || !entry->IsFresh());
3139 :
3140 0 : CacheIndexEntryAutoManage entryMng(&hash, this);
3141 :
3142 0 : if (NS_FAILED(rv)) {
3143 0 : LOG(("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
3144 : "failed, removing file. [name=%s]", leaf.get()));
3145 : } else {
3146 0 : entry = mIndex.PutEntry(hash);
3147 0 : rv = InitEntryFromDiskData(entry, meta, size);
3148 0 : if (NS_FAILED(rv)) {
3149 0 : LOG(("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData "
3150 : "failed, removing file. [name=%s]", leaf.get()));
3151 : }
3152 : }
3153 :
3154 0 : if (NS_FAILED(rv)) {
3155 0 : file->Remove(false);
3156 0 : if (entry) {
3157 0 : entry->MarkRemoved();
3158 0 : entry->MarkFresh();
3159 0 : entry->MarkDirty();
3160 : }
3161 : } else {
3162 0 : LOG(("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
3163 : "[name=%s]", leaf.get()));
3164 0 : entry->Log();
3165 : }
3166 0 : }
3167 :
3168 : NS_NOTREACHED("We should never get here");
3169 : }
3170 :
3171 : void
3172 0 : CacheIndex::FinishUpdate(bool aSucceeded)
3173 : {
3174 0 : LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded));
3175 :
3176 0 : MOZ_ASSERT(mState == UPDATING || mState == BUILDING ||
3177 : (!aSucceeded && mState == SHUTDOWN));
3178 :
3179 0 : sLock.AssertCurrentThreadOwns();
3180 :
3181 0 : if (mDirEnumerator) {
3182 0 : if (NS_IsMainThread()) {
3183 0 : LOG(("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3184 : " Cannot safely release mDirEnumerator, leaking it!"));
3185 0 : NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3186 : // This can happen only in case dispatching event to IO thread failed in
3187 : // CacheIndex::PreShutdown().
3188 0 : Unused << mDirEnumerator.forget(); // Leak it since dir enumerator is not threadsafe
3189 : } else {
3190 0 : mDirEnumerator->Close();
3191 0 : mDirEnumerator = nullptr;
3192 : }
3193 : }
3194 :
3195 0 : if (!aSucceeded) {
3196 0 : mDontMarkIndexClean = true;
3197 : }
3198 :
3199 0 : if (mState == SHUTDOWN) {
3200 0 : return;
3201 : }
3202 :
3203 0 : if (mState == UPDATING && aSucceeded) {
3204 : // If we've iterated over all entries successfully then all entries that
3205 : // really exist on the disk are now marked as fresh. All non-fresh entries
3206 : // don't exist anymore and must be removed from the index.
3207 0 : RemoveNonFreshEntries();
3208 : }
3209 :
3210 : // Make sure we won't start update. If the build or update failed, there is no
3211 : // reason to believe that it will succeed next time.
3212 0 : mIndexNeedsUpdate = false;
3213 :
3214 0 : ChangeState(READY);
3215 0 : mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately
3216 : }
3217 :
3218 : void
3219 1 : CacheIndex::RemoveNonFreshEntries()
3220 : {
3221 1 : for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
3222 0 : CacheIndexEntry* entry = iter.Get();
3223 0 : if (entry->IsFresh()) {
3224 0 : continue;
3225 : }
3226 :
3227 0 : LOG(("CacheIndex::RemoveNonFreshEntries() - Removing entry. "
3228 : "[hash=%08x%08x%08x%08x%08x]", LOGSHA1(entry->Hash())));
3229 :
3230 : {
3231 0 : CacheIndexEntryAutoManage emng(entry->Hash(), this);
3232 0 : emng.DoNotSearchInIndex();
3233 : }
3234 :
3235 0 : iter.Remove();
3236 : }
3237 1 : }
3238 :
3239 : // static
3240 : char const *
3241 0 : CacheIndex::StateString(EState aState)
3242 : {
3243 0 : switch (aState) {
3244 0 : case INITIAL: return "INITIAL";
3245 0 : case READING: return "READING";
3246 0 : case WRITING: return "WRITING";
3247 0 : case BUILDING: return "BUILDING";
3248 0 : case UPDATING: return "UPDATING";
3249 0 : case READY: return "READY";
3250 0 : case SHUTDOWN: return "SHUTDOWN";
3251 : }
3252 :
3253 0 : MOZ_ASSERT(false, "Unexpected state!");
3254 : return "?";
3255 : }
3256 :
3257 : void
3258 2 : CacheIndex::ChangeState(EState aNewState)
3259 : {
3260 2 : LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState),
3261 : StateString(aNewState)));
3262 :
3263 : // All pending updates should be processed before changing state
3264 2 : MOZ_ASSERT(mPendingUpdates.Count() == 0);
3265 :
3266 : // PreShutdownInternal() should change the state to READY from every state. It
3267 : // may go through different states, but once we are in READY state the only
3268 : // possible transition is to SHUTDOWN state.
3269 2 : MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN);
3270 :
3271 : // Start updating process when switching to READY state if needed
3272 2 : if (aNewState == READY && StartUpdatingIndexIfNeeded(true)) {
3273 0 : return;
3274 : }
3275 :
3276 2 : if ((mState == READING || mState == BUILDING || mState == UPDATING) &&
3277 : aNewState == READY) {
3278 0 : ReportHashStats();
3279 : }
3280 :
3281 : // Try to evict entries over limit everytime we're leaving state READING,
3282 : // BUILDING or UPDATING, but not during shutdown or when removing all
3283 : // entries.
3284 4 : if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN &&
3285 3 : (mState == READING || mState == BUILDING || mState == UPDATING)) {
3286 1 : CacheFileIOManager::EvictIfOverLimit();
3287 : }
3288 :
3289 2 : mState = aNewState;
3290 :
3291 2 : if (mState != SHUTDOWN) {
3292 2 : CacheFileIOManager::CacheIndexStateChanged();
3293 : }
3294 :
3295 2 : NotifyAsyncGetDiskConsumptionCallbacks();
3296 : }
3297 :
3298 : void
3299 2 : CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks()
3300 : {
3301 2 : if ((mState == READY || mState == WRITING) && !mAsyncGetDiskConsumptionBlocked && mDiskConsumptionObservers.Length()) {
3302 0 : for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) {
3303 0 : DiskConsumptionObserver* o = mDiskConsumptionObservers[i];
3304 : // Safe to call under the lock. We always post to the main thread.
3305 0 : o->OnDiskConsumption(mIndexStats.Size() << 10);
3306 : }
3307 :
3308 0 : mDiskConsumptionObservers.Clear();
3309 : }
3310 2 : }
3311 :
3312 : void
3313 0 : CacheIndex::AllocBuffer()
3314 : {
3315 0 : switch (mState) {
3316 : case WRITING:
3317 0 : mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) +
3318 0 : mProcessEntries * sizeof(CacheIndexRecord);
3319 0 : if (mRWBufSize > kMaxBufSize) {
3320 0 : mRWBufSize = kMaxBufSize;
3321 : }
3322 0 : break;
3323 : case READING:
3324 0 : mRWBufSize = kMaxBufSize;
3325 0 : break;
3326 : default:
3327 0 : MOZ_ASSERT(false, "Unexpected state!");
3328 : }
3329 :
3330 0 : mRWBuf = static_cast<char *>(moz_xmalloc(mRWBufSize));
3331 0 : }
3332 :
3333 : void
3334 1 : CacheIndex::ReleaseBuffer()
3335 : {
3336 1 : sLock.AssertCurrentThreadOwns();
3337 :
3338 1 : if (!mRWBuf || mRWPending) {
3339 1 : return;
3340 : }
3341 :
3342 0 : LOG(("CacheIndex::ReleaseBuffer() releasing buffer"));
3343 :
3344 0 : free(mRWBuf);
3345 0 : mRWBuf = nullptr;
3346 0 : mRWBufSize = 0;
3347 0 : mRWBufPos = 0;
3348 : }
3349 :
3350 : void
3351 14 : CacheIndex::FrecencyArray::AppendRecord(CacheIndexRecord *aRecord)
3352 : {
3353 14 : LOG(("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x"
3354 : "%08x%08x]", aRecord, LOGSHA1(aRecord->mHash)));
3355 :
3356 14 : MOZ_ASSERT(!mRecs.Contains(aRecord));
3357 14 : mRecs.AppendElement(aRecord);
3358 :
3359 : // If the new frecency is 0, the element should be at the end of the array,
3360 : // i.e. this change doesn't affect order of the array
3361 14 : if (aRecord->mFrecency != 0) {
3362 9 : ++mUnsortedElements;
3363 : }
3364 14 : }
3365 :
3366 : void
3367 9 : CacheIndex::FrecencyArray::RemoveRecord(CacheIndexRecord *aRecord)
3368 : {
3369 9 : LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord));
3370 :
3371 : decltype(mRecs)::index_type idx;
3372 9 : idx = mRecs.IndexOf(aRecord);
3373 9 : MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3374 9 : mRecs[idx] = nullptr;
3375 9 : ++mRemovedElements;
3376 :
3377 : // Calling SortIfNeeded ensures that we get rid of removed elements in the
3378 : // array once we hit the limit.
3379 9 : SortIfNeeded();
3380 9 : }
3381 :
3382 : void
3383 0 : CacheIndex::FrecencyArray::ReplaceRecord(CacheIndexRecord *aOldRecord,
3384 : CacheIndexRecord *aNewRecord)
3385 : {
3386 0 : LOG(("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, "
3387 : "newRecord=%p]", aOldRecord, aNewRecord));
3388 :
3389 : decltype(mRecs)::index_type idx;
3390 0 : idx = mRecs.IndexOf(aOldRecord);
3391 0 : MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3392 0 : mRecs[idx] = aNewRecord;
3393 0 : }
3394 :
3395 : void
3396 9 : CacheIndex::FrecencyArray::SortIfNeeded()
3397 : {
3398 9 : const uint32_t kMaxUnsortedCount = 512;
3399 9 : const uint32_t kMaxUnsortedPercent = 10;
3400 9 : const uint32_t kMaxRemovedCount = 512;
3401 :
3402 : uint32_t unsortedLimit =
3403 9 : std::min<uint32_t>(kMaxUnsortedCount, Length() * kMaxUnsortedPercent / 100);
3404 :
3405 10 : if (mUnsortedElements > unsortedLimit ||
3406 1 : mRemovedElements > kMaxRemovedCount) {
3407 8 : LOG(("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array "
3408 : "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, "
3409 : "maxRemovedCount=%u]", mUnsortedElements, unsortedLimit,
3410 : mRemovedElements, kMaxRemovedCount));
3411 :
3412 8 : mRecs.Sort(FrecencyComparator());
3413 8 : mUnsortedElements = 0;
3414 8 : if (mRemovedElements) {
3415 : #ifdef DEBUG
3416 17 : for (uint32_t i = Length(); i < mRecs.Length(); ++i) {
3417 9 : MOZ_ASSERT(!mRecs[i]);
3418 : }
3419 : #endif
3420 : // Removed elements are at the end after sorting.
3421 8 : mRecs.RemoveElementsAt(Length(), mRemovedElements);
3422 8 : mRemovedElements = 0;
3423 : }
3424 : }
3425 9 : }
3426 :
3427 : void
3428 5 : CacheIndex::AddRecordToIterators(CacheIndexRecord *aRecord)
3429 : {
3430 5 : sLock.AssertCurrentThreadOwns();
3431 :
3432 5 : for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3433 : // Add a new record only when iterator is supposed to be updated.
3434 0 : if (mIterators[i]->ShouldBeNewAdded()) {
3435 0 : mIterators[i]->AddRecord(aRecord);
3436 : }
3437 : }
3438 5 : }
3439 :
3440 : void
3441 0 : CacheIndex::RemoveRecordFromIterators(CacheIndexRecord *aRecord)
3442 : {
3443 0 : sLock.AssertCurrentThreadOwns();
3444 :
3445 0 : for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3446 : // Remove the record from iterator always, it makes no sence to return
3447 : // non-existing entries. Also the pointer to the record is no longer valid
3448 : // once the entry is removed from index.
3449 0 : mIterators[i]->RemoveRecord(aRecord);
3450 : }
3451 0 : }
3452 :
3453 : void
3454 0 : CacheIndex::ReplaceRecordInIterators(CacheIndexRecord *aOldRecord,
3455 : CacheIndexRecord *aNewRecord)
3456 : {
3457 0 : sLock.AssertCurrentThreadOwns();
3458 :
3459 0 : for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3460 : // We have to replace the record always since the pointer is no longer
3461 : // valid after this point. NOTE: Replacing the record doesn't mean that
3462 : // a new entry was added, it just means that the data in the entry was
3463 : // changed (e.g. a file size) and we had to track this change in
3464 : // mPendingUpdates since mIndex was read-only.
3465 0 : mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord);
3466 : }
3467 0 : }
3468 :
3469 : nsresult
3470 0 : CacheIndex::Run()
3471 : {
3472 0 : LOG(("CacheIndex::Run()"));
3473 :
3474 0 : StaticMutexAutoLock lock(sLock);
3475 :
3476 0 : if (!IsIndexUsable()) {
3477 0 : return NS_ERROR_NOT_AVAILABLE;
3478 : }
3479 :
3480 0 : if (mState == READY && mShuttingDown) {
3481 0 : return NS_OK;
3482 : }
3483 :
3484 0 : mUpdateEventPending = false;
3485 :
3486 0 : switch (mState) {
3487 : case BUILDING:
3488 0 : BuildIndex();
3489 0 : break;
3490 : case UPDATING:
3491 0 : UpdateIndex();
3492 0 : break;
3493 : default:
3494 0 : LOG(("CacheIndex::Run() - Update/Build was canceled"));
3495 : }
3496 :
3497 0 : return NS_OK;
3498 : }
3499 :
3500 : nsresult
3501 1 : CacheIndex::OnFileOpenedInternal(FileOpenHelper *aOpener,
3502 : CacheFileHandle *aHandle, nsresult aResult)
3503 : {
3504 1 : LOG(("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3505 : "result=0x%08" PRIx32 "]", aOpener, aHandle, static_cast<uint32_t>(aResult)));
3506 :
3507 1 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3508 :
3509 : nsresult rv;
3510 :
3511 1 : sLock.AssertCurrentThreadOwns();
3512 :
3513 1 : MOZ_RELEASE_ASSERT(IsIndexUsable());
3514 :
3515 1 : if (mState == READY && mShuttingDown) {
3516 0 : return NS_OK;
3517 : }
3518 :
3519 1 : switch (mState) {
3520 : case WRITING:
3521 0 : MOZ_ASSERT(aOpener == mIndexFileOpener);
3522 0 : mIndexFileOpener = nullptr;
3523 :
3524 0 : if (NS_FAILED(aResult)) {
3525 0 : LOG(("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3526 : "writing [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(aResult)));
3527 0 : FinishWrite(false);
3528 : } else {
3529 0 : mIndexHandle = aHandle;
3530 0 : WriteRecords();
3531 : }
3532 0 : break;
3533 : case READING:
3534 1 : if (aOpener == mIndexFileOpener) {
3535 1 : mIndexFileOpener = nullptr;
3536 :
3537 1 : if (NS_SUCCEEDED(aResult)) {
3538 0 : if (aHandle->FileSize() == 0) {
3539 0 : FinishRead(false);
3540 0 : CacheFileIOManager::DoomFile(aHandle, nullptr);
3541 0 : break;
3542 : } else {
3543 0 : mIndexHandle = aHandle;
3544 : }
3545 : } else {
3546 1 : FinishRead(false);
3547 1 : break;
3548 : }
3549 0 : } else if (aOpener == mJournalFileOpener) {
3550 0 : mJournalFileOpener = nullptr;
3551 0 : mJournalHandle = aHandle;
3552 0 : } else if (aOpener == mTmpFileOpener) {
3553 0 : mTmpFileOpener = nullptr;
3554 0 : mTmpHandle = aHandle;
3555 : } else {
3556 0 : MOZ_ASSERT(false, "Unexpected state!");
3557 : }
3558 :
3559 0 : if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) {
3560 : // Some opener still didn't finish
3561 0 : break;
3562 : }
3563 :
3564 : // We fail and cancel all other openers when we opening index file fails.
3565 0 : MOZ_ASSERT(mIndexHandle);
3566 :
3567 0 : if (mTmpHandle) {
3568 0 : CacheFileIOManager::DoomFile(mTmpHandle, nullptr);
3569 0 : mTmpHandle = nullptr;
3570 :
3571 0 : if (mJournalHandle) { // this shouldn't normally happen
3572 0 : LOG(("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3573 : "files [%s, %s, %s] should never exist. Removing whole index.",
3574 : INDEX_NAME, JOURNAL_NAME, TEMP_INDEX_NAME));
3575 0 : FinishRead(false);
3576 0 : break;
3577 : }
3578 : }
3579 :
3580 0 : if (mJournalHandle) {
3581 : // Rename journal to make sure we update index on next start in case
3582 : // firefox crashes
3583 0 : rv = CacheFileIOManager::RenameFile(
3584 0 : mJournalHandle, NS_LITERAL_CSTRING(TEMP_INDEX_NAME), this);
3585 0 : if (NS_FAILED(rv)) {
3586 0 : LOG(("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3587 : "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3588 : static_cast<uint32_t>(rv)));
3589 0 : FinishRead(false);
3590 0 : break;
3591 : }
3592 : } else {
3593 0 : StartReadingIndex();
3594 : }
3595 :
3596 0 : break;
3597 : default:
3598 0 : MOZ_ASSERT(false, "Unexpected state!");
3599 : }
3600 :
3601 1 : return NS_OK;
3602 : }
3603 :
3604 : nsresult
3605 0 : CacheIndex::OnFileOpened(CacheFileHandle *aHandle, nsresult aResult)
3606 : {
3607 0 : MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3608 : return NS_ERROR_UNEXPECTED;
3609 : }
3610 :
3611 : nsresult
3612 0 : CacheIndex::OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
3613 : nsresult aResult)
3614 : {
3615 0 : LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32 "]", aHandle,
3616 : static_cast<uint32_t>(aResult)));
3617 :
3618 0 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3619 :
3620 : nsresult rv;
3621 :
3622 0 : StaticMutexAutoLock lock(sLock);
3623 :
3624 0 : MOZ_RELEASE_ASSERT(IsIndexUsable());
3625 0 : MOZ_RELEASE_ASSERT(mRWPending);
3626 0 : mRWPending = false;
3627 :
3628 0 : if (mState == READY && mShuttingDown) {
3629 0 : return NS_OK;
3630 : }
3631 :
3632 0 : switch (mState) {
3633 : case WRITING:
3634 0 : MOZ_ASSERT(mIndexHandle == aHandle);
3635 :
3636 0 : if (NS_FAILED(aResult)) {
3637 0 : FinishWrite(false);
3638 : } else {
3639 0 : if (mSkipEntries == mProcessEntries) {
3640 0 : rv = CacheFileIOManager::RenameFile(mIndexHandle,
3641 0 : NS_LITERAL_CSTRING(INDEX_NAME),
3642 0 : this);
3643 0 : if (NS_FAILED(rv)) {
3644 0 : LOG(("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3645 : "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3646 : static_cast<uint32_t>(rv)));
3647 0 : FinishWrite(false);
3648 : }
3649 : } else {
3650 0 : WriteRecords();
3651 : }
3652 : }
3653 0 : break;
3654 : default:
3655 : // Writing was canceled.
3656 0 : LOG(("CacheIndex::OnDataWritten() - ignoring notification since the "
3657 : "operation was previously canceled [state=%d]", mState));
3658 0 : ReleaseBuffer();
3659 : }
3660 :
3661 0 : return NS_OK;
3662 : }
3663 :
3664 : nsresult
3665 0 : CacheIndex::OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult)
3666 : {
3667 0 : LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32 "]", aHandle,
3668 : static_cast<uint32_t>(aResult)));
3669 :
3670 0 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3671 :
3672 0 : StaticMutexAutoLock lock(sLock);
3673 :
3674 0 : MOZ_RELEASE_ASSERT(IsIndexUsable());
3675 0 : MOZ_RELEASE_ASSERT(mRWPending);
3676 0 : mRWPending = false;
3677 :
3678 0 : switch (mState) {
3679 : case READING:
3680 0 : MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle);
3681 :
3682 0 : if (NS_FAILED(aResult)) {
3683 0 : FinishRead(false);
3684 : } else {
3685 0 : if (!mIndexOnDiskIsValid) {
3686 0 : ParseRecords();
3687 : } else {
3688 0 : ParseJournal();
3689 : }
3690 : }
3691 0 : break;
3692 : default:
3693 : // Reading was canceled.
3694 0 : LOG(("CacheIndex::OnDataRead() - ignoring notification since the "
3695 : "operation was previously canceled [state=%d]", mState));
3696 0 : ReleaseBuffer();
3697 : }
3698 :
3699 0 : return NS_OK;
3700 : }
3701 :
3702 : nsresult
3703 0 : CacheIndex::OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult)
3704 : {
3705 0 : MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3706 : return NS_ERROR_UNEXPECTED;
3707 : }
3708 :
3709 : nsresult
3710 0 : CacheIndex::OnEOFSet(CacheFileHandle *aHandle, nsresult aResult)
3711 : {
3712 0 : MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3713 : return NS_ERROR_UNEXPECTED;
3714 : }
3715 :
3716 : nsresult
3717 0 : CacheIndex::OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult)
3718 : {
3719 0 : LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32 "]", aHandle,
3720 : static_cast<uint32_t>(aResult)));
3721 :
3722 0 : MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3723 :
3724 0 : StaticMutexAutoLock lock(sLock);
3725 :
3726 0 : MOZ_RELEASE_ASSERT(IsIndexUsable());
3727 :
3728 0 : if (mState == READY && mShuttingDown) {
3729 0 : return NS_OK;
3730 : }
3731 :
3732 0 : switch (mState) {
3733 : case WRITING:
3734 : // This is a result of renaming the new index written to tmpfile to index
3735 : // file. This is the last step when writing the index and the whole
3736 : // writing process is successful iff renaming was successful.
3737 :
3738 0 : if (mIndexHandle != aHandle) {
3739 0 : LOG(("CacheIndex::OnFileRenamed() - ignoring notification since it "
3740 : "belongs to previously canceled operation [state=%d]", mState));
3741 0 : break;
3742 : }
3743 :
3744 0 : FinishWrite(NS_SUCCEEDED(aResult));
3745 0 : break;
3746 : case READING:
3747 : // This is a result of renaming journal file to tmpfile. It is renamed
3748 : // before we start reading index and journal file and it should normally
3749 : // succeed. If it fails give up reading of index.
3750 :
3751 0 : if (mJournalHandle != aHandle) {
3752 0 : LOG(("CacheIndex::OnFileRenamed() - ignoring notification since it "
3753 : "belongs to previously canceled operation [state=%d]", mState));
3754 0 : break;
3755 : }
3756 :
3757 0 : if (NS_FAILED(aResult)) {
3758 0 : FinishRead(false);
3759 : } else {
3760 0 : StartReadingIndex();
3761 : }
3762 0 : break;
3763 : default:
3764 : // Reading/writing was canceled.
3765 0 : LOG(("CacheIndex::OnFileRenamed() - ignoring notification since the "
3766 : "operation was previously canceled [state=%d]", mState));
3767 : }
3768 :
3769 0 : return NS_OK;
3770 : }
3771 :
3772 : // Memory reporting
3773 :
3774 : size_t
3775 0 : CacheIndex::SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const
3776 : {
3777 0 : sLock.AssertCurrentThreadOwns();
3778 :
3779 0 : size_t n = 0;
3780 0 : nsCOMPtr<nsISizeOf> sizeOf;
3781 :
3782 : // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3783 : // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3784 : // handles array.
3785 :
3786 0 : sizeOf = do_QueryInterface(mCacheDirectory);
3787 0 : if (sizeOf) {
3788 0 : n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3789 : }
3790 :
3791 0 : sizeOf = do_QueryInterface(mUpdateTimer);
3792 0 : if (sizeOf) {
3793 0 : n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3794 : }
3795 :
3796 0 : n += mallocSizeOf(mRWBuf);
3797 0 : n += mallocSizeOf(mRWHash);
3798 :
3799 0 : n += mIndex.SizeOfExcludingThis(mallocSizeOf);
3800 0 : n += mPendingUpdates.SizeOfExcludingThis(mallocSizeOf);
3801 0 : n += mTmpJournal.SizeOfExcludingThis(mallocSizeOf);
3802 :
3803 : // mFrecencyArray items are reported by mIndex/mPendingUpdates
3804 0 : n += mFrecencyArray.mRecs.ShallowSizeOfExcludingThis(mallocSizeOf);
3805 0 : n += mDiskConsumptionObservers.ShallowSizeOfExcludingThis(mallocSizeOf);
3806 :
3807 0 : return n;
3808 : }
3809 :
3810 : // static
3811 : size_t
3812 0 : CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf)
3813 : {
3814 0 : sLock.AssertCurrentThreadOwns();
3815 :
3816 0 : if (!gInstance)
3817 0 : return 0;
3818 :
3819 0 : return gInstance->SizeOfExcludingThisInternal(mallocSizeOf);
3820 : }
3821 :
3822 : // static
3823 : size_t
3824 0 : CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf)
3825 : {
3826 0 : StaticMutexAutoLock lock(sLock);
3827 :
3828 0 : return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf);
3829 : }
3830 :
3831 : namespace {
3832 :
3833 : class HashComparator
3834 : {
3835 : public:
3836 0 : bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
3837 0 : return memcmp(&a->mHash, &b->mHash, sizeof(SHA1Sum::Hash)) == 0;
3838 : }
3839 0 : bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
3840 0 : return memcmp(&a->mHash, &b->mHash, sizeof(SHA1Sum::Hash)) < 0;
3841 : }
3842 : };
3843 :
3844 : void
3845 0 : ReportHashSizeMatch(const SHA1Sum::Hash *aHash1, const SHA1Sum::Hash *aHash2)
3846 : {
3847 0 : const uint32_t *h1 = reinterpret_cast<const uint32_t *>(aHash1);
3848 0 : const uint32_t *h2 = reinterpret_cast<const uint32_t *>(aHash2);
3849 :
3850 0 : for (uint32_t i = 0; i < 5; ++i) {
3851 0 : if (h1[i] != h2[i]) {
3852 0 : uint32_t bitsDiff = h1[i] ^ h2[i];
3853 0 : bitsDiff = NetworkEndian::readUint32(&bitsDiff);
3854 :
3855 : // count leading zeros in bitsDiff
3856 : static const uint8_t debruijn32[32] =
3857 : { 0, 31, 9, 30, 3, 8, 13, 29, 2, 5, 7, 21, 12, 24, 28, 19,
3858 : 1, 10, 4, 14, 6, 22, 25, 20, 11, 15, 23, 26, 16, 27, 17, 18};
3859 :
3860 0 : bitsDiff |= bitsDiff>>1;
3861 0 : bitsDiff |= bitsDiff>>2;
3862 0 : bitsDiff |= bitsDiff>>4;
3863 0 : bitsDiff |= bitsDiff>>8;
3864 0 : bitsDiff |= bitsDiff>>16;
3865 0 : bitsDiff++;
3866 :
3867 0 : uint8_t hashSizeMatch = debruijn32[bitsDiff*0x076be629>>27] + (i<<5);
3868 0 : Telemetry::Accumulate(Telemetry::NETWORK_CACHE_HASH_STATS, hashSizeMatch);
3869 :
3870 0 : return;
3871 : }
3872 : }
3873 :
3874 0 : MOZ_ASSERT(false, "Found a collision in the index!");
3875 : }
3876 :
3877 : } // namespace
3878 :
3879 : void
3880 0 : CacheIndex::ReportHashStats()
3881 : {
3882 : // We're gathering the hash stats only once, exclude too small caches.
3883 0 : if (CacheObserver::HashStatsReported() || mFrecencyArray.Length() < 15000) {
3884 0 : return;
3885 : }
3886 :
3887 0 : nsTArray<CacheIndexRecord *> records;
3888 0 : for (auto iter = mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
3889 0 : records.AppendElement(iter.Get());
3890 : }
3891 :
3892 0 : records.Sort(HashComparator());
3893 :
3894 0 : for (uint32_t i = 1; i < records.Length(); i++) {
3895 0 : ReportHashSizeMatch(&records[i-1]->mHash, &records[i]->mHash);
3896 : }
3897 :
3898 0 : CacheObserver::SetHashStatsReported();
3899 : }
3900 :
3901 : // static
3902 : void
3903 0 : CacheIndex::OnAsyncEviction(bool aEvicting)
3904 : {
3905 0 : RefPtr<CacheIndex> index = gInstance;
3906 0 : if (!index) {
3907 0 : return;
3908 : }
3909 :
3910 0 : StaticMutexAutoLock lock(sLock);
3911 0 : index->mAsyncGetDiskConsumptionBlocked = aEvicting;
3912 0 : if (!aEvicting) {
3913 0 : index->NotifyAsyncGetDiskConsumptionCallbacks();
3914 : }
3915 : }
3916 :
3917 : } // namespace net
3918 : } // namespace mozilla
|
