Line data Source code
1 : //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this
4 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #include "Classifier.h"
7 : #include "LookupCacheV4.h"
8 : #include "nsIPrefBranch.h"
9 : #include "nsIPrefService.h"
10 : #include "nsISimpleEnumerator.h"
11 : #include "nsIRandomGenerator.h"
12 : #include "nsIInputStream.h"
13 : #include "nsISeekableStream.h"
14 : #include "nsIFile.h"
15 : #include "nsNetCID.h"
16 : #include "nsPrintfCString.h"
17 : #include "nsThreadUtils.h"
18 : #include "mozilla/Telemetry.h"
19 : #include "mozilla/IntegerPrintfMacros.h"
20 : #include "mozilla/Logging.h"
21 : #include "mozilla/SyncRunnable.h"
22 : #include "mozilla/Base64.h"
23 : #include "mozilla/Unused.h"
24 : #include "mozilla/SizePrintfMacros.h"
25 : #include "mozilla/UniquePtr.h"
26 : #include "nsIUrlClassifierUtils.h"
27 : #include "nsUrlClassifierDBService.h"
28 :
29 : // MOZ_LOG=UrlClassifierDbService:5
30 : extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
31 : #define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
32 : #define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
33 :
34 : #define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing")
35 : #define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
36 : #define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup")
37 : #define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating")
38 :
39 : #define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata")
40 :
41 : namespace mozilla {
42 : namespace safebrowsing {
43 :
44 : namespace {
45 :
46 : // A scoped-clearer for nsTArray<TableUpdate*>.
47 : // The owning elements will be deleted and the array itself
48 : // will be cleared on exiting the scope.
49 : class ScopedUpdatesClearer {
50 : public:
51 1 : explicit ScopedUpdatesClearer(nsTArray<TableUpdate*> *aUpdates)
52 1 : : mUpdatesArrayRef(aUpdates)
53 : {
54 7 : for (auto update : *aUpdates) {
55 6 : mUpdatesPointerHolder.AppendElement(update);
56 : }
57 1 : }
58 :
59 1 : ~ScopedUpdatesClearer()
60 1 : {
61 1 : mUpdatesArrayRef->Clear();
62 1 : }
63 :
64 : private:
65 : nsTArray<TableUpdate*>* mUpdatesArrayRef;
66 : nsTArray<UniquePtr<TableUpdate>> mUpdatesPointerHolder;
67 : };
68 :
69 : } // End of unnamed namespace.
70 :
71 : void
72 9 : Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
73 : {
74 9 : tables.Clear();
75 :
76 9 : nsACString::const_iterator begin, iter, end;
77 9 : str.BeginReading(begin);
78 9 : str.EndReading(end);
79 179 : while (begin != end) {
80 85 : iter = begin;
81 85 : FindCharInReadable(',', iter, end);
82 170 : nsDependentCSubstring table = Substring(begin,iter);
83 85 : if (!table.IsEmpty()) {
84 85 : tables.AppendElement(Substring(begin, iter));
85 : }
86 85 : begin = iter;
87 85 : if (begin != end) {
88 76 : begin++;
89 : }
90 : }
91 9 : }
92 :
93 : nsresult
94 62 : Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
95 : const nsACString& aTableName,
96 : const nsACString& aProvider,
97 : nsIFile** aPrivateStoreDirectory)
98 : {
99 62 : NS_ENSURE_ARG_POINTER(aPrivateStoreDirectory);
100 :
101 62 : if (!StringEndsWith(aTableName, NS_LITERAL_CSTRING("-proto"))) {
102 : // Only V4 table names (ends with '-proto') would be stored
103 : // to per-provider sub-directory.
104 62 : nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
105 62 : return NS_OK;
106 : }
107 :
108 0 : if (aProvider.IsEmpty()) {
109 : // When failing to get provider, just store in the root folder.
110 0 : nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
111 0 : return NS_OK;
112 : }
113 :
114 0 : nsCOMPtr<nsIFile> providerDirectory;
115 :
116 : // Clone first since we are gonna create a new directory.
117 0 : nsresult rv = aRootStoreDirectory->Clone(getter_AddRefs(providerDirectory));
118 0 : NS_ENSURE_SUCCESS(rv, rv);
119 :
120 : // Append the provider name to the root store directory.
121 0 : rv = providerDirectory->AppendNative(aProvider);
122 0 : NS_ENSURE_SUCCESS(rv, rv);
123 :
124 : // Ensure existence of the provider directory.
125 : bool dirExists;
126 0 : rv = providerDirectory->Exists(&dirExists);
127 0 : NS_ENSURE_SUCCESS(rv, rv);
128 :
129 0 : if (!dirExists) {
130 0 : LOG(("Creating private directory for %s", nsCString(aTableName).get()));
131 0 : rv = providerDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
132 0 : NS_ENSURE_SUCCESS(rv, rv);
133 0 : providerDirectory.forget(aPrivateStoreDirectory);
134 0 : return rv;
135 : }
136 :
137 : // Store directory exists. Check if it's a directory.
138 : bool isDir;
139 0 : rv = providerDirectory->IsDirectory(&isDir);
140 0 : NS_ENSURE_SUCCESS(rv, rv);
141 0 : if (!isDir) {
142 0 : return NS_ERROR_FILE_DESTINATION_NOT_DIR;
143 : }
144 :
145 0 : providerDirectory.forget(aPrivateStoreDirectory);
146 :
147 0 : return NS_OK;
148 : }
149 :
150 1 : Classifier::Classifier()
151 : : mIsTableRequestResultOutdated(true)
152 1 : , mUpdateInterrupted(true)
153 : {
154 3 : NS_NewNamedThread(NS_LITERAL_CSTRING("Classifier Update"),
155 4 : getter_AddRefs(mUpdateThread));
156 1 : }
157 :
158 0 : Classifier::~Classifier()
159 : {
160 0 : Close();
161 0 : }
162 :
163 : nsresult
164 1 : Classifier::SetupPathNames()
165 : {
166 : // Get the root directory where to store all the databases.
167 1 : nsresult rv = mCacheDirectory->Clone(getter_AddRefs(mRootStoreDirectory));
168 1 : NS_ENSURE_SUCCESS(rv, rv);
169 :
170 1 : rv = mRootStoreDirectory->AppendNative(STORE_DIRECTORY);
171 1 : NS_ENSURE_SUCCESS(rv, rv);
172 :
173 : // Make sure LookupCaches (which are persistent and survive updates)
174 : // are reading/writing in the right place. We will be moving their
175 : // files "underneath" them during backup/restore.
176 1 : for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
177 0 : mLookupCaches[i]->UpdateRootDirHandle(mRootStoreDirectory);
178 : }
179 :
180 : // Directory where to move a backup before an update.
181 1 : rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory));
182 1 : NS_ENSURE_SUCCESS(rv, rv);
183 :
184 1 : rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
185 1 : NS_ENSURE_SUCCESS(rv, rv);
186 :
187 : // Directory where to be working on the update.
188 1 : rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory));
189 1 : NS_ENSURE_SUCCESS(rv, rv);
190 :
191 1 : rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX);
192 1 : NS_ENSURE_SUCCESS(rv, rv);
193 :
194 : // Directory where to move the backup so we can atomically
195 : // delete (really move) it.
196 1 : rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
197 1 : NS_ENSURE_SUCCESS(rv, rv);
198 :
199 1 : rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX);
200 1 : NS_ENSURE_SUCCESS(rv, rv);
201 :
202 1 : return NS_OK;
203 : }
204 :
205 : nsresult
206 1 : Classifier::CreateStoreDirectory()
207 : {
208 : // Ensure the safebrowsing directory exists.
209 : bool storeExists;
210 1 : nsresult rv = mRootStoreDirectory->Exists(&storeExists);
211 1 : NS_ENSURE_SUCCESS(rv, rv);
212 :
213 1 : if (!storeExists) {
214 0 : rv = mRootStoreDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
215 0 : NS_ENSURE_SUCCESS(rv, rv);
216 : } else {
217 : bool storeIsDir;
218 1 : rv = mRootStoreDirectory->IsDirectory(&storeIsDir);
219 1 : NS_ENSURE_SUCCESS(rv, rv);
220 1 : if (!storeIsDir)
221 0 : return NS_ERROR_FILE_DESTINATION_NOT_DIR;
222 : }
223 :
224 1 : return NS_OK;
225 : }
226 :
227 : nsresult
228 1 : Classifier::Open(nsIFile& aCacheDirectory)
229 : {
230 : // Remember the Local profile directory.
231 1 : nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory));
232 1 : NS_ENSURE_SUCCESS(rv, rv);
233 :
234 : // Create the handles to the update and backup directories.
235 1 : rv = SetupPathNames();
236 1 : NS_ENSURE_SUCCESS(rv, rv);
237 :
238 : // Clean up any to-delete directories that haven't been deleted yet.
239 : // This is still required for backward compatibility.
240 1 : rv = CleanToDelete();
241 1 : NS_ENSURE_SUCCESS(rv, rv);
242 :
243 : // If we met a crash during the previous update, "safebrowsing-updating"
244 : // directory will exist and let's remove it.
245 1 : rv = mUpdatingDirectory->Remove(true);
246 1 : if (NS_SUCCEEDED(rv)) {
247 : // If the "safebrowsing-updating" exists, it implies a crash occurred
248 : // in the previous update.
249 0 : LOG(("We may have hit a crash in the previous update."));
250 : }
251 :
252 : // Check whether we have an incomplete update and recover from the
253 : // backup if so.
254 1 : rv = RecoverBackups();
255 1 : NS_ENSURE_SUCCESS(rv, rv);
256 :
257 : // Make sure the main store directory exists.
258 1 : rv = CreateStoreDirectory();
259 1 : NS_ENSURE_SUCCESS(rv, rv);
260 :
261 1 : mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
262 1 : NS_ENSURE_SUCCESS(rv, rv);
263 :
264 : // Build the list of know urlclassifier lists
265 : // XXX: Disk IO potentially on the main thread during startup
266 1 : RegenActiveTables();
267 :
268 1 : return NS_OK;
269 : }
270 :
271 : void
272 0 : Classifier::Close()
273 : {
274 0 : DropStores();
275 0 : }
276 :
277 : void
278 0 : Classifier::Reset()
279 : {
280 0 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
281 : "Reset() MUST NOT be called on update thread");
282 :
283 0 : LOG(("Reset() is called so we interrupt the update."));
284 0 : mUpdateInterrupted = true;
285 :
286 0 : auto resetFunc = [=] {
287 0 : DropStores();
288 :
289 0 : mRootStoreDirectory->Remove(true);
290 0 : mBackupDirectory->Remove(true);
291 0 : mUpdatingDirectory->Remove(true);
292 0 : mToDeleteDirectory->Remove(true);
293 :
294 0 : CreateStoreDirectory();
295 :
296 0 : RegenActiveTables();
297 0 : };
298 :
299 0 : if (!mUpdateThread) {
300 0 : LOG(("Async update has been disabled. Just Reset() on worker thread."));
301 0 : resetFunc();
302 0 : return;
303 : }
304 :
305 : nsCOMPtr<nsIRunnable> r =
306 0 : NS_NewRunnableFunction("safebrowsing::Classifier::Reset", resetFunc);
307 0 : SyncRunnable::DispatchToThread(mUpdateThread, r);
308 : }
309 :
310 : void
311 0 : Classifier::ResetTables(ClearType aType, const nsTArray<nsCString>& aTables)
312 : {
313 0 : for (uint32_t i = 0; i < aTables.Length(); i++) {
314 0 : LOG(("Resetting table: %s", aTables[i].get()));
315 0 : LookupCache *cache = GetLookupCache(aTables[i]);
316 0 : if (cache) {
317 : // Remove any cached Completes for this table if clear type is Clear_Cache
318 0 : if (aType == Clear_Cache) {
319 0 : cache->ClearCache();
320 : } else {
321 0 : cache->ClearAll();
322 : }
323 : }
324 : }
325 :
326 : // Clear on-disk database if clear type is Clear_All
327 0 : if (aType == Clear_All) {
328 0 : DeleteTables(mRootStoreDirectory, aTables);
329 :
330 0 : RegenActiveTables();
331 : }
332 0 : }
333 :
334 : void
335 0 : Classifier::DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables)
336 : {
337 0 : nsCOMPtr<nsISimpleEnumerator> entries;
338 0 : nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
339 0 : NS_ENSURE_SUCCESS_VOID(rv);
340 :
341 : bool hasMore;
342 0 : while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
343 0 : nsCOMPtr<nsISupports> supports;
344 0 : rv = entries->GetNext(getter_AddRefs(supports));
345 0 : NS_ENSURE_SUCCESS_VOID(rv);
346 :
347 0 : nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
348 0 : NS_ENSURE_TRUE_VOID(file);
349 :
350 : // If |file| is a directory, recurse to find its entries as well.
351 : bool isDirectory;
352 0 : if (NS_FAILED(file->IsDirectory(&isDirectory))) {
353 0 : continue;
354 : }
355 0 : if (isDirectory) {
356 0 : DeleteTables(file, aTables);
357 0 : continue;
358 : }
359 :
360 0 : nsCString leafName;
361 0 : rv = file->GetNativeLeafName(leafName);
362 0 : NS_ENSURE_SUCCESS_VOID(rv);
363 :
364 0 : leafName.Truncate(leafName.RFind("."));
365 0 : if (aTables.Contains(leafName)) {
366 0 : if (NS_FAILED(file->Remove(false))) {
367 0 : NS_WARNING(nsPrintfCString("Fail to remove file %s from the disk",
368 0 : leafName.get()).get());
369 : }
370 : }
371 : }
372 0 : NS_ENSURE_SUCCESS_VOID(rv);
373 : }
374 :
375 : void
376 2 : Classifier::TableRequest(nsACString& aResult)
377 : {
378 2 : MOZ_ASSERT(!NS_IsMainThread(),
379 : "TableRequest must be called on the classifier worker thread.");
380 :
381 : // This function and all disk I/O are guaranteed to occur
382 : // on the same thread so we don't need to add a lock around.
383 2 : if (!mIsTableRequestResultOutdated) {
384 0 : aResult = mTableRequestResult;
385 0 : return;
386 : }
387 :
388 : // Generating v2 table info.
389 4 : nsTArray<nsCString> tables;
390 2 : ActiveTables(tables);
391 14 : for (uint32_t i = 0; i < tables.Length(); i++) {
392 24 : HashStore store(tables[i], GetProvider(tables[i]), mRootStoreDirectory);
393 :
394 12 : nsresult rv = store.Open();
395 12 : if (NS_FAILED(rv)) {
396 0 : continue;
397 : }
398 :
399 12 : ChunkSet &adds = store.AddChunks();
400 12 : ChunkSet &subs = store.SubChunks();
401 :
402 : // Open HashStore will always succeed even that is not a v2 table.
403 : // So skip tables without add and sub chunks.
404 12 : if (adds.Length() == 0 && subs.Length() == 0) {
405 0 : continue;
406 : }
407 :
408 12 : aResult.Append(store.TableName());
409 12 : aResult.Append(';');
410 :
411 12 : if (adds.Length() > 0) {
412 12 : aResult.AppendLiteral("a:");
413 24 : nsAutoCString addList;
414 12 : adds.Serialize(addList);
415 12 : aResult.Append(addList);
416 : }
417 :
418 12 : if (subs.Length() > 0) {
419 0 : if (adds.Length() > 0)
420 0 : aResult.Append(':');
421 0 : aResult.AppendLiteral("s:");
422 0 : nsAutoCString subList;
423 0 : subs.Serialize(subList);
424 0 : aResult.Append(subList);
425 : }
426 :
427 12 : aResult.Append('\n');
428 : }
429 :
430 : // Load meta data from *.metadata files in the root directory.
431 : // Specifically for v4 tables.
432 4 : nsCString metadata;
433 2 : nsresult rv = LoadMetadata(mRootStoreDirectory, metadata);
434 2 : if (NS_SUCCEEDED(rv)) {
435 2 : aResult.Append(metadata);
436 : }
437 :
438 : // Update the TableRequest result in-memory cache.
439 2 : mTableRequestResult = aResult;
440 2 : mIsTableRequestResultOutdated = false;
441 : }
442 :
443 : nsresult
444 1 : Classifier::Check(const nsACString& aSpec,
445 : const nsACString& aTables,
446 : LookupResultArray& aResults)
447 : {
448 2 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_CHECK_TIME> timer;
449 :
450 : // Get the set of fragments based on the url. This is necessary because we
451 : // only look up at most 5 URLs per aSpec, even if aSpec has more than 5
452 : // components.
453 2 : nsTArray<nsCString> fragments;
454 1 : nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments);
455 1 : NS_ENSURE_SUCCESS(rv, rv);
456 :
457 2 : nsTArray<nsCString> activeTables;
458 1 : SplitTables(aTables, activeTables);
459 :
460 2 : nsTArray<LookupCache*> cacheArray;
461 3 : for (uint32_t i = 0; i < activeTables.Length(); i++) {
462 2 : LOG(("Checking table %s", activeTables[i].get()));
463 2 : LookupCache *cache = GetLookupCache(activeTables[i]);
464 2 : if (cache) {
465 2 : cacheArray.AppendElement(cache);
466 : } else {
467 0 : return NS_ERROR_FAILURE;
468 : }
469 : }
470 :
471 : // Now check each lookup fragment against the entries in the DB.
472 3 : for (uint32_t i = 0; i < fragments.Length(); i++) {
473 : Completion lookupHash;
474 2 : lookupHash.FromPlaintext(fragments[i], mCryptoHash);
475 :
476 2 : if (LOG_ENABLED()) {
477 0 : nsAutoCString checking;
478 0 : lookupHash.ToHexString(checking);
479 0 : LOG(("Checking fragment %s, hash %s (%X)", fragments[i].get(),
480 : checking.get(), lookupHash.ToUint32()));
481 : }
482 :
483 6 : for (uint32_t i = 0; i < cacheArray.Length(); i++) {
484 4 : LookupCache *cache = cacheArray[i];
485 : bool has, confirmed;
486 : uint32_t matchLength;
487 :
488 4 : rv = cache->Has(lookupHash, &has, &matchLength, &confirmed);
489 4 : NS_ENSURE_SUCCESS(rv, rv);
490 :
491 4 : if (has) {
492 0 : LookupResult *result = aResults.AppendElement();
493 0 : if (!result)
494 0 : return NS_ERROR_OUT_OF_MEMORY;
495 :
496 0 : LOG(("Found a result in %s: %s",
497 : cache->TableName().get(),
498 : confirmed ? "confirmed." : "Not confirmed."));
499 :
500 0 : result->hash.complete = lookupHash;
501 0 : result->mConfirmed = confirmed;
502 0 : result->mTableName.Assign(cache->TableName());
503 0 : result->mPartialHashLength = confirmed ? COMPLETE_SIZE : matchLength;
504 0 : result->mProtocolV2 = LookupCache::Cast<LookupCacheV2>(cache);
505 : }
506 : }
507 : }
508 :
509 1 : return NS_OK;
510 : }
511 :
512 : static nsresult
513 1 : SwapDirectoryContent(nsIFile* aDir1,
514 : nsIFile* aDir2,
515 : nsIFile* aParentDir,
516 : nsIFile* aTempDir)
517 : {
518 : // Pre-condition: |aDir1| and |aDir2| are directory and their parent
519 : // are both |aParentDir|.
520 : //
521 : // Post-condition: The locations where aDir1 and aDir2 point to will not
522 : // change but their contents will be exchanged. If we failed
523 : // to swap their content, everything will be rolled back.
524 :
525 2 : nsAutoCString tempDirName;
526 1 : aTempDir->GetNativeLeafName(tempDirName);
527 :
528 : nsresult rv;
529 :
530 2 : nsAutoCString dirName1, dirName2;
531 1 : aDir1->GetNativeLeafName(dirName1);
532 1 : aDir2->GetNativeLeafName(dirName2);
533 :
534 1 : LOG(("Swapping directories %s and %s...", dirName1.get(),
535 : dirName2.get()));
536 :
537 : // 1. Rename "dirName1" to "temp"
538 1 : rv = aDir1->RenameToNative(nullptr, tempDirName);
539 1 : if (NS_FAILED(rv)) {
540 0 : LOG(("Unable to rename %s to %s", dirName1.get(),
541 : tempDirName.get()));
542 0 : return rv; // Nothing to roll back.
543 : }
544 :
545 : // 1.1. Create a handle for temp directory. This is required since
546 : // |nsIFile.rename| will not change the location where the
547 : // object points to.
548 2 : nsCOMPtr<nsIFile> tempDirectory;
549 1 : rv = aParentDir->Clone(getter_AddRefs(tempDirectory));
550 1 : rv = tempDirectory->AppendNative(tempDirName);
551 :
552 : // 2. Rename "dirName2" to "dirName1".
553 1 : rv = aDir2->RenameToNative(nullptr, dirName1);
554 1 : if (NS_FAILED(rv)) {
555 0 : LOG(("Failed to rename %s to %s. Rename temp directory back to %s",
556 : dirName2.get(), dirName1.get(), dirName1.get()));
557 0 : nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
558 0 : NS_ENSURE_SUCCESS(rbrv, rbrv);
559 0 : return rv;
560 : }
561 :
562 : // 3. Rename "temp" to "dirName2".
563 1 : rv = tempDirectory->RenameToNative(nullptr, dirName2);
564 1 : if (NS_FAILED(rv)) {
565 0 : LOG(("Failed to rename temp directory to %s. ", dirName2.get()));
566 : // We've done (1) renaming "dir1 to temp" and
567 : // (2) renaming "dir2 to dir1"
568 : // so the rollback is
569 : // (1) renaming "dir1 to dir2" and
570 : // (2) renaming "temp to dir1"
571 : nsresult rbrv; // rollback result
572 0 : rbrv = aDir1->RenameToNative(nullptr, dirName2);
573 0 : NS_ENSURE_SUCCESS(rbrv, rbrv);
574 0 : rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
575 0 : NS_ENSURE_SUCCESS(rbrv, rbrv);
576 0 : return rv;
577 : }
578 :
579 1 : return rv;
580 : }
581 :
582 : void
583 1 : Classifier::RemoveUpdateIntermediaries()
584 : {
585 : // Remove old LookupCaches.
586 7 : for (auto c: mNewLookupCaches) {
587 6 : delete c;
588 : }
589 1 : mNewLookupCaches.Clear();
590 :
591 : // Remove the "old" directory. (despite its looking-new name)
592 1 : if (NS_FAILED(mUpdatingDirectory->Remove(true))) {
593 : // If the directory is locked from removal for some reason,
594 : // we will fail here and it doesn't matter until the next
595 : // update. (the next udpate will fail due to the removable
596 : // "safebrowsing-udpating" directory.)
597 0 : LOG(("Failed to remove updating directory."));
598 : }
599 1 : }
600 :
601 : void
602 1 : Classifier::CopyAndInvalidateFullHashCache()
603 : {
604 1 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
605 : "CopyAndInvalidateFullHashCache cannot be called on update thread "
606 : "since it mutates mLookupCaches which is only safe on "
607 : "worker thread.");
608 :
609 : // New lookup caches are built from disk, data likes cache which is
610 : // generated online won't exist. We have to manually copy cache from
611 : // old LookupCache to new LookupCache.
612 7 : for (auto& newCache: mNewLookupCaches) {
613 21 : for (auto& oldCache: mLookupCaches) {
614 21 : if (oldCache->TableName() == newCache->TableName()) {
615 6 : newCache->CopyFullHashCache(oldCache);
616 6 : break;
617 : }
618 : }
619 : }
620 :
621 : // Clear cache when update.
622 : // Invalidate cache entries in CopyAndInvalidateFullHashCache because only
623 : // at this point we will have cache data in LookupCache.
624 7 : for (auto& newCache: mNewLookupCaches) {
625 6 : newCache->InvalidateExpiredCacheEntries();
626 : }
627 1 : }
628 :
629 : void
630 1 : Classifier::MergeNewLookupCaches()
631 : {
632 1 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
633 : "MergeNewLookupCaches cannot be called on update thread "
634 : "since it mutates mLookupCaches which is only safe on "
635 : "worker thread.");
636 :
637 7 : for (auto& newCache: mNewLookupCaches) {
638 : // For each element in mNewLookCaches, it will be swapped with
639 : // - An old cache in mLookupCache with the same table name or
640 : // - nullptr (mLookupCache will be expaned) otherwise.
641 6 : size_t swapIndex = 0;
642 36 : for (; swapIndex < mLookupCaches.Length(); swapIndex++) {
643 21 : if (mLookupCaches[swapIndex]->TableName() == newCache->TableName()) {
644 6 : break;
645 : }
646 : }
647 6 : if (swapIndex == mLookupCaches.Length()) {
648 0 : mLookupCaches.AppendElement(nullptr);
649 : }
650 :
651 6 : Swap(mLookupCaches[swapIndex], newCache);
652 6 : mLookupCaches[swapIndex]->UpdateRootDirHandle(mRootStoreDirectory);
653 : }
654 :
655 : // At this point, mNewLookupCaches's length remains the same but
656 : // will contain either old cache (override) or nullptr (append).
657 1 : }
658 :
659 : nsresult
660 1 : Classifier::SwapInNewTablesAndCleanup()
661 : {
662 : nsresult rv;
663 :
664 : // Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup"
665 : // as the intermediary directory is we can get databases recovered if
666 : // crash occurred in any step of the swap. (We will recover from
667 : // "safebrowsing-backup" in OpenDb().)
668 1 : rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables
669 : mRootStoreDirectory, // contains old tables
670 : mCacheDirectory, // common parent dir
671 1 : mBackupDirectory); // intermediary dir for swap
672 1 : if (NS_FAILED(rv)) {
673 0 : LOG(("Failed to swap in on-disk tables."));
674 0 : RemoveUpdateIntermediaries();
675 0 : return rv;
676 : }
677 :
678 : // Step 2. Merge mNewLookupCaches into mLookupCaches. The outdated
679 : // LookupCaches will be stored in mNewLookupCaches and be cleaned
680 : // up later.
681 1 : MergeNewLookupCaches();
682 :
683 : // Step 3. Re-generate active tables based on on-disk tables.
684 1 : rv = RegenActiveTables();
685 1 : if (NS_FAILED(rv)) {
686 0 : LOG(("Failed to re-generate active tables!"));
687 : }
688 :
689 : // Step 4. Clean up intermediaries for update.
690 1 : RemoveUpdateIntermediaries();
691 :
692 : // Step 5. Invalidate cached tableRequest request.
693 1 : mIsTableRequestResultOutdated = true;
694 :
695 1 : LOG(("Done swap in updated tables."));
696 :
697 1 : return rv;
698 : }
699 :
700 0 : void Classifier::FlushAndDisableAsyncUpdate()
701 : {
702 0 : LOG(("Classifier::FlushAndDisableAsyncUpdate [%p, %p]", this, mUpdateThread.get()));
703 :
704 0 : if (!mUpdateThread) {
705 0 : LOG(("Async update has been disabled."));
706 0 : return;
707 : }
708 :
709 0 : mUpdateThread->Shutdown();
710 0 : mUpdateThread = nullptr;
711 : }
712 :
713 : nsresult
714 1 : Classifier::AsyncApplyUpdates(nsTArray<TableUpdate*>* aUpdates,
715 : const AsyncUpdateCallback& aCallback)
716 : {
717 1 : LOG(("Classifier::AsyncApplyUpdates"));
718 :
719 1 : if (!mUpdateThread) {
720 0 : LOG(("Async update has already been disabled."));
721 0 : return NS_ERROR_FAILURE;
722 : }
723 :
724 : // Caller thread | Update thread
725 : // --------------------------------------------------------
726 : // | ApplyUpdatesBackground
727 : // (processing other task) | (bg-update done. ping back to caller thread)
728 : // (processing other task) | idle...
729 : // ApplyUpdatesForeground |
730 : // callback |
731 :
732 1 : mUpdateInterrupted = false;
733 1 : nsresult rv = mRootStoreDirectory->Clone(getter_AddRefs(mRootStoreDirectoryForUpdate));
734 1 : if (NS_FAILED(rv)) {
735 0 : LOG(("Failed to clone mRootStoreDirectory for update."));
736 0 : return rv;
737 : }
738 :
739 2 : nsCOMPtr<nsIThread> callerThread = NS_GetCurrentThread();
740 1 : MOZ_ASSERT(callerThread != mUpdateThread);
741 :
742 : nsCOMPtr<nsIRunnable> bgRunnable =
743 7 : NS_NewRunnableFunction("safebrowsing::Classifier::AsyncApplyUpdates", [=] {
744 3 : MOZ_ASSERT(NS_GetCurrentThread() == mUpdateThread,
745 : "MUST be on update thread");
746 :
747 1 : LOG(("Step 1. ApplyUpdatesBackground on update thread."));
748 2 : nsCString failedTableName;
749 2 : nsresult bgRv = ApplyUpdatesBackground(aUpdates, failedTableName);
750 :
751 4 : nsCOMPtr<nsIRunnable> fgRunnable = NS_NewRunnableFunction(
752 4 : "safebrowsing::Classifier::AsyncApplyUpdates", [=] {
753 1 : MOZ_ASSERT(NS_GetCurrentThread() == callerThread,
754 : "MUST be on caller thread");
755 :
756 1 : LOG(("Step 2. ApplyUpdatesForeground on caller thread"));
757 1 : nsresult rv = ApplyUpdatesForeground(bgRv, failedTableName);
758 : ;
759 :
760 1 : LOG(("Step 3. Updates applied! Fire callback."));
761 :
762 1 : aCallback(rv);
763 4 : });
764 1 : callerThread->Dispatch(fgRunnable, NS_DISPATCH_NORMAL);
765 4 : });
766 :
767 1 : return mUpdateThread->Dispatch(bgRunnable, NS_DISPATCH_NORMAL);
768 : }
769 :
770 : nsresult
771 1 : Classifier::ApplyUpdatesBackground(nsTArray<TableUpdate*>* aUpdates,
772 : nsACString& aFailedTableName)
773 : {
774 : // |mUpdateInterrupted| is guaranteed to have been unset.
775 : // If |mUpdateInterrupted| is set at any point, Reset() must have
776 : // been called then we need to interrupt the update process.
777 : // We only add checkpoints for non-trivial tasks.
778 :
779 1 : if (!aUpdates || aUpdates->Length() == 0) {
780 0 : return NS_OK;
781 : }
782 :
783 : nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
784 2 : do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
785 :
786 2 : nsCString provider;
787 : // Assume all TableUpdate objects should have the same provider.
788 1 : urlUtil->GetTelemetryProvider((*aUpdates)[0]->TableName(), provider);
789 :
790 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_KEYED_UPDATE_TIME>
791 2 : keyedTimer(provider);
792 :
793 1 : PRIntervalTime clockStart = 0;
794 1 : if (LOG_ENABLED()) {
795 0 : clockStart = PR_IntervalNow();
796 : }
797 :
798 : nsresult rv;
799 :
800 : {
801 2 : ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
802 :
803 : {
804 : // Check point 1: Copying file takes time so we check here.
805 1 : if (mUpdateInterrupted) {
806 0 : LOG(("Update is interrupted. Don't copy files."));
807 0 : return NS_OK;
808 : }
809 :
810 1 : rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup.
811 1 : if (NS_FAILED(rv)) {
812 0 : LOG(("Failed to copy in-use directory for update."));
813 0 : return rv;
814 : }
815 : }
816 :
817 1 : LOG(("Applying %" PRIuSIZE " table updates.", aUpdates->Length()));
818 :
819 7 : for (uint32_t i = 0; i < aUpdates->Length(); i++) {
820 : // Previous UpdateHashStore() may have consumed this update..
821 6 : if ((*aUpdates)[i]) {
822 : // Run all updates for one table
823 12 : nsCString updateTable(aUpdates->ElementAt(i)->TableName());
824 :
825 : // Check point 2: Processing downloaded data takes time.
826 6 : if (mUpdateInterrupted) {
827 0 : LOG(("Update is interrupted. Stop building new tables."));
828 0 : return NS_OK;
829 : }
830 :
831 : // Will update the mirrored in-memory and on-disk databases.
832 6 : if (TableUpdate::Cast<TableUpdateV2>((*aUpdates)[i])) {
833 6 : rv = UpdateHashStore(aUpdates, updateTable);
834 : } else {
835 0 : rv = UpdateTableV4(aUpdates, updateTable);
836 : }
837 :
838 6 : if (NS_FAILED(rv)) {
839 0 : aFailedTableName = updateTable;
840 0 : RemoveUpdateIntermediaries();
841 0 : return rv;
842 : }
843 : }
844 : }
845 :
846 : } // End of scopedUpdatesClearer scope.
847 :
848 1 : if (LOG_ENABLED()) {
849 0 : PRIntervalTime clockEnd = PR_IntervalNow();
850 0 : LOG(("update took %dms\n",
851 : PR_IntervalToMilliseconds(clockEnd - clockStart)));
852 : }
853 :
854 1 : return rv;
855 : }
856 :
857 : nsresult
858 1 : Classifier::ApplyUpdatesForeground(nsresult aBackgroundRv,
859 : const nsACString& aFailedTableName)
860 : {
861 1 : if (mUpdateInterrupted) {
862 0 : LOG(("Update is interrupted! Just remove update intermediaries."));
863 0 : RemoveUpdateIntermediaries();
864 0 : return NS_OK;
865 : }
866 1 : if (NS_SUCCEEDED(aBackgroundRv)) {
867 : // Copy and Invalidate fullhash cache here because this call requires
868 : // mLookupCaches which is only available on work-thread
869 1 : CopyAndInvalidateFullHashCache();
870 :
871 1 : return SwapInNewTablesAndCleanup();
872 : }
873 0 : if (NS_ERROR_OUT_OF_MEMORY != aBackgroundRv) {
874 0 : ResetTables(Clear_All, nsTArray<nsCString> { nsCString(aFailedTableName) });
875 : }
876 0 : return aBackgroundRv;
877 : }
878 :
879 : nsresult
880 0 : Classifier::ApplyFullHashes(nsTArray<TableUpdate*>* aUpdates)
881 : {
882 0 : LOG(("Applying %" PRIuSIZE " table gethashes.", aUpdates->Length()));
883 :
884 0 : ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
885 0 : for (uint32_t i = 0; i < aUpdates->Length(); i++) {
886 0 : TableUpdate *update = aUpdates->ElementAt(i);
887 :
888 0 : nsresult rv = UpdateCache(update);
889 0 : NS_ENSURE_SUCCESS(rv, rv);
890 :
891 0 : aUpdates->ElementAt(i) = nullptr;
892 : }
893 :
894 0 : return NS_OK;
895 : }
896 :
897 : void
898 0 : Classifier::GetCacheInfo(const nsACString& aTable,
899 : nsIUrlClassifierCacheInfo** aCache)
900 : {
901 0 : LookupCache* lookupCache = GetLookupCache(aTable);
902 0 : if (!lookupCache) {
903 0 : return;
904 : }
905 :
906 0 : lookupCache->GetCacheInfo(aCache);
907 : }
908 :
909 : void
910 0 : Classifier::DropStores()
911 : {
912 0 : for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
913 0 : delete mLookupCaches[i];
914 : }
915 0 : mLookupCaches.Clear();
916 0 : }
917 :
918 : nsresult
919 2 : Classifier::RegenActiveTables()
920 : {
921 2 : mActiveTablesCache.Clear();
922 :
923 4 : nsTArray<nsCString> foundTables;
924 2 : ScanStoreDir(mRootStoreDirectory, foundTables);
925 :
926 14 : for (uint32_t i = 0; i < foundTables.Length(); i++) {
927 24 : nsCString table(foundTables[i]);
928 :
929 12 : LookupCache *lookupCache = GetLookupCache(table);
930 12 : if (!lookupCache) {
931 0 : continue;
932 : }
933 :
934 12 : if (!lookupCache->IsPrimed()) {
935 0 : continue;
936 : }
937 :
938 12 : if (LookupCache::Cast<LookupCacheV4>(lookupCache)) {
939 0 : LOG(("Active v4 table: %s", table.get()));
940 : } else {
941 24 : HashStore store(table, GetProvider(table), mRootStoreDirectory);
942 :
943 12 : nsresult rv = store.Open();
944 12 : if (NS_FAILED(rv)) {
945 0 : continue;
946 : }
947 :
948 12 : const ChunkSet &adds = store.AddChunks();
949 12 : const ChunkSet &subs = store.SubChunks();
950 :
951 12 : if (adds.Length() == 0 && subs.Length() == 0) {
952 0 : continue;
953 : }
954 :
955 12 : LOG(("Active v2 table: %s", store.TableName().get()));
956 : }
957 :
958 12 : mActiveTablesCache.AppendElement(table);
959 : }
960 :
961 4 : return NS_OK;
962 : }
963 :
964 : nsresult
965 2 : Classifier::ScanStoreDir(nsIFile* aDirectory, nsTArray<nsCString>& aTables)
966 : {
967 4 : nsCOMPtr<nsISimpleEnumerator> entries;
968 2 : nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
969 2 : NS_ENSURE_SUCCESS(rv, rv);
970 :
971 : bool hasMore;
972 50 : while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
973 48 : nsCOMPtr<nsISupports> supports;
974 24 : rv = entries->GetNext(getter_AddRefs(supports));
975 24 : NS_ENSURE_SUCCESS(rv, rv);
976 :
977 48 : nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
978 :
979 : // If |file| is a directory, recurse to find its entries as well.
980 : bool isDirectory;
981 24 : if (NS_FAILED(file->IsDirectory(&isDirectory))) {
982 0 : continue;
983 : }
984 24 : if (isDirectory) {
985 0 : ScanStoreDir(file, aTables);
986 0 : continue;
987 : }
988 :
989 48 : nsCString leafName;
990 24 : rv = file->GetNativeLeafName(leafName);
991 24 : NS_ENSURE_SUCCESS(rv, rv);
992 :
993 : // Both v2 and v4 contain .pset file
994 48 : nsCString suffix(NS_LITERAL_CSTRING(".pset"));
995 :
996 24 : int32_t dot = leafName.RFind(suffix, 0);
997 24 : if (dot != -1) {
998 12 : leafName.Cut(dot, suffix.Length());
999 12 : aTables.AppendElement(leafName);
1000 : }
1001 : }
1002 2 : NS_ENSURE_SUCCESS(rv, rv);
1003 :
1004 2 : return NS_OK;
1005 : }
1006 :
1007 : nsresult
1008 2 : Classifier::ActiveTables(nsTArray<nsCString>& aTables)
1009 : {
1010 2 : aTables = mActiveTablesCache;
1011 2 : return NS_OK;
1012 : }
1013 :
1014 : nsresult
1015 1 : Classifier::CleanToDelete()
1016 : {
1017 : bool exists;
1018 1 : nsresult rv = mToDeleteDirectory->Exists(&exists);
1019 1 : NS_ENSURE_SUCCESS(rv, rv);
1020 :
1021 1 : if (exists) {
1022 0 : rv = mToDeleteDirectory->Remove(true);
1023 0 : NS_ENSURE_SUCCESS(rv, rv);
1024 : }
1025 :
1026 1 : return NS_OK;
1027 : }
1028 :
1029 : #ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
1030 :
1031 : already_AddRefed<nsIFile>
1032 0 : Classifier::GetFailedUpdateDirectroy()
1033 : {
1034 0 : nsCString failedUpdatekDirName = STORE_DIRECTORY + nsCString("-failedupdate");
1035 :
1036 0 : nsCOMPtr<nsIFile> failedUpdatekDirectory;
1037 0 : if (NS_FAILED(mCacheDirectory->Clone(getter_AddRefs(failedUpdatekDirectory))) ||
1038 0 : NS_FAILED(failedUpdatekDirectory->AppendNative(failedUpdatekDirName))) {
1039 0 : LOG(("Failed to init failedUpdatekDirectory."));
1040 0 : return nullptr;
1041 : }
1042 :
1043 0 : return failedUpdatekDirectory.forget();
1044 : }
1045 :
1046 : nsresult
1047 0 : Classifier::DumpRawTableUpdates(const nsACString& aRawUpdates)
1048 : {
1049 0 : LOG(("Dumping raw table updates..."));
1050 :
1051 0 : DumpFailedUpdate();
1052 :
1053 0 : nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy();
1054 :
1055 : // Create tableupdate.bin and dump raw table update data.
1056 0 : nsCOMPtr<nsIFile> rawTableUpdatesFile;
1057 0 : nsCOMPtr<nsIOutputStream> outputStream;
1058 0 : if (NS_FAILED(failedUpdatekDirectory->Clone(getter_AddRefs(rawTableUpdatesFile))) ||
1059 0 : NS_FAILED(rawTableUpdatesFile->AppendNative(nsCString("tableupdates.bin"))) ||
1060 0 : NS_FAILED(NS_NewLocalFileOutputStream(getter_AddRefs(outputStream),
1061 : rawTableUpdatesFile,
1062 : PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE))) {
1063 0 : LOG(("Failed to create file to dump raw table updates."));
1064 0 : return NS_ERROR_FAILURE;
1065 : }
1066 :
1067 : // Write out the data.
1068 : uint32_t written;
1069 0 : nsresult rv = outputStream->Write(aRawUpdates.BeginReading(),
1070 0 : aRawUpdates.Length(), &written);
1071 0 : NS_ENSURE_SUCCESS(rv, rv);
1072 0 : NS_ENSURE_TRUE(written == aRawUpdates.Length(), NS_ERROR_FAILURE);
1073 :
1074 0 : return rv;
1075 : }
1076 :
1077 : nsresult
1078 0 : Classifier::DumpFailedUpdate()
1079 : {
1080 0 : LOG(("Dumping failed update..."));
1081 :
1082 0 : nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy();
1083 :
1084 : // Remove the "failed update" directory no matter it exists or not.
1085 : // Failure is fine because the directory may not exist.
1086 0 : failedUpdatekDirectory->Remove(true);
1087 :
1088 0 : nsCString failedUpdatekDirName;
1089 0 : nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName);
1090 0 : NS_ENSURE_SUCCESS(rv, rv);
1091 :
1092 : // Copy the in-use directory to a clean "failed update" directory.
1093 0 : nsCOMPtr<nsIFile> inUseDirectory;
1094 0 : if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) ||
1095 0 : NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) {
1096 0 : LOG(("Failed to move in-use to the \"failed update\" directory %s",
1097 : failedUpdatekDirName.get()));
1098 0 : return NS_ERROR_FAILURE;
1099 : }
1100 :
1101 0 : return rv;
1102 : }
1103 :
1104 : #endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
1105 :
1106 : nsresult
1107 1 : Classifier::CopyInUseDirForUpdate()
1108 : {
1109 1 : LOG(("Copy in-use directory content for update."));
1110 :
1111 : // We copy everything from in-use directory to a temporary directory
1112 : // for updating.
1113 :
1114 2 : nsCString updatingDirName;
1115 1 : nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName);
1116 1 : NS_ENSURE_SUCCESS(rv, rv);
1117 :
1118 : // Remove the destination directory first (just in case) the do the copy.
1119 1 : mUpdatingDirectory->Remove(true);
1120 1 : if (!mRootStoreDirectoryForUpdate) {
1121 0 : LOG(("mRootStoreDirectoryForUpdate is null."));
1122 0 : return NS_ERROR_NULL_POINTER;
1123 : }
1124 1 : rv = mRootStoreDirectoryForUpdate->CopyToNative(nullptr, updatingDirName);
1125 1 : NS_ENSURE_SUCCESS(rv, rv);
1126 :
1127 1 : return NS_OK;
1128 : }
1129 :
1130 : nsresult
1131 1 : Classifier::RecoverBackups()
1132 : {
1133 : bool backupExists;
1134 1 : nsresult rv = mBackupDirectory->Exists(&backupExists);
1135 1 : NS_ENSURE_SUCCESS(rv, rv);
1136 :
1137 1 : if (backupExists) {
1138 : // Remove the safebrowsing dir if it exists
1139 0 : nsCString storeDirName;
1140 0 : rv = mRootStoreDirectory->GetNativeLeafName(storeDirName);
1141 0 : NS_ENSURE_SUCCESS(rv, rv);
1142 :
1143 : bool storeExists;
1144 0 : rv = mRootStoreDirectory->Exists(&storeExists);
1145 0 : NS_ENSURE_SUCCESS(rv, rv);
1146 :
1147 0 : if (storeExists) {
1148 0 : rv = mRootStoreDirectory->Remove(true);
1149 0 : NS_ENSURE_SUCCESS(rv, rv);
1150 : }
1151 :
1152 : // Move the backup to the store location
1153 0 : rv = mBackupDirectory->MoveToNative(nullptr, storeDirName);
1154 0 : NS_ENSURE_SUCCESS(rv, rv);
1155 :
1156 : // mBackupDirectory now points to storeDir, fix up.
1157 0 : rv = SetupPathNames();
1158 0 : NS_ENSURE_SUCCESS(rv, rv);
1159 : }
1160 :
1161 1 : return NS_OK;
1162 : }
1163 :
1164 : bool
1165 6 : Classifier::CheckValidUpdate(nsTArray<TableUpdate*>* aUpdates,
1166 : const nsACString& aTable)
1167 : {
1168 : // take the quick exit if there is no valid update for us
1169 : // (common case)
1170 6 : uint32_t validupdates = 0;
1171 :
1172 42 : for (uint32_t i = 0; i < aUpdates->Length(); i++) {
1173 36 : TableUpdate *update = aUpdates->ElementAt(i);
1174 36 : if (!update || !update->TableName().Equals(aTable))
1175 30 : continue;
1176 6 : if (update->Empty()) {
1177 0 : aUpdates->ElementAt(i) = nullptr;
1178 0 : continue;
1179 : }
1180 6 : validupdates++;
1181 : }
1182 :
1183 6 : if (!validupdates) {
1184 : // This can happen if the update was only valid for one table.
1185 0 : return false;
1186 : }
1187 :
1188 6 : return true;
1189 : }
1190 :
1191 : nsCString
1192 43 : Classifier::GetProvider(const nsACString& aTableName)
1193 : {
1194 : nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
1195 86 : do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
1196 :
1197 86 : nsCString provider;
1198 43 : nsresult rv = urlUtil->GetProvider(aTableName, provider);
1199 :
1200 86 : return NS_SUCCEEDED(rv) ? provider : EmptyCString();
1201 : }
1202 :
1203 : /*
1204 : * This will consume+delete updates from the passed nsTArray.
1205 : */
1206 : nsresult
1207 6 : Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
1208 : const nsACString& aTable)
1209 : {
1210 6 : if (nsUrlClassifierDBService::ShutdownHasStarted()) {
1211 0 : return NS_ERROR_UC_UPDATE_SHUTDOWNING;
1212 : }
1213 :
1214 6 : LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
1215 :
1216 12 : HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory);
1217 :
1218 6 : if (!CheckValidUpdate(aUpdates, store.TableName())) {
1219 0 : return NS_OK;
1220 : }
1221 :
1222 6 : nsresult rv = store.Open();
1223 6 : NS_ENSURE_SUCCESS(rv, rv);
1224 6 : rv = store.BeginUpdate();
1225 6 : NS_ENSURE_SUCCESS(rv, rv);
1226 :
1227 : // Read the part of the store that is (only) in the cache
1228 : LookupCacheV2* lookupCache =
1229 6 : LookupCache::Cast<LookupCacheV2>(GetLookupCacheForUpdate(store.TableName()));
1230 6 : if (!lookupCache) {
1231 0 : return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
1232 : }
1233 :
1234 12 : FallibleTArray<uint32_t> AddPrefixHashes;
1235 6 : rv = lookupCache->GetPrefixes(AddPrefixHashes);
1236 6 : NS_ENSURE_SUCCESS(rv, rv);
1237 6 : rv = store.AugmentAdds(AddPrefixHashes);
1238 6 : NS_ENSURE_SUCCESS(rv, rv);
1239 6 : AddPrefixHashes.Clear();
1240 :
1241 6 : uint32_t applied = 0;
1242 :
1243 42 : for (uint32_t i = 0; i < aUpdates->Length(); i++) {
1244 36 : TableUpdate *update = aUpdates->ElementAt(i);
1245 36 : if (!update || !update->TableName().Equals(store.TableName()))
1246 30 : continue;
1247 :
1248 6 : rv = store.ApplyUpdate(*update);
1249 6 : NS_ENSURE_SUCCESS(rv, rv);
1250 :
1251 6 : applied++;
1252 :
1253 6 : auto updateV2 = TableUpdate::Cast<TableUpdateV2>(update);
1254 6 : if (updateV2) {
1255 6 : LOG(("Applied update to table %s:", store.TableName().get()));
1256 6 : LOG((" %d add chunks", updateV2->AddChunks().Length()));
1257 6 : LOG((" %" PRIuSIZE " add prefixes", updateV2->AddPrefixes().Length()));
1258 6 : LOG((" %" PRIuSIZE " add completions", updateV2->AddCompletes().Length()));
1259 6 : LOG((" %d sub chunks", updateV2->SubChunks().Length()));
1260 6 : LOG((" %" PRIuSIZE " sub prefixes", updateV2->SubPrefixes().Length()));
1261 6 : LOG((" %" PRIuSIZE " sub completions", updateV2->SubCompletes().Length()));
1262 6 : LOG((" %d add expirations", updateV2->AddExpirations().Length()));
1263 6 : LOG((" %d sub expirations", updateV2->SubExpirations().Length()));
1264 : }
1265 :
1266 6 : aUpdates->ElementAt(i) = nullptr;
1267 : }
1268 :
1269 6 : LOG(("Applied %d update(s) to %s.", applied, store.TableName().get()));
1270 :
1271 6 : rv = store.Rebuild();
1272 6 : NS_ENSURE_SUCCESS(rv, rv);
1273 :
1274 6 : LOG(("Table %s now has:", store.TableName().get()));
1275 6 : LOG((" %d add chunks", store.AddChunks().Length()));
1276 6 : LOG((" %" PRIuSIZE " add prefixes", store.AddPrefixes().Length()));
1277 6 : LOG((" %" PRIuSIZE " add completions", store.AddCompletes().Length()));
1278 6 : LOG((" %d sub chunks", store.SubChunks().Length()));
1279 6 : LOG((" %" PRIuSIZE " sub prefixes", store.SubPrefixes().Length()));
1280 6 : LOG((" %" PRIuSIZE " sub completions", store.SubCompletes().Length()));
1281 :
1282 6 : rv = store.WriteFile();
1283 6 : NS_ENSURE_SUCCESS(rv, rv);
1284 :
1285 : // At this point the store is updated and written out to disk, but
1286 : // the data is still in memory. Build our quick-lookup table here.
1287 6 : rv = lookupCache->Build(store.AddPrefixes(), store.AddCompletes());
1288 6 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_BUILD_PREFIX_FAILURE);
1289 :
1290 : #if defined(DEBUG)
1291 6 : lookupCache->DumpCompletions();
1292 : #endif
1293 6 : rv = lookupCache->WriteFile();
1294 6 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
1295 :
1296 6 : LOG(("Successfully updated %s", store.TableName().get()));
1297 :
1298 6 : return NS_OK;
1299 : }
1300 :
1301 : nsresult
1302 0 : Classifier::UpdateTableV4(nsTArray<TableUpdate*>* aUpdates,
1303 : const nsACString& aTable)
1304 : {
1305 0 : MOZ_ASSERT(!NS_IsMainThread(),
1306 : "UpdateTableV4 must be called on the classifier worker thread.");
1307 0 : if (nsUrlClassifierDBService::ShutdownHasStarted()) {
1308 0 : return NS_ERROR_UC_UPDATE_SHUTDOWNING;
1309 : }
1310 :
1311 0 : LOG(("Classifier::UpdateTableV4(%s)", PromiseFlatCString(aTable).get()));
1312 :
1313 0 : if (!CheckValidUpdate(aUpdates, aTable)) {
1314 0 : return NS_OK;
1315 : }
1316 :
1317 : LookupCacheV4* lookupCache =
1318 0 : LookupCache::Cast<LookupCacheV4>(GetLookupCacheForUpdate(aTable));
1319 0 : if (!lookupCache) {
1320 0 : return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
1321 : }
1322 :
1323 0 : nsresult rv = NS_OK;
1324 :
1325 : // If there are multiple updates for the same table, prefixes1 & prefixes2
1326 : // will act as input and output in turn to reduce memory copy overhead.
1327 0 : PrefixStringMap prefixes1, prefixes2;
1328 0 : PrefixStringMap* input = &prefixes1;
1329 0 : PrefixStringMap* output = &prefixes2;
1330 :
1331 0 : TableUpdateV4* lastAppliedUpdate = nullptr;
1332 0 : for (uint32_t i = 0; i < aUpdates->Length(); i++) {
1333 0 : TableUpdate *update = aUpdates->ElementAt(i);
1334 0 : if (!update || !update->TableName().Equals(aTable)) {
1335 0 : continue;
1336 : }
1337 :
1338 0 : auto updateV4 = TableUpdate::Cast<TableUpdateV4>(update);
1339 0 : NS_ENSURE_TRUE(updateV4, NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND);
1340 :
1341 0 : if (updateV4->IsFullUpdate()) {
1342 0 : input->Clear();
1343 0 : output->Clear();
1344 0 : rv = lookupCache->ApplyUpdate(updateV4, *input, *output);
1345 0 : if (NS_FAILED(rv)) {
1346 0 : return rv;
1347 : }
1348 : } else {
1349 : // If both prefix sets are empty, this means we are doing a partial update
1350 : // without a prior full/partial update in the loop. In this case we should
1351 : // get prefixes from the lookup cache first.
1352 0 : if (prefixes1.IsEmpty() && prefixes2.IsEmpty()) {
1353 0 : lookupCache->GetPrefixes(prefixes1);
1354 : } else {
1355 0 : MOZ_ASSERT(prefixes1.IsEmpty() ^ prefixes2.IsEmpty());
1356 :
1357 : // When there are multiple partial updates, input should always point
1358 : // to the non-empty prefix set(filled by previous full/partial update).
1359 : // output should always point to the empty prefix set.
1360 0 : input = prefixes1.IsEmpty() ? &prefixes2 : &prefixes1;
1361 0 : output = prefixes1.IsEmpty() ? &prefixes1 : &prefixes2;
1362 : }
1363 :
1364 0 : rv = lookupCache->ApplyUpdate(updateV4, *input, *output);
1365 0 : if (NS_FAILED(rv)) {
1366 0 : return rv;
1367 : }
1368 :
1369 0 : input->Clear();
1370 : }
1371 :
1372 : // Keep track of the last applied update.
1373 0 : lastAppliedUpdate = updateV4;
1374 :
1375 0 : aUpdates->ElementAt(i) = nullptr;
1376 : }
1377 :
1378 0 : rv = lookupCache->Build(*output);
1379 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_BUILD_PREFIX_FAILURE);
1380 :
1381 0 : rv = lookupCache->WriteFile();
1382 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
1383 :
1384 0 : if (lastAppliedUpdate) {
1385 0 : LOG(("Write meta data of the last applied update."));
1386 0 : rv = lookupCache->WriteMetadata(lastAppliedUpdate);
1387 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
1388 : }
1389 :
1390 0 : LOG(("Successfully updated %s\n", PromiseFlatCString(aTable).get()));
1391 :
1392 0 : return NS_OK;
1393 : }
1394 :
1395 : nsresult
1396 0 : Classifier::UpdateCache(TableUpdate* aUpdate)
1397 : {
1398 0 : if (!aUpdate) {
1399 0 : return NS_OK;
1400 : }
1401 :
1402 0 : nsAutoCString table(aUpdate->TableName());
1403 0 : LOG(("Classifier::UpdateCache(%s)", table.get()));
1404 :
1405 0 : LookupCache *lookupCache = GetLookupCache(table);
1406 0 : if (!lookupCache) {
1407 0 : return NS_ERROR_FAILURE;
1408 : }
1409 :
1410 0 : auto lookupV2 = LookupCache::Cast<LookupCacheV2>(lookupCache);
1411 0 : if (lookupV2) {
1412 0 : auto updateV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate);
1413 0 : lookupV2->AddGethashResultToCache(updateV2->AddCompletes(),
1414 0 : updateV2->MissPrefixes());
1415 : } else {
1416 0 : auto lookupV4 = LookupCache::Cast<LookupCacheV4>(lookupCache);
1417 0 : if (!lookupV4) {
1418 0 : return NS_ERROR_FAILURE;
1419 : }
1420 :
1421 0 : auto updateV4 = TableUpdate::Cast<TableUpdateV4>(aUpdate);
1422 0 : lookupV4->AddFullHashResponseToCache(updateV4->FullHashResponse());
1423 : }
1424 :
1425 : #if defined(DEBUG)
1426 0 : lookupCache->DumpCache();
1427 : #endif
1428 :
1429 0 : return NS_OK;
1430 : }
1431 :
1432 : LookupCache *
1433 20 : Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate)
1434 : {
1435 20 : if (aForUpdate) {
1436 6 : MOZ_ASSERT(NS_GetCurrentThread() == mUpdateThread,
1437 : "GetLookupCache(aForUpdate==true) can only be called on update thread.");
1438 : }
1439 :
1440 : nsTArray<LookupCache*>& lookupCaches = aForUpdate ? mNewLookupCaches
1441 20 : : mLookupCaches;
1442 : auto& rootStoreDirectory = aForUpdate ? mUpdatingDirectory
1443 20 : : mRootStoreDirectory;
1444 :
1445 74 : for (auto c: lookupCaches) {
1446 61 : if (c->TableName().Equals(aTable)) {
1447 7 : return c;
1448 : }
1449 : }
1450 :
1451 : // TODO : Bug 1302600, It would be better if we have a more general non-main
1452 : // thread method to convert table name to protocol version. Currently
1453 : // we can only know this by checking if the table name ends with '-proto'.
1454 26 : UniquePtr<LookupCache> cache;
1455 26 : nsCString provider = GetProvider(aTable);
1456 13 : if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
1457 0 : cache = MakeUnique<LookupCacheV4>(aTable, provider, rootStoreDirectory);
1458 : } else {
1459 13 : cache = MakeUnique<LookupCacheV2>(aTable, provider, rootStoreDirectory);
1460 : }
1461 :
1462 13 : nsresult rv = cache->Init();
1463 13 : if (NS_FAILED(rv)) {
1464 0 : return nullptr;
1465 : }
1466 13 : rv = cache->Open();
1467 13 : if (NS_SUCCEEDED(rv)) {
1468 13 : lookupCaches.AppendElement(cache.get());
1469 13 : return cache.release();
1470 : }
1471 :
1472 : // At this point we failed to open LookupCache.
1473 : //
1474 : // GetLookupCache for update and for other usage will run on update thread
1475 : // and worker thread respectively (Bug 1339760). Removing stuff only in
1476 : // their own realms potentially increases the concurrency.
1477 :
1478 0 : if (aForUpdate) {
1479 : // Remove intermediaries no matter if it's due to file corruption or not.
1480 0 : RemoveUpdateIntermediaries();
1481 0 : return nullptr;
1482 : }
1483 :
1484 : // Non-update case.
1485 0 : if (rv == NS_ERROR_FILE_CORRUPTED) {
1486 0 : Reset(); // Not including the update intermediaries.
1487 : }
1488 0 : return nullptr;
1489 : }
1490 :
1491 : nsresult
1492 0 : Classifier::ReadNoiseEntries(const Prefix& aPrefix,
1493 : const nsACString& aTableName,
1494 : uint32_t aCount,
1495 : PrefixArray* aNoiseEntries)
1496 : {
1497 0 : FallibleTArray<uint32_t> prefixes;
1498 : nsresult rv;
1499 :
1500 0 : LookupCache *cache = GetLookupCache(aTableName);
1501 0 : if (!cache) {
1502 0 : return NS_ERROR_FAILURE;
1503 : }
1504 :
1505 0 : LookupCacheV2* cacheV2 = LookupCache::Cast<LookupCacheV2>(cache);
1506 0 : if (cacheV2) {
1507 0 : rv = cacheV2->GetPrefixes(prefixes);
1508 : } else {
1509 0 : rv = LookupCache::Cast<LookupCacheV4>(cache)->GetFixedLengthPrefixes(prefixes);
1510 : }
1511 :
1512 0 : NS_ENSURE_SUCCESS(rv, rv);
1513 :
1514 0 : if (prefixes.Length() == 0) {
1515 0 : NS_WARNING("Could not find prefix in PrefixSet during noise lookup");
1516 0 : return NS_ERROR_FAILURE;
1517 : }
1518 :
1519 : // We do not want to simply pick random prefixes, because this would allow
1520 : // averaging out the noise by analysing the traffic from Firefox users.
1521 : // Instead, we ensure the 'noise' is the same for the same prefix by seeding
1522 : // the random number generator with the prefix. We prefer not to use rand()
1523 : // which isn't thread safe, and the reseeding of which could trip up other
1524 : // parts othe code that expect actual random numbers.
1525 : // Here we use a simple LCG (Linear Congruential Generator) to generate
1526 : // random numbers. We seed the LCG with the prefix we are generating noise
1527 : // for.
1528 : // http://en.wikipedia.org/wiki/Linear_congruential_generator
1529 :
1530 0 : uint32_t m = prefixes.Length();
1531 0 : uint32_t a = aCount % m;
1532 0 : uint32_t idx = aPrefix.ToUint32() % m;
1533 :
1534 0 : for (size_t i = 0; i < aCount; i++) {
1535 0 : idx = (a * idx + a) % m;
1536 :
1537 : Prefix newPrefix;
1538 0 : uint32_t hash = prefixes[idx];
1539 : // In the case V4 little endian, we did swapping endian when converting from char* to
1540 : // int, should revert endian to make sure we will send hex string correctly
1541 : // See https://bugzilla.mozilla.org/show_bug.cgi?id=1283007#c23
1542 0 : if (!cacheV2 && !bool(MOZ_BIG_ENDIAN)) {
1543 0 : hash = NativeEndian::swapFromBigEndian(prefixes[idx]);
1544 : }
1545 :
1546 0 : newPrefix.FromUint32(hash);
1547 0 : if (newPrefix != aPrefix) {
1548 0 : aNoiseEntries->AppendElement(newPrefix);
1549 : }
1550 : }
1551 :
1552 0 : return NS_OK;
1553 : }
1554 :
1555 : nsresult
1556 2 : Classifier::LoadMetadata(nsIFile* aDirectory, nsACString& aResult)
1557 : {
1558 4 : nsCOMPtr<nsISimpleEnumerator> entries;
1559 2 : nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
1560 2 : NS_ENSURE_SUCCESS(rv, rv);
1561 2 : NS_ENSURE_ARG_POINTER(entries);
1562 :
1563 : bool hasMore;
1564 50 : while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
1565 24 : nsCOMPtr<nsISupports> supports;
1566 24 : rv = entries->GetNext(getter_AddRefs(supports));
1567 24 : NS_ENSURE_SUCCESS(rv, rv);
1568 :
1569 24 : nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
1570 :
1571 : // If |file| is a directory, recurse to find its entries as well.
1572 : bool isDirectory;
1573 24 : if (NS_FAILED(file->IsDirectory(&isDirectory))) {
1574 0 : continue;
1575 : }
1576 24 : if (isDirectory) {
1577 0 : LoadMetadata(file, aResult);
1578 0 : continue;
1579 : }
1580 :
1581 : // Truncate file extension to get the table name.
1582 24 : nsCString tableName;
1583 24 : rv = file->GetNativeLeafName(tableName);
1584 24 : NS_ENSURE_SUCCESS(rv, rv);
1585 :
1586 24 : int32_t dot = tableName.RFind(METADATA_SUFFIX, 0);
1587 24 : if (dot == -1) {
1588 24 : continue;
1589 : }
1590 0 : tableName.Cut(dot, METADATA_SUFFIX.Length());
1591 :
1592 : LookupCacheV4* lookupCache =
1593 0 : LookupCache::Cast<LookupCacheV4>(GetLookupCache(tableName));
1594 0 : if (!lookupCache) {
1595 0 : continue;
1596 : }
1597 :
1598 0 : nsCString state;
1599 0 : nsCString checksum;
1600 0 : rv = lookupCache->LoadMetadata(state, checksum);
1601 0 : if (NS_FAILED(rv)) {
1602 0 : LOG(("Failed to get metadata for table %s", tableName.get()));
1603 0 : continue;
1604 : }
1605 :
1606 : // The state might include '\n' so that we have to encode.
1607 0 : nsAutoCString stateBase64;
1608 0 : rv = Base64Encode(state, stateBase64);
1609 0 : NS_ENSURE_SUCCESS(rv, rv);
1610 :
1611 0 : nsAutoCString checksumBase64;
1612 0 : rv = Base64Encode(checksum, checksumBase64);
1613 0 : NS_ENSURE_SUCCESS(rv, rv);
1614 :
1615 0 : LOG(("Appending state '%s' and checksum '%s' for table %s",
1616 : stateBase64.get(), checksumBase64.get(), tableName.get()));
1617 :
1618 0 : aResult.AppendPrintf("%s;%s:%s\n", tableName.get(),
1619 : stateBase64.get(),
1620 0 : checksumBase64.get());
1621 : }
1622 :
1623 2 : return rv;
1624 : }
1625 :
1626 : } // namespace safebrowsing
1627 : } // namespace mozilla
|
