Line data Source code
1 : //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this
4 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #include "nsCRT.h"
7 : #include "nsIHttpChannel.h"
8 : #include "nsIObserverService.h"
9 : #include "nsIStringStream.h"
10 : #include "nsIUploadChannel.h"
11 : #include "nsIURI.h"
12 : #include "nsIUrlClassifierDBService.h"
13 : #include "nsUrlClassifierUtils.h"
14 : #include "nsNetUtil.h"
15 : #include "nsStreamUtils.h"
16 : #include "nsStringStream.h"
17 : #include "nsToolkitCompsCID.h"
18 : #include "nsUrlClassifierStreamUpdater.h"
19 : #include "mozilla/BasePrincipal.h"
20 : #include "mozilla/ErrorNames.h"
21 : #include "mozilla/Logging.h"
22 : #include "nsIInterfaceRequestor.h"
23 : #include "mozilla/LoadContext.h"
24 : #include "mozilla/Telemetry.h"
25 : #include "nsContentUtils.h"
26 : #include "nsIURLFormatter.h"
27 : #include "Classifier.h"
28 :
29 : using namespace mozilla::safebrowsing;
30 :
31 : #define DEFAULT_RESPONSE_TIMEOUT_MS 15 * 1000
32 : #define DEFAULT_TIMEOUT_MS 60 * 1000
33 : static_assert(DEFAULT_TIMEOUT_MS > DEFAULT_RESPONSE_TIMEOUT_MS,
34 : "General timeout must be greater than reponse timeout");
35 :
36 : static const char* gQuitApplicationMessage = "quit-application";
37 :
38 : static uint32_t sResponseTimeoutMs = DEFAULT_RESPONSE_TIMEOUT_MS;
39 : static uint32_t sTimeoutMs = DEFAULT_TIMEOUT_MS;
40 :
41 : // Limit the list file size to 32mb
42 : const uint32_t MAX_FILE_SIZE = (32 * 1024 * 1024);
43 :
44 : // Retry delay when we failed to DownloadUpdate() if due to
45 : // DBService busy.
46 : const uint32_t FETCH_NEXT_REQUEST_RETRY_DELAY_MS = 1000;
47 :
48 : #undef LOG
49 :
50 : // MOZ_LOG=UrlClassifierStreamUpdater:5
51 : static mozilla::LazyLogModule gUrlClassifierStreamUpdaterLog("UrlClassifierStreamUpdater");
52 : #define LOG(args) TrimAndLog args
53 :
54 : // Calls nsIURLFormatter::TrimSensitiveURLs to remove sensitive
55 : // info from the logging message.
56 14 : static MOZ_FORMAT_PRINTF(1, 2) void TrimAndLog(const char* aFmt, ...)
57 : {
58 28 : nsString raw;
59 :
60 : va_list ap;
61 14 : va_start(ap, aFmt);
62 14 : raw.AppendPrintf(aFmt, ap);
63 14 : va_end(ap);
64 :
65 : nsCOMPtr<nsIURLFormatter> urlFormatter =
66 28 : do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
67 :
68 28 : nsString trimmed;
69 14 : nsresult rv = urlFormatter->TrimSensitiveURLs(raw, trimmed);
70 14 : if (NS_FAILED(rv)) {
71 0 : trimmed = EmptyString();
72 : }
73 :
74 : // Use %s so we aren't exposing random strings to printf interpolation.
75 14 : MOZ_LOG(gUrlClassifierStreamUpdaterLog,
76 : mozilla::LogLevel::Debug,
77 : ("%s", NS_ConvertUTF16toUTF8(trimmed).get()));
78 14 : }
79 :
80 : // This class does absolutely nothing, except pass requests onto the DBService.
81 :
82 : ///////////////////////////////////////////////////////////////////////////////
83 : // nsIUrlClassiferStreamUpdater implementation
84 : // Handles creating/running the stream listener
85 :
86 1 : nsUrlClassifierStreamUpdater::nsUrlClassifierStreamUpdater()
87 : : mIsUpdating(false), mInitialized(false), mDownloadError(false),
88 1 : mBeganStream(false), mChannel(nullptr), mTelemetryClockStart(0)
89 : {
90 1 : LOG(("nsUrlClassifierStreamUpdater init [this=%p]", this));
91 1 : }
92 :
93 103 : NS_IMPL_ISUPPORTS(nsUrlClassifierStreamUpdater,
94 : nsIUrlClassifierStreamUpdater,
95 : nsIUrlClassifierUpdateObserver,
96 : nsIRequestObserver,
97 : nsIStreamListener,
98 : nsIObserver,
99 : nsIInterfaceRequestor,
100 : nsITimerCallback)
101 :
102 : /**
103 : * Clear out the update.
104 : */
105 : void
106 1 : nsUrlClassifierStreamUpdater::DownloadDone()
107 : {
108 1 : LOG(("nsUrlClassifierStreamUpdater::DownloadDone [this=%p]", this));
109 1 : mIsUpdating = false;
110 :
111 1 : mPendingUpdates.Clear();
112 1 : mDownloadError = false;
113 1 : mSuccessCallback = nullptr;
114 1 : mUpdateErrorCallback = nullptr;
115 1 : mDownloadErrorCallback = nullptr;
116 1 : }
117 :
118 : ///////////////////////////////////////////////////////////////////////////////
119 : // nsIUrlClassifierStreamUpdater implementation
120 :
121 : nsresult
122 1 : nsUrlClassifierStreamUpdater::FetchUpdate(nsIURI *aUpdateUrl,
123 : const nsACString & aRequestPayload,
124 : bool aIsPostRequest,
125 : const nsACString & aStreamTable)
126 : {
127 :
128 : #ifdef DEBUG
129 2 : LOG(("Fetching update %s from %s",
130 1 : aRequestPayload.Data(), aUpdateUrl->GetSpecOrDefault().get()));
131 : #endif
132 :
133 : nsresult rv;
134 : uint32_t loadFlags = nsIChannel::INHIBIT_CACHING |
135 1 : nsIChannel::LOAD_BYPASS_CACHE;
136 1 : rv = NS_NewChannel(getter_AddRefs(mChannel),
137 : aUpdateUrl,
138 : nsContentUtils::GetSystemPrincipal(),
139 : nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
140 : nsIContentPolicy::TYPE_OTHER,
141 : nullptr, // aLoadGroup
142 : this, // aInterfaceRequestor
143 : loadFlags);
144 :
145 1 : NS_ENSURE_SUCCESS(rv, rv);
146 :
147 2 : nsCOMPtr<nsILoadInfo> loadInfo = mChannel->GetLoadInfo();
148 2 : mozilla::OriginAttributes attrs;
149 1 : attrs.mFirstPartyDomain.AssignLiteral(NECKO_SAFEBROWSING_FIRST_PARTY_DOMAIN);
150 1 : if (loadInfo) {
151 1 : loadInfo->SetOriginAttributes(attrs);
152 : }
153 :
154 1 : mBeganStream = false;
155 :
156 1 : if (!aIsPostRequest) {
157 : // We use POST method to send our request in v2. In v4, the request
158 : // needs to be embedded to the URL and use GET method to send.
159 : // However, from the Chromium source code, a extended HTTP header has
160 : // to be sent along with the request to make the request succeed.
161 : // The following description is from Chromium source code:
162 : //
163 : // "The following header informs the envelope server (which sits in
164 : // front of Google's stubby server) that the received GET request should be
165 : // interpreted as a POST."
166 : //
167 0 : nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv);
168 0 : NS_ENSURE_SUCCESS(rv, rv);
169 :
170 0 : rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("X-HTTP-Method-Override"),
171 0 : NS_LITERAL_CSTRING("POST"),
172 0 : false);
173 0 : NS_ENSURE_SUCCESS(rv, rv);
174 1 : } else if (!aRequestPayload.IsEmpty()) {
175 1 : rv = AddRequestBody(aRequestPayload);
176 1 : NS_ENSURE_SUCCESS(rv, rv);
177 : }
178 :
179 : // Set the appropriate content type for file/data URIs, for unit testing
180 : // purposes.
181 : // This is only used for testing and should be deleted.
182 : bool match;
183 2 : if ((NS_SUCCEEDED(aUpdateUrl->SchemeIs("file", &match)) && match) ||
184 2 : (NS_SUCCEEDED(aUpdateUrl->SchemeIs("data", &match)) && match)) {
185 0 : mChannel->SetContentType(NS_LITERAL_CSTRING("application/vnd.google.safebrowsing-update"));
186 : } else {
187 : // We assume everything else is an HTTP request.
188 :
189 : // Disable keepalive.
190 2 : nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv);
191 1 : NS_ENSURE_SUCCESS(rv, rv);
192 1 : rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("Connection"), NS_LITERAL_CSTRING("close"), false);
193 1 : NS_ENSURE_SUCCESS(rv, rv);
194 : }
195 :
196 : // Make the request.
197 1 : rv = mChannel->AsyncOpen2(this);
198 1 : NS_ENSURE_SUCCESS(rv, rv);
199 :
200 1 : mTelemetryClockStart = PR_IntervalNow();
201 1 : mStreamTable = aStreamTable;
202 :
203 : static bool preferencesInitialized = false;
204 :
205 1 : if (!preferencesInitialized) {
206 : mozilla::Preferences::AddUintVarCache(&sTimeoutMs,
207 : "urlclassifier.update.timeout_ms",
208 1 : DEFAULT_TIMEOUT_MS);
209 : mozilla::Preferences::AddUintVarCache(&sResponseTimeoutMs,
210 : "urlclassifier.update.response_timeout_ms",
211 1 : DEFAULT_RESPONSE_TIMEOUT_MS);
212 1 : preferencesInitialized = true;
213 : }
214 :
215 1 : if (sResponseTimeoutMs > sTimeoutMs) {
216 : NS_WARNING("Safe Browsing response timeout is greater than the general "
217 0 : "timeout. Disabling these update timeouts.");
218 0 : return NS_OK;
219 : }
220 1 : mResponseTimeoutTimer = do_CreateInstance("@mozilla.org/timer;1", &rv);
221 1 : if (NS_SUCCEEDED(rv)) {
222 2 : rv = mResponseTimeoutTimer->InitWithCallback(this,
223 : sResponseTimeoutMs,
224 1 : nsITimer::TYPE_ONE_SHOT);
225 : }
226 :
227 1 : mTimeoutTimer = do_CreateInstance("@mozilla.org/timer;1", &rv);
228 :
229 1 : if (NS_SUCCEEDED(rv)) {
230 1 : if (sTimeoutMs < DEFAULT_TIMEOUT_MS) {
231 : LOG(("Download update timeout %d ms (< %d ms) would be too small",
232 0 : sTimeoutMs, DEFAULT_TIMEOUT_MS));
233 : }
234 2 : rv = mTimeoutTimer->InitWithCallback(this,
235 : sTimeoutMs,
236 1 : nsITimer::TYPE_ONE_SHOT);
237 : }
238 :
239 1 : return NS_OK;
240 : }
241 :
242 : nsresult
243 1 : nsUrlClassifierStreamUpdater::FetchUpdate(const nsACString & aUpdateUrl,
244 : const nsACString & aRequestPayload,
245 : bool aIsPostRequest,
246 : const nsACString & aStreamTable)
247 : {
248 1 : LOG(("(pre) Fetching update from %s\n", PromiseFlatCString(aUpdateUrl).get()));
249 :
250 2 : nsCString updateUrl(aUpdateUrl);
251 1 : if (!aIsPostRequest) {
252 0 : updateUrl.AppendPrintf("&$req=%s", nsCString(aRequestPayload).get());
253 : }
254 :
255 2 : nsCOMPtr<nsIURI> uri;
256 1 : nsresult rv = NS_NewURI(getter_AddRefs(uri), updateUrl);
257 1 : NS_ENSURE_SUCCESS(rv, rv);
258 :
259 2 : nsAutoCString urlSpec;
260 1 : uri->GetAsciiSpec(urlSpec);
261 :
262 1 : LOG(("(post) Fetching update from %s\n", urlSpec.get()));
263 :
264 1 : return FetchUpdate(uri, aRequestPayload, aIsPostRequest, aStreamTable);
265 : }
266 :
267 : NS_IMETHODIMP
268 1 : nsUrlClassifierStreamUpdater::DownloadUpdates(
269 : const nsACString &aRequestTables,
270 : const nsACString &aRequestPayload,
271 : bool aIsPostRequest,
272 : const nsACString &aUpdateUrl,
273 : nsIUrlClassifierCallback *aSuccessCallback,
274 : nsIUrlClassifierCallback *aUpdateErrorCallback,
275 : nsIUrlClassifierCallback *aDownloadErrorCallback,
276 : bool *_retval)
277 : {
278 1 : NS_ENSURE_ARG(aSuccessCallback);
279 1 : NS_ENSURE_ARG(aUpdateErrorCallback);
280 1 : NS_ENSURE_ARG(aDownloadErrorCallback);
281 :
282 1 : if (mIsUpdating) {
283 0 : LOG(("Already updating, queueing update %s from %s", aRequestPayload.Data(),
284 0 : aUpdateUrl.Data()));
285 0 : *_retval = false;
286 0 : PendingRequest *request = mPendingRequests.AppendElement();
287 0 : request->mTables = aRequestTables;
288 0 : request->mRequestPayload = aRequestPayload;
289 0 : request->mIsPostRequest = aIsPostRequest;
290 0 : request->mUrl = aUpdateUrl;
291 0 : request->mSuccessCallback = aSuccessCallback;
292 0 : request->mUpdateErrorCallback = aUpdateErrorCallback;
293 0 : request->mDownloadErrorCallback = aDownloadErrorCallback;
294 0 : return NS_OK;
295 : }
296 :
297 1 : if (aUpdateUrl.IsEmpty()) {
298 0 : NS_ERROR("updateUrl not set");
299 0 : return NS_ERROR_NOT_INITIALIZED;
300 : }
301 :
302 : nsresult rv;
303 :
304 1 : if (!mInitialized) {
305 : // Add an observer for shutdown so we can cancel any pending list
306 : // downloads. quit-application is the same event that the download
307 : // manager listens for and uses to cancel pending downloads.
308 : nsCOMPtr<nsIObserverService> observerService =
309 2 : mozilla::services::GetObserverService();
310 1 : if (!observerService)
311 0 : return NS_ERROR_FAILURE;
312 :
313 1 : observerService->AddObserver(this, gQuitApplicationMessage, false);
314 :
315 1 : mDBService = do_GetService(NS_URLCLASSIFIERDBSERVICE_CONTRACTID, &rv);
316 1 : NS_ENSURE_SUCCESS(rv, rv);
317 :
318 1 : mInitialized = true;
319 : }
320 :
321 1 : rv = mDBService->BeginUpdate(this, aRequestTables);
322 1 : if (rv == NS_ERROR_NOT_AVAILABLE) {
323 0 : LOG(("Service busy, already updating, queuing update %s from %s",
324 0 : aRequestPayload.Data(), aUpdateUrl.Data()));
325 0 : *_retval = false;
326 0 : PendingRequest *request = mPendingRequests.AppendElement();
327 0 : request->mTables = aRequestTables;
328 0 : request->mRequestPayload = aRequestPayload;
329 0 : request->mIsPostRequest = aIsPostRequest;
330 0 : request->mUrl = aUpdateUrl;
331 0 : request->mSuccessCallback = aSuccessCallback;
332 0 : request->mUpdateErrorCallback = aUpdateErrorCallback;
333 0 : request->mDownloadErrorCallback = aDownloadErrorCallback;
334 :
335 : // We cannot guarantee that we will be notified when DBService is done
336 : // processing the current update, so we fire a retry timer on our own.
337 : nsresult rv;
338 0 : mFetchNextRequestTimer = do_CreateInstance("@mozilla.org/timer;1", &rv);
339 0 : if (NS_SUCCEEDED(rv)) {
340 0 : rv = mFetchNextRequestTimer->InitWithCallback(this,
341 : FETCH_NEXT_REQUEST_RETRY_DELAY_MS,
342 0 : nsITimer::TYPE_ONE_SHOT);
343 : }
344 :
345 0 : return NS_OK;
346 : }
347 :
348 1 : if (NS_FAILED(rv)) {
349 0 : return rv;
350 : }
351 :
352 : nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
353 2 : do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
354 :
355 2 : nsTArray<nsCString> tables;
356 1 : mozilla::safebrowsing::Classifier::SplitTables(aRequestTables, tables);
357 2 : urlUtil->GetTelemetryProvider(tables.SafeElementAt(0, EmptyCString()),
358 2 : mTelemetryProvider);
359 :
360 1 : mSuccessCallback = aSuccessCallback;
361 1 : mUpdateErrorCallback = aUpdateErrorCallback;
362 1 : mDownloadErrorCallback = aDownloadErrorCallback;
363 :
364 1 : mIsUpdating = true;
365 1 : *_retval = true;
366 :
367 1 : LOG(("FetchUpdate: %s", aUpdateUrl.Data()));
368 :
369 1 : return FetchUpdate(aUpdateUrl, aRequestPayload, aIsPostRequest, EmptyCString());
370 : }
371 :
372 : ///////////////////////////////////////////////////////////////////////////////
373 : // nsIUrlClassifierUpdateObserver implementation
374 :
375 : NS_IMETHODIMP
376 0 : nsUrlClassifierStreamUpdater::UpdateUrlRequested(const nsACString &aUrl,
377 : const nsACString &aTable)
378 : {
379 0 : LOG(("Queuing requested update from %s\n", PromiseFlatCString(aUrl).get()));
380 :
381 0 : PendingUpdate *update = mPendingUpdates.AppendElement();
382 0 : if (!update)
383 0 : return NS_ERROR_OUT_OF_MEMORY;
384 :
385 : // Allow data: and file: urls for unit testing purposes, otherwise assume http
386 0 : if (StringBeginsWith(aUrl, NS_LITERAL_CSTRING("data:")) ||
387 0 : StringBeginsWith(aUrl, NS_LITERAL_CSTRING("file:"))) {
388 0 : update->mUrl = aUrl;
389 : } else {
390 : // For unittesting update urls to localhost should use http, not https
391 : // (otherwise the connection will fail silently, since there will be no
392 : // cert available).
393 0 : if (!StringBeginsWith(aUrl, NS_LITERAL_CSTRING("localhost"))) {
394 0 : update->mUrl = NS_LITERAL_CSTRING("https://") + aUrl;
395 : } else {
396 0 : update->mUrl = NS_LITERAL_CSTRING("http://") + aUrl;
397 : }
398 : }
399 0 : update->mTable = aTable;
400 :
401 0 : return NS_OK;
402 : }
403 :
404 : nsresult
405 0 : nsUrlClassifierStreamUpdater::FetchNext()
406 : {
407 0 : if (mPendingUpdates.Length() == 0) {
408 0 : return NS_OK;
409 : }
410 :
411 0 : PendingUpdate &update = mPendingUpdates[0];
412 0 : LOG(("Fetching update url: %s\n", update.mUrl.get()));
413 0 : nsresult rv = FetchUpdate(update.mUrl,
414 0 : EmptyCString(),
415 : true, // This method is for v2 and v2 is always a POST.
416 0 : update.mTable);
417 0 : if (NS_FAILED(rv)) {
418 0 : LOG(("Error fetching update url: %s\n", update.mUrl.get()));
419 : // We can commit the urls that we've applied so far. This is
420 : // probably a transient server problem, so trigger backoff.
421 0 : mDownloadError = true;
422 0 : mDBService->FinishUpdate();
423 0 : return rv;
424 : }
425 :
426 0 : mPendingUpdates.RemoveElementAt(0);
427 :
428 0 : return NS_OK;
429 : }
430 :
431 : nsresult
432 1 : nsUrlClassifierStreamUpdater::FetchNextRequest()
433 : {
434 1 : if (mPendingRequests.Length() == 0) {
435 1 : LOG(("No more requests, returning"));
436 1 : return NS_OK;
437 : }
438 :
439 0 : PendingRequest request = mPendingRequests[0];
440 0 : mPendingRequests.RemoveElementAt(0);
441 0 : LOG(("Stream updater: fetching next request: %s, %s",
442 0 : request.mTables.get(), request.mUrl.get()));
443 : bool dummy;
444 0 : DownloadUpdates(
445 : request.mTables,
446 : request.mRequestPayload,
447 0 : request.mIsPostRequest,
448 : request.mUrl,
449 : request.mSuccessCallback,
450 : request.mUpdateErrorCallback,
451 : request.mDownloadErrorCallback,
452 0 : &dummy);
453 0 : return NS_OK;
454 : }
455 :
456 : NS_IMETHODIMP
457 0 : nsUrlClassifierStreamUpdater::StreamFinished(nsresult status,
458 : uint32_t requestedDelay)
459 : {
460 : // We are a service and may not be reset with Init between calls, so reset
461 : // mBeganStream manually.
462 0 : mBeganStream = false;
463 : LOG(("nsUrlClassifierStreamUpdater::StreamFinished [%" PRIx32 ", %d]",
464 0 : static_cast<uint32_t>(status), requestedDelay));
465 0 : if (NS_FAILED(status) || mPendingUpdates.Length() == 0) {
466 : // We're done.
467 0 : LOG(("nsUrlClassifierStreamUpdater::Done [this=%p]", this));
468 0 : mDBService->FinishUpdate();
469 0 : return NS_OK;
470 : }
471 :
472 : // This timer is for fetching indirect updates ("forwards") from any "u:" lines
473 : // that we encountered while processing the server response. It is NOT for
474 : // scheduling the next time we pull the list from the server. That's a different
475 : // timer in listmanager.js (see bug 1110891).
476 : nsresult rv;
477 0 : mFetchIndirectUpdatesTimer = do_CreateInstance("@mozilla.org/timer;1", &rv);
478 0 : if (NS_SUCCEEDED(rv)) {
479 0 : rv = mFetchIndirectUpdatesTimer->InitWithCallback(this, requestedDelay,
480 0 : nsITimer::TYPE_ONE_SHOT);
481 : }
482 :
483 0 : if (NS_FAILED(rv)) {
484 0 : NS_WARNING("Unable to initialize timer, fetching next safebrowsing item immediately");
485 0 : return FetchNext();
486 : }
487 :
488 0 : return NS_OK;
489 : }
490 :
491 : NS_IMETHODIMP
492 1 : nsUrlClassifierStreamUpdater::UpdateSuccess(uint32_t requestedTimeout)
493 : {
494 1 : LOG(("nsUrlClassifierStreamUpdater::UpdateSuccess [this=%p]", this));
495 1 : if (mPendingUpdates.Length() != 0) {
496 0 : NS_WARNING("Didn't fetch all safebrowsing update redirects");
497 : }
498 :
499 : // DownloadDone() clears mSuccessCallback, so we save it off here.
500 2 : nsCOMPtr<nsIUrlClassifierCallback> successCallback = mDownloadError ? nullptr : mSuccessCallback.get();
501 2 : nsCOMPtr<nsIUrlClassifierCallback> downloadErrorCallback = mDownloadError ? mDownloadErrorCallback.get() : nullptr;
502 1 : DownloadDone();
503 :
504 2 : nsAutoCString strTimeout;
505 1 : strTimeout.AppendInt(requestedTimeout);
506 1 : if (successCallback) {
507 : LOG(("nsUrlClassifierStreamUpdater::UpdateSuccess callback [this=%p]",
508 0 : this));
509 0 : successCallback->HandleEvent(strTimeout);
510 1 : } else if (downloadErrorCallback) {
511 1 : downloadErrorCallback->HandleEvent(mDownloadErrorStatusStr);
512 1 : mDownloadErrorStatusStr = EmptyCString();
513 1 : LOG(("Notify download error callback in UpdateSuccess [this=%p]", this));
514 : }
515 : // Now fetch the next request
516 1 : LOG(("stream updater: calling into fetch next request"));
517 1 : FetchNextRequest();
518 :
519 2 : return NS_OK;
520 : }
521 :
522 : NS_IMETHODIMP
523 0 : nsUrlClassifierStreamUpdater::UpdateError(nsresult result)
524 : {
525 0 : LOG(("nsUrlClassifierStreamUpdater::UpdateError [this=%p]", this));
526 :
527 : // DownloadDone() clears mUpdateErrorCallback, so we save it off here.
528 0 : nsCOMPtr<nsIUrlClassifierCallback> errorCallback = mDownloadError ? nullptr : mUpdateErrorCallback.get();
529 0 : nsCOMPtr<nsIUrlClassifierCallback> downloadErrorCallback = mDownloadError ? mDownloadErrorCallback.get() : nullptr;
530 0 : DownloadDone();
531 :
532 0 : nsAutoCString strResult;
533 0 : strResult.AppendInt(static_cast<uint32_t>(result));
534 0 : if (errorCallback) {
535 0 : errorCallback->HandleEvent(strResult);
536 0 : } else if (downloadErrorCallback) {
537 0 : LOG(("Notify download error callback in UpdateError [this=%p]", this));
538 0 : downloadErrorCallback->HandleEvent(mDownloadErrorStatusStr);
539 0 : mDownloadErrorStatusStr = EmptyCString();
540 : }
541 :
542 0 : return NS_OK;
543 : }
544 :
545 : nsresult
546 1 : nsUrlClassifierStreamUpdater::AddRequestBody(const nsACString &aRequestBody)
547 : {
548 : nsresult rv;
549 : nsCOMPtr<nsIStringInputStream> strStream =
550 2 : do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID, &rv);
551 1 : NS_ENSURE_SUCCESS(rv, rv);
552 :
553 2 : rv = strStream->SetData(aRequestBody.BeginReading(),
554 1 : aRequestBody.Length());
555 1 : NS_ENSURE_SUCCESS(rv, rv);
556 :
557 2 : nsCOMPtr<nsIUploadChannel> uploadChannel = do_QueryInterface(mChannel, &rv);
558 1 : NS_ENSURE_SUCCESS(rv, rv);
559 :
560 3 : rv = uploadChannel->SetUploadStream(strStream,
561 2 : NS_LITERAL_CSTRING("text/plain"),
562 3 : -1);
563 1 : NS_ENSURE_SUCCESS(rv, rv);
564 :
565 2 : nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv);
566 1 : NS_ENSURE_SUCCESS(rv, rv);
567 :
568 1 : rv = httpChannel->SetRequestMethod(NS_LITERAL_CSTRING("POST"));
569 1 : NS_ENSURE_SUCCESS(rv, rv);
570 :
571 1 : return NS_OK;
572 : }
573 :
574 : // We might need to expand the bucket here if telemetry shows lots of errors
575 : // are neither connection errors nor DNS errors.
576 0 : static uint8_t NetworkErrorToBucket(nsresult rv)
577 : {
578 0 : switch(rv) {
579 : // Connection errors
580 0 : case NS_ERROR_ALREADY_CONNECTED: return 2;
581 0 : case NS_ERROR_NOT_CONNECTED: return 3;
582 0 : case NS_ERROR_CONNECTION_REFUSED: return 4;
583 0 : case NS_ERROR_NET_TIMEOUT: return 5;
584 0 : case NS_ERROR_OFFLINE: return 6;
585 0 : case NS_ERROR_PORT_ACCESS_NOT_ALLOWED: return 7;
586 0 : case NS_ERROR_NET_RESET: return 8;
587 0 : case NS_ERROR_NET_INTERRUPT: return 9;
588 0 : case NS_ERROR_PROXY_CONNECTION_REFUSED: return 10;
589 0 : case NS_ERROR_NET_PARTIAL_TRANSFER: return 11;
590 0 : case NS_ERROR_NET_INADEQUATE_SECURITY: return 12;
591 : // DNS errors
592 0 : case NS_ERROR_UNKNOWN_HOST: return 13;
593 0 : case NS_ERROR_DNS_LOOKUP_QUEUE_FULL: return 14;
594 0 : case NS_ERROR_UNKNOWN_PROXY_HOST: return 15;
595 : // Others
596 0 : default: return 1;
597 : }
598 : }
599 :
600 : // Map the HTTP response code to a Telemetry bucket
601 1 : static uint32_t HTTPStatusToBucket(uint32_t status)
602 : {
603 : uint32_t statusBucket;
604 1 : switch (status) {
605 : case 100:
606 : case 101:
607 : // Unexpected 1xx return code
608 0 : statusBucket = 0;
609 0 : break;
610 : case 200:
611 : // OK - Data is available in the HTTP response body.
612 0 : statusBucket = 1;
613 0 : break;
614 : case 201:
615 : case 202:
616 : case 203:
617 : case 205:
618 : case 206:
619 : // Unexpected 2xx return code
620 0 : statusBucket = 2;
621 0 : break;
622 : case 204:
623 : // No Content
624 0 : statusBucket = 3;
625 0 : break;
626 : case 300:
627 : case 301:
628 : case 302:
629 : case 303:
630 : case 304:
631 : case 305:
632 : case 307:
633 : case 308:
634 : // Unexpected 3xx return code
635 0 : statusBucket = 4;
636 0 : break;
637 : case 400:
638 : // Bad Request - The HTTP request was not correctly formed.
639 : // The client did not provide all required CGI parameters.
640 0 : statusBucket = 5;
641 0 : break;
642 : case 401:
643 : case 402:
644 : case 405:
645 : case 406:
646 : case 407:
647 : case 409:
648 : case 410:
649 : case 411:
650 : case 412:
651 : case 414:
652 : case 415:
653 : case 416:
654 : case 417:
655 : case 421:
656 : case 426:
657 : case 428:
658 : case 429:
659 : case 431:
660 : case 451:
661 : // Unexpected 4xx return code
662 0 : statusBucket = 6;
663 0 : break;
664 : case 403:
665 : // Forbidden - The client id is invalid.
666 0 : statusBucket = 7;
667 0 : break;
668 : case 404:
669 : // Not Found
670 1 : statusBucket = 8;
671 1 : break;
672 : case 408:
673 : // Request Timeout
674 0 : statusBucket = 9;
675 0 : break;
676 : case 413:
677 : // Request Entity Too Large - Bug 1150334
678 0 : statusBucket = 10;
679 0 : break;
680 : case 500:
681 : case 501:
682 : case 510:
683 : // Unexpected 5xx return code
684 0 : statusBucket = 11;
685 0 : break;
686 : case 502:
687 : case 504:
688 : case 511:
689 : // Local network errors, we'll ignore these.
690 0 : statusBucket = 12;
691 0 : break;
692 : case 503:
693 : // Service Unavailable - The server cannot handle the request.
694 : // Clients MUST follow the backoff behavior specified in the
695 : // Request Frequency section.
696 0 : statusBucket = 13;
697 0 : break;
698 : case 505:
699 : // HTTP Version Not Supported - The server CANNOT handle the requested
700 : // protocol major version.
701 0 : statusBucket = 14;
702 0 : break;
703 : default:
704 0 : statusBucket = 15;
705 : };
706 1 : return statusBucket;
707 : }
708 :
709 : ///////////////////////////////////////////////////////////////////////////////
710 : // nsIStreamListenerObserver implementation
711 :
712 : NS_IMETHODIMP
713 1 : nsUrlClassifierStreamUpdater::OnStartRequest(nsIRequest *request,
714 : nsISupports* context)
715 : {
716 : nsresult rv;
717 1 : bool downloadError = false;
718 2 : nsAutoCString strStatus;
719 1 : nsresult status = NS_OK;
720 :
721 : // Only update if we got http success header
722 2 : nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(request);
723 1 : if (httpChannel) {
724 1 : rv = httpChannel->GetStatus(&status);
725 1 : NS_ENSURE_SUCCESS(rv, rv);
726 :
727 1 : if (MOZ_LOG_TEST(gUrlClassifierStreamUpdaterLog, mozilla::LogLevel::Debug)) {
728 0 : nsAutoCString errorName, spec;
729 0 : mozilla::GetErrorName(status, errorName);
730 0 : nsCOMPtr<nsIURI> uri;
731 0 : rv = httpChannel->GetURI(getter_AddRefs(uri));
732 0 : if (NS_SUCCEEDED(rv) && uri) {
733 0 : uri->GetAsciiSpec(spec);
734 : }
735 0 : LOG(("nsUrlClassifierStreamUpdater::OnStartRequest "
736 : "(status=%s, uri=%s, this=%p)", errorName.get(),
737 0 : spec.get(), this));
738 : }
739 1 : if (mTelemetryClockStart > 0) {
740 1 : uint32_t msecs = PR_IntervalToMilliseconds(PR_IntervalNow() - mTelemetryClockStart);
741 1 : mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_SERVER_RESPONSE_TIME,
742 1 : mTelemetryProvider, msecs);
743 :
744 : }
745 :
746 1 : if (mResponseTimeoutTimer) {
747 1 : mResponseTimeoutTimer->Cancel();
748 1 : mResponseTimeoutTimer = nullptr;
749 : }
750 :
751 1 : uint8_t netErrCode = NS_FAILED(status) ? NetworkErrorToBucket(status) : 0;
752 1 : mozilla::Telemetry::Accumulate(
753 : mozilla::Telemetry::URLCLASSIFIER_UPDATE_REMOTE_NETWORK_ERROR,
754 1 : mTelemetryProvider, netErrCode);
755 :
756 1 : if (NS_FAILED(status)) {
757 : // Assume we're overloading the server and trigger backoff.
758 0 : downloadError = true;
759 : } else {
760 1 : bool succeeded = false;
761 1 : rv = httpChannel->GetRequestSucceeded(&succeeded);
762 1 : NS_ENSURE_SUCCESS(rv, rv);
763 :
764 : uint32_t requestStatus;
765 1 : rv = httpChannel->GetResponseStatus(&requestStatus);
766 1 : NS_ENSURE_SUCCESS(rv, rv);
767 1 : mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_REMOTE_STATUS2,
768 1 : mTelemetryProvider, HTTPStatusToBucket(requestStatus));
769 1 : LOG(("nsUrlClassifierStreamUpdater::OnStartRequest %s (%d)", succeeded ?
770 2 : "succeeded" : "failed", requestStatus));
771 1 : if (!succeeded) {
772 : // 404 or other error, pass error status back
773 1 : strStatus.AppendInt(requestStatus);
774 1 : downloadError = true;
775 : }
776 : }
777 : }
778 :
779 1 : if (downloadError) {
780 1 : LOG(("nsUrlClassifierStreamUpdater::Download error [this=%p]", this));
781 1 : mDownloadError = true;
782 1 : mDownloadErrorStatusStr = strStatus;
783 1 : status = NS_ERROR_ABORT;
784 0 : } else if (NS_SUCCEEDED(status)) {
785 0 : MOZ_ASSERT(mDownloadErrorCallback);
786 0 : mBeganStream = true;
787 0 : LOG(("nsUrlClassifierStreamUpdater::Beginning stream [this=%p]", this));
788 0 : rv = mDBService->BeginStream(mStreamTable);
789 0 : NS_ENSURE_SUCCESS(rv, rv);
790 : }
791 :
792 1 : mStreamTable.Truncate();
793 :
794 1 : return status;
795 : }
796 :
797 : NS_IMETHODIMP
798 0 : nsUrlClassifierStreamUpdater::OnDataAvailable(nsIRequest *request,
799 : nsISupports* context,
800 : nsIInputStream *aIStream,
801 : uint64_t aSourceOffset,
802 : uint32_t aLength)
803 : {
804 0 : if (!mDBService)
805 0 : return NS_ERROR_NOT_INITIALIZED;
806 :
807 0 : LOG(("OnDataAvailable (%d bytes)", aLength));
808 :
809 0 : if (aSourceOffset > MAX_FILE_SIZE) {
810 0 : LOG(("OnDataAvailable::Abort because exceeded the maximum file size(%" PRIu64 ")", aSourceOffset));
811 0 : return NS_ERROR_FILE_TOO_BIG;
812 : }
813 :
814 : nsresult rv;
815 :
816 : // Copy the data into a nsCString
817 0 : nsCString chunk;
818 0 : rv = NS_ConsumeStream(aIStream, aLength, chunk);
819 0 : NS_ENSURE_SUCCESS(rv, rv);
820 :
821 : //LOG(("Chunk (%d): %s\n\n", chunk.Length(), chunk.get()));
822 0 : rv = mDBService->UpdateStream(chunk);
823 0 : NS_ENSURE_SUCCESS(rv, rv);
824 :
825 0 : return NS_OK;
826 : }
827 :
828 : NS_IMETHODIMP
829 1 : nsUrlClassifierStreamUpdater::OnStopRequest(nsIRequest *request, nsISupports* context,
830 : nsresult aStatus)
831 : {
832 1 : if (!mDBService)
833 0 : return NS_ERROR_NOT_INITIALIZED;
834 :
835 1 : LOG(("OnStopRequest (status %" PRIx32 ", beganStream %s, this=%p)",
836 1 : static_cast<uint32_t>(aStatus), mBeganStream ? "true" : "false", this));
837 :
838 : nsresult rv;
839 :
840 1 : if (NS_SUCCEEDED(aStatus)) {
841 : // Success, finish this stream and move on to the next.
842 0 : rv = mDBService->FinishStream();
843 1 : } else if (mBeganStream) {
844 0 : LOG(("OnStopRequest::Canceling update [this=%p]", this));
845 : // We began this stream and couldn't finish it. We have to cancel the
846 : // update, it's not in a consistent state.
847 0 : rv = mDBService->CancelUpdate();
848 : } else {
849 1 : LOG(("OnStopRequest::Finishing update [this=%p]", this));
850 : // The fetch failed, but we didn't start the stream (probably a
851 : // server or connection error). We can commit what we've applied
852 : // so far, and request again later.
853 1 : rv = mDBService->FinishUpdate();
854 : }
855 :
856 1 : if (mResponseTimeoutTimer) {
857 0 : mResponseTimeoutTimer->Cancel();
858 0 : mResponseTimeoutTimer = nullptr;
859 : }
860 :
861 : // mResponseTimeoutTimer may be cleared in OnStartRequest, so we check mTimeoutTimer
862 : // to see whether the update was has timed out
863 1 : if (mTimeoutTimer) {
864 1 : mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_TIMEOUT,
865 : mTelemetryProvider,
866 1 : static_cast<uint8_t>(eNoTimeout));
867 1 : mTimeoutTimer->Cancel();
868 1 : mTimeoutTimer = nullptr;
869 : }
870 :
871 1 : mTelemetryProvider.Truncate();
872 1 : mTelemetryClockStart = 0;
873 1 : mChannel = nullptr;
874 :
875 : // If the fetch failed, return the network status rather than NS_OK, the
876 : // result of finishing a possibly-empty update
877 1 : if (NS_SUCCEEDED(aStatus)) {
878 0 : return rv;
879 : }
880 1 : return aStatus;
881 : }
882 :
883 : ///////////////////////////////////////////////////////////////////////////////
884 : // nsIObserver implementation
885 :
886 : NS_IMETHODIMP
887 0 : nsUrlClassifierStreamUpdater::Observe(nsISupports *aSubject, const char *aTopic,
888 : const char16_t *aData)
889 : {
890 0 : if (nsCRT::strcmp(aTopic, gQuitApplicationMessage) == 0) {
891 0 : if (mIsUpdating && mChannel) {
892 0 : LOG(("Cancel download"));
893 : nsresult rv;
894 0 : rv = mChannel->Cancel(NS_ERROR_ABORT);
895 0 : NS_ENSURE_SUCCESS(rv, rv);
896 0 : mIsUpdating = false;
897 0 : mChannel = nullptr;
898 0 : mTelemetryClockStart = 0;
899 : }
900 0 : if (mFetchIndirectUpdatesTimer) {
901 0 : mFetchIndirectUpdatesTimer->Cancel();
902 0 : mFetchIndirectUpdatesTimer = nullptr;
903 : }
904 0 : if (mFetchNextRequestTimer) {
905 0 : mFetchNextRequestTimer->Cancel();
906 0 : mFetchNextRequestTimer = nullptr;
907 : }
908 0 : if (mResponseTimeoutTimer) {
909 0 : mResponseTimeoutTimer->Cancel();
910 0 : mResponseTimeoutTimer = nullptr;
911 : }
912 0 : if (mTimeoutTimer) {
913 0 : mTimeoutTimer->Cancel();
914 0 : mTimeoutTimer = nullptr;
915 : }
916 :
917 : }
918 0 : return NS_OK;
919 : }
920 :
921 : ///////////////////////////////////////////////////////////////////////////////
922 : // nsIInterfaceRequestor implementation
923 :
924 : NS_IMETHODIMP
925 18 : nsUrlClassifierStreamUpdater::GetInterface(const nsIID & eventSinkIID, void* *_retval)
926 : {
927 18 : return QueryInterface(eventSinkIID, _retval);
928 : }
929 :
930 :
931 : ///////////////////////////////////////////////////////////////////////////////
932 : // nsITimerCallback implementation
933 : NS_IMETHODIMP
934 0 : nsUrlClassifierStreamUpdater::Notify(nsITimer *timer)
935 : {
936 0 : LOG(("nsUrlClassifierStreamUpdater::Notify [%p]", this));
937 :
938 0 : if (timer == mFetchNextRequestTimer) {
939 0 : mFetchNextRequestTimer = nullptr;
940 0 : FetchNextRequest();
941 0 : return NS_OK;
942 : }
943 :
944 0 : if (timer == mFetchIndirectUpdatesTimer) {
945 0 : mFetchIndirectUpdatesTimer = nullptr;
946 : // Start the update process up again.
947 0 : FetchNext();
948 0 : return NS_OK;
949 : }
950 :
951 0 : bool updateFailed = false;
952 0 : if (timer == mResponseTimeoutTimer) {
953 0 : mResponseTimeoutTimer = nullptr;
954 0 : if (mTimeoutTimer) {
955 0 : mTimeoutTimer->Cancel();
956 0 : mTimeoutTimer = nullptr;
957 : }
958 0 : mDownloadError = true; // Trigger backoff
959 0 : updateFailed = true;
960 0 : mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_TIMEOUT,
961 : mTelemetryProvider,
962 0 : static_cast<uint8_t>(eResponseTimeout));
963 : }
964 :
965 0 : if (timer == mTimeoutTimer) {
966 0 : mTimeoutTimer = nullptr;
967 : // No backoff since the connection may just be temporarily slow.
968 0 : updateFailed = true;
969 0 : mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_TIMEOUT,
970 : mTelemetryProvider,
971 0 : static_cast<uint8_t>(eDownloadTimeout));
972 : }
973 :
974 0 : if (updateFailed) {
975 : // Cancelling the channel will trigger OnStopRequest.
976 0 : mozilla::Unused << mChannel->Cancel(NS_ERROR_ABORT);
977 0 : mChannel = nullptr;
978 0 : mTelemetryClockStart = 0;
979 :
980 0 : if (mFetchIndirectUpdatesTimer) {
981 0 : mFetchIndirectUpdatesTimer->Cancel();
982 0 : mFetchIndirectUpdatesTimer = nullptr;
983 : }
984 0 : if (mFetchNextRequestTimer) {
985 0 : mFetchNextRequestTimer->Cancel();
986 0 : mFetchNextRequestTimer = nullptr;
987 : }
988 :
989 0 : return NS_OK;
990 : }
991 :
992 0 : MOZ_ASSERT_UNREACHABLE("A timer is fired from nowhere.");
993 : return NS_OK;
994 : }
995 :
|
