Line data Source code
1 : /* This Source Code Form is subject to the terms of the Mozilla Public
2 : * License, v. 2.0. If a copy of the MPL was not distributed with this
3 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4 :
5 : #ifndef HashStore_h__
6 : #define HashStore_h__
7 :
8 : #include "Entries.h"
9 : #include "ChunkSet.h"
10 :
11 : #include "nsString.h"
12 : #include "nsTArray.h"
13 : #include "nsIFile.h"
14 : #include "nsIFileStreams.h"
15 : #include "nsCOMPtr.h"
16 : #include "nsClassHashtable.h"
17 : #include "safebrowsing.pb.h"
18 : #include <string>
19 :
20 : namespace mozilla {
21 : namespace safebrowsing {
22 :
23 : // The abstract class of TableUpdateV2 and TableUpdateV4. This
24 : // is convenient for passing the TableUpdate* around associated
25 : // with v2 and v4 instance.
26 : class TableUpdate {
27 : public:
28 6 : TableUpdate(const nsACString& aTable)
29 6 : : mTable(aTable)
30 : {
31 6 : }
32 :
33 6 : virtual ~TableUpdate() {}
34 :
35 : // To be overriden.
36 : virtual bool Empty() const = 0;
37 :
38 : // Common interfaces.
39 92 : const nsCString& TableName() const { return mTable; }
40 :
41 : template<typename T>
42 24 : static T* Cast(TableUpdate* aThat) {
43 24 : return (T::TAG == aThat->Tag() ? reinterpret_cast<T*>(aThat) : nullptr);
44 : }
45 :
46 : private:
47 : virtual int Tag() const = 0;
48 :
49 : nsCString mTable;
50 : };
51 :
52 : // A table update is built from a single update chunk from the server. As the
53 : // protocol parser processes each chunk, it constructs a table update with the
54 : // new hashes.
55 18 : class TableUpdateV2 : public TableUpdate {
56 : public:
57 6 : explicit TableUpdateV2(const nsACString& aTable)
58 6 : : TableUpdate(aTable) {}
59 :
60 6 : bool Empty() const override {
61 6 : return mAddChunks.Length() == 0 &&
62 0 : mSubChunks.Length() == 0 &&
63 0 : mAddExpirations.Length() == 0 &&
64 0 : mSubExpirations.Length() == 0 &&
65 0 : mAddPrefixes.Length() == 0 &&
66 0 : mSubPrefixes.Length() == 0 &&
67 0 : mAddCompletes.Length() == 0 &&
68 6 : mSubCompletes.Length() == 0 &&
69 6 : mMissPrefixes.Length() == 0;
70 : }
71 :
72 : // Throughout, uint32_t aChunk refers only to the chunk number. Chunk data is
73 : // stored in the Prefix structures.
74 7 : MOZ_MUST_USE nsresult NewAddChunk(uint32_t aChunk) {
75 7 : return mAddChunks.Set(aChunk);
76 : };
77 0 : MOZ_MUST_USE nsresult NewSubChunk(uint32_t aChunk) {
78 0 : return mSubChunks.Set(aChunk);
79 : };
80 6 : MOZ_MUST_USE nsresult NewAddExpiration(uint32_t aChunk) {
81 6 : return mAddExpirations.Set(aChunk);
82 : };
83 0 : MOZ_MUST_USE nsresult NewSubExpiration(uint32_t aChunk) {
84 0 : return mSubExpirations.Set(aChunk);
85 : };
86 : MOZ_MUST_USE nsresult NewAddPrefix(uint32_t aAddChunk, const Prefix& aPrefix);
87 : MOZ_MUST_USE nsresult NewSubPrefix(uint32_t aAddChunk,
88 : const Prefix& aPrefix,
89 : uint32_t aSubChunk);
90 : MOZ_MUST_USE nsresult NewAddComplete(uint32_t aChunk,
91 : const Completion& aCompletion);
92 : MOZ_MUST_USE nsresult NewSubComplete(uint32_t aAddChunk,
93 : const Completion& aCompletion,
94 : uint32_t aSubChunk);
95 : MOZ_MUST_USE nsresult NewMissPrefix(const Prefix& aPrefix);
96 :
97 12 : ChunkSet& AddChunks() { return mAddChunks; }
98 12 : ChunkSet& SubChunks() { return mSubChunks; }
99 :
100 : // Expirations for chunks.
101 6 : ChunkSet& AddExpirations() { return mAddExpirations; }
102 6 : ChunkSet& SubExpirations() { return mSubExpirations; }
103 :
104 : // Hashes associated with this chunk.
105 6 : AddPrefixArray& AddPrefixes() { return mAddPrefixes; }
106 6 : SubPrefixArray& SubPrefixes() { return mSubPrefixes; }
107 6 : AddCompleteArray& AddCompletes() { return mAddCompletes; }
108 6 : SubCompleteArray& SubCompletes() { return mSubCompletes; }
109 :
110 : // Entries that cannot be completed.
111 0 : MissPrefixArray& MissPrefixes() { return mMissPrefixes; }
112 :
113 : // For downcasting.
114 : static const int TAG = 2;
115 :
116 : private:
117 :
118 : // The list of chunk numbers that we have for each of the type of chunks.
119 : ChunkSet mAddChunks;
120 : ChunkSet mSubChunks;
121 : ChunkSet mAddExpirations;
122 : ChunkSet mSubExpirations;
123 :
124 : // 4-byte sha256 prefixes.
125 : AddPrefixArray mAddPrefixes;
126 : SubPrefixArray mSubPrefixes;
127 :
128 : // This is only used by gethash so don't add this to Header.
129 : MissPrefixArray mMissPrefixes;
130 :
131 : // 32-byte hashes.
132 : AddCompleteArray mAddCompletes;
133 : SubCompleteArray mSubCompletes;
134 :
135 24 : virtual int Tag() const override { return TAG; }
136 : };
137 :
138 : // Structure for DBService/HashStore/Classifiers to update.
139 : // It would contain the prefixes (both fixed and variable length)
140 : // for addition and indices to removal. See Bug 1283009.
141 0 : class TableUpdateV4 : public TableUpdate {
142 : public:
143 0 : struct PrefixStdString {
144 : private:
145 : std::string mStorage;
146 : nsDependentCSubstring mString;
147 :
148 : public:
149 0 : explicit PrefixStdString(std::string& aString)
150 0 : {
151 0 : aString.swap(mStorage);
152 0 : mString.Rebind(mStorage.data(), mStorage.size());
153 0 : };
154 :
155 0 : const nsACString& GetPrefixString() const { return mString; };
156 : };
157 :
158 : typedef nsClassHashtable<nsUint32HashKey, PrefixStdString> PrefixStdStringMap;
159 : typedef nsTArray<int32_t> RemovalIndiceArray;
160 :
161 : public:
162 0 : explicit TableUpdateV4(const nsACString& aTable)
163 0 : : TableUpdate(aTable)
164 0 : , mFullUpdate(false)
165 : {
166 0 : }
167 :
168 0 : bool Empty() const override
169 : {
170 0 : return mPrefixesMap.IsEmpty() &&
171 0 : mRemovalIndiceArray.IsEmpty() &&
172 0 : mFullHashResponseMap.IsEmpty();
173 : }
174 :
175 0 : bool IsFullUpdate() const { return mFullUpdate; }
176 0 : PrefixStdStringMap& Prefixes() { return mPrefixesMap; }
177 0 : RemovalIndiceArray& RemovalIndices() { return mRemovalIndiceArray; }
178 0 : const nsACString& ClientState() const { return mClientState; }
179 0 : const nsACString& Checksum() const { return mChecksum; }
180 0 : const FullHashResponseMap& FullHashResponse() const { return mFullHashResponseMap; }
181 :
182 : // For downcasting.
183 : static const int TAG = 4;
184 :
185 0 : void SetFullUpdate(bool aIsFullUpdate) { mFullUpdate = aIsFullUpdate; }
186 : void NewPrefixes(int32_t aSize, std::string& aPrefixes);
187 : void NewRemovalIndices(const uint32_t* aIndices, size_t aNumOfIndices);
188 0 : void SetNewClientState(const nsACString& aState) { mClientState = aState; }
189 : void NewChecksum(const std::string& aChecksum);
190 : nsresult NewFullHashResponse(const Prefix& aPrefix,
191 : CachedFullHashResponse& aResponse);
192 :
193 : private:
194 0 : virtual int Tag() const override { return TAG; }
195 :
196 : bool mFullUpdate;
197 : PrefixStdStringMap mPrefixesMap;
198 : RemovalIndiceArray mRemovalIndiceArray;
199 : nsCString mClientState;
200 : nsCString mChecksum;
201 :
202 : // This is used to store response from fullHashes.find.
203 : FullHashResponseMap mFullHashResponseMap;
204 : };
205 :
206 : // There is one hash store per table.
207 43 : class HashStore {
208 : public:
209 : HashStore(const nsACString& aTableName,
210 : const nsACString& aProvider,
211 : nsIFile* aRootStoreFile);
212 : ~HashStore();
213 :
214 45 : const nsCString& TableName() const { return mTableName; }
215 :
216 : nsresult Open();
217 : // Add Prefixes are stored partly in the PrefixSet (contains the
218 : // Prefix data organized for fast lookup/low RAM usage) and partly in the
219 : // HashStore (Add Chunk numbers - only used for updates, slow retrieval).
220 : // AugmentAdds function joins the separate datasets into one complete
221 : // prefixes+chunknumbers dataset.
222 : nsresult AugmentAdds(const nsTArray<uint32_t>& aPrefixes);
223 :
224 : ChunkSet& AddChunks();
225 : ChunkSet& SubChunks();
226 6 : AddPrefixArray& AddPrefixes() { return mAddPrefixes; }
227 0 : SubPrefixArray& SubPrefixes() { return mSubPrefixes; }
228 : AddCompleteArray& AddCompletes();
229 : SubCompleteArray& SubCompletes();
230 :
231 : // =======
232 : // Updates
233 : // =======
234 : // Begin the update process. Reads the store into memory.
235 : nsresult BeginUpdate();
236 :
237 : // Imports the data from a TableUpdate.
238 : nsresult ApplyUpdate(TableUpdate &aUpdate);
239 :
240 : // Process expired chunks
241 : nsresult Expire();
242 :
243 : // Rebuild the store, Incorporating all the applied updates.
244 : nsresult Rebuild();
245 :
246 : // Write the current state of the store to disk.
247 : // If you call between ApplyUpdate() and Rebuild(), you'll
248 : // have a mess on your hands.
249 : nsresult WriteFile();
250 :
251 : // Wipe out all Completes.
252 : void ClearCompletes();
253 :
254 : private:
255 : nsresult Reset();
256 :
257 : nsresult ReadHeader();
258 : nsresult SanityCheck();
259 : nsresult CalculateChecksum(nsAutoCString& aChecksum, uint32_t aFileSize,
260 : bool aChecksumPresent);
261 : nsresult CheckChecksum(uint32_t aFileSize);
262 : void UpdateHeader();
263 :
264 : nsresult ReadCompletions();
265 : nsresult ReadChunkNumbers();
266 : nsresult ReadHashes();
267 :
268 : nsresult ReadAddPrefixes();
269 : nsresult ReadSubPrefixes();
270 :
271 : nsresult WriteAddPrefixes(nsIOutputStream* aOut);
272 : nsresult WriteSubPrefixes(nsIOutputStream* aOut);
273 :
274 : nsresult ProcessSubs();
275 :
276 : nsresult PrepareForUpdate();
277 :
278 : bool AlreadyReadChunkNumbers();
279 : bool AlreadyReadCompletions();
280 :
281 : // This is used for checking that the database is correct and for figuring out
282 : // the number of chunks, etc. to read from disk on restart.
283 : struct Header {
284 : uint32_t magic;
285 : uint32_t version;
286 : uint32_t numAddChunks;
287 : uint32_t numSubChunks;
288 : uint32_t numAddPrefixes;
289 : uint32_t numSubPrefixes;
290 : uint32_t numAddCompletes;
291 : uint32_t numSubCompletes;
292 : };
293 :
294 : Header mHeader;
295 :
296 : // The name of the table (must end in -shavar or -digest256, or evidently
297 : // -simple for unittesting.
298 : nsCString mTableName;
299 : nsCOMPtr<nsIFile> mStoreDirectory;
300 :
301 : bool mInUpdate;
302 :
303 : nsCOMPtr<nsIInputStream> mInputStream;
304 :
305 : // Chunk numbers, stored as uint32_t arrays.
306 : ChunkSet mAddChunks;
307 : ChunkSet mSubChunks;
308 :
309 : ChunkSet mAddExpirations;
310 : ChunkSet mSubExpirations;
311 :
312 : // Chunk data for shavar tables. See Entries.h for format.
313 : AddPrefixArray mAddPrefixes;
314 : SubPrefixArray mSubPrefixes;
315 :
316 : // See bug 806422 for background. We must be able to distinguish between
317 : // updates from the completion server and updates from the regular server.
318 : AddCompleteArray mAddCompletes;
319 : SubCompleteArray mSubCompletes;
320 :
321 : uint32_t mFileSize;
322 :
323 : // For gtest to inspect private members.
324 : friend class PerProviderDirectoryTestUtils;
325 : };
326 :
327 : } // namespace safebrowsing
328 : } // namespace mozilla
329 :
330 : #endif
|
