Line data Source code
1 : // © 2016 and later: Unicode, Inc. and others.
2 : // License & terms of use: http://www.unicode.org/copyright.html
3 : /*
4 : ***************************************************************************
5 : * Copyright (C) 2008-2013, International Business Machines Corporation
6 : * and others. All Rights Reserved.
7 : ***************************************************************************
8 : *
9 : * uspoof_impl.h
10 : *
11 : * Implemenation header for spoof detection
12 : *
13 : */
14 :
15 : #ifndef USPOOFIM_H
16 : #define USPOOFIM_H
17 :
18 : #include "uassert.h"
19 : #include "unicode/utypes.h"
20 : #include "unicode/uspoof.h"
21 : #include "unicode/uscript.h"
22 : #include "unicode/udata.h"
23 :
24 : #include "utrie2.h"
25 :
26 : #if !UCONFIG_NO_NORMALIZATION
27 :
28 : #ifdef __cplusplus
29 :
30 : U_NAMESPACE_BEGIN
31 :
32 : // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
33 : // a single input code point. This is function of the unicode.org data.
34 : #define USPOOF_MAX_SKELETON_EXPANSION 20
35 :
36 : // The default stack buffer size for copies or conversions or normalizations
37 : // of input strings being checked. (Used in multiple places.)
38 : #define USPOOF_STACK_BUFFER_SIZE 100
39 :
40 : // Magic number for sanity checking spoof data.
41 : #define USPOOF_MAGIC 0x3845fdef
42 :
43 : // Magic number for sanity checking spoof checkers.
44 : #define USPOOF_CHECK_MAGIC 0x2734ecde
45 :
46 : class ScriptSet;
47 : class SpoofData;
48 : struct SpoofDataHeader;
49 : class ConfusableDataUtils;
50 :
51 : /**
52 : * Class SpoofImpl corresponds directly to the plain C API opaque type
53 : * USpoofChecker. One can be cast to the other.
54 : */
55 : class SpoofImpl : public UObject {
56 : public:
57 : SpoofImpl(SpoofData *data, UErrorCode& status);
58 : SpoofImpl(UErrorCode& status);
59 : SpoofImpl();
60 : void construct(UErrorCode& status);
61 : virtual ~SpoofImpl();
62 :
63 : /** Copy constructor, used by the user level uspoof_clone() function.
64 : */
65 : SpoofImpl(const SpoofImpl &src, UErrorCode &status);
66 :
67 : USpoofChecker *asUSpoofChecker();
68 : static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
69 : static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
70 :
71 : /** Set and Get AllowedLocales, implementations of the corresponding API */
72 : void setAllowedLocales(const char *localesList, UErrorCode &status);
73 : const char * getAllowedLocales(UErrorCode &status);
74 :
75 : // Add (union) to the UnicodeSet all of the characters for the scripts used for
76 : // the specified locale. Part of the implementation of setAllowedLocales.
77 : void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
78 :
79 : // Functions implementing the features of UTS 39 section 5.
80 : static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status);
81 : void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const;
82 : void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const;
83 : void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const;
84 : URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const;
85 :
86 : /** parse a hex number. Untility used by the builders. */
87 : static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);
88 :
89 : static UClassID U_EXPORT2 getStaticClassID(void);
90 : virtual UClassID getDynamicClassID(void) const;
91 :
92 : //
93 : // Data Members
94 : //
95 :
96 : int32_t fMagic; // Internal sanity check.
97 : int32_t fChecks; // Bit vector of checks to perform.
98 :
99 : SpoofData *fSpoofData;
100 :
101 : const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters.
102 : // for this Spoof Checker. Defaults to all chars.
103 :
104 : const char *fAllowedLocales; // The list of allowed locales.
105 : URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier.
106 : };
107 :
108 : /**
109 : * Class CheckResult corresponds directly to the plain C API opaque type
110 : * USpoofCheckResult. One can be cast to the other.
111 : */
112 : class CheckResult : public UObject {
113 : public:
114 : CheckResult();
115 : virtual ~CheckResult();
116 :
117 : USpoofCheckResult *asUSpoofCheckResult();
118 : static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status);
119 : static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status);
120 :
121 : void clear();
122 :
123 : // Used to convert this CheckResult to the older int32_t return value API
124 : int32_t toCombinedBitmask(int32_t expectedChecks);
125 :
126 : // Data Members
127 : int32_t fMagic; // Internal sanity check.
128 : int32_t fChecks; // Bit vector of checks that were failed.
129 : UnicodeSet fNumerics; // Set of numerics found in the string.
130 : URestrictionLevel fRestrictionLevel; // The restriction level of the string.
131 : };
132 :
133 :
134 : //
135 : // Confusable Mappings Data Structures, version 2.0
136 : //
137 : // For the confusable data, we are essentially implementing a map,
138 : // key: a code point
139 : // value: a string. Most commonly one char in length, but can be more.
140 : //
141 : // The keys are stored as a sorted array of 32 bit ints.
142 : // bits 0-23 a code point value
143 : // bits 24-31 length of value string, in UChars (between 1 and 256 UChars).
144 : // The key table is sorted in ascending code point order. (not on the
145 : // 32 bit int value, the flag bits do not participate in the sorting.)
146 : //
147 : // Lookup is done by means of a binary search in the key table.
148 : //
149 : // The corresponding values are kept in a parallel array of 16 bit ints.
150 : // If the value string is of length 1, it is literally in the value array.
151 : // For longer strings, the value array contains an index into the strings table.
152 : //
153 : // String Table:
154 : // The strings table contains all of the value strings (those of length two or greater)
155 : // concatentated together into one long UChar (UTF-16) array.
156 : //
157 : // There is no nul character or other mark between adjacent strings.
158 : //
159 : //----------------------------------------------------------------------------
160 : //
161 : // Changes from format version 1 to format version 2:
162 : // 1) Removal of the whole-script confusable data tables.
163 : // 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask.
164 : // 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits.
165 : // 4) Removal of the string lengths table since 8 bits is sufficient for the
166 : // lengths of all entries in confusables.txt.
167 :
168 :
169 :
170 : // Internal functions for manipulating confusable data table keys
171 : #define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58
172 : class ConfusableDataUtils {
173 : public:
174 0 : inline static UChar32 keyToCodePoint(int32_t key) {
175 0 : return key & 0x00ffffff;
176 : }
177 0 : inline static int32_t keyToLength(int32_t key) {
178 0 : return ((key & 0xff000000) >> 24) + 1;
179 : }
180 : inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) {
181 : U_ASSERT((codePoint & 0x00ffffff) == codePoint);
182 : U_ASSERT(length <= 256);
183 : return codePoint | ((length - 1) << 24);
184 : }
185 : };
186 :
187 :
188 : //-------------------------------------------------------------------------------------
189 : //
190 : // SpoofData
191 : //
192 : // A small class that wraps the raw (usually memory mapped) spoof data.
193 : // Serves two primary functions:
194 : // 1. Convenience. Contains real pointers to the data, to avoid dealing with
195 : // the offsets in the raw data.
196 : // 2. Reference counting. When a spoof checker is cloned, the raw data is shared
197 : // and must be retained until all checkers using the data are closed.
198 : // Nothing in this struct includes state that is specific to any particular
199 : // USpoofDetector object.
200 : //
201 : //---------------------------------------------------------------------------------------
202 : class SpoofData: public UMemory {
203 : public:
204 : static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data.
205 : static void releaseDefault(); // Cleanup reference to default spoof data.
206 :
207 : SpoofData(UErrorCode &status); // Create new spoof data wrapper.
208 : // Only used when building new data from rules.
209 :
210 : // Constructor for use when creating from prebuilt default data.
211 : // A UDataMemory is what the ICU internal data loading functions provide.
212 : // The udm is adopted by the SpoofData.
213 : SpoofData(UDataMemory *udm, UErrorCode &status);
214 :
215 : // Constructor for use when creating from serialized data.
216 : //
217 : SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
218 :
219 : // Check raw Spoof Data Version compatibility.
220 : // Return TRUE it looks good.
221 : UBool validateDataVersion(UErrorCode &status) const;
222 :
223 : ~SpoofData(); // Destructor not normally used.
224 : // Use removeReference() instead.
225 : // Reference Counting functions.
226 : // Clone of a user-level spoof detector increments the ref count on the data.
227 : // Close of a user-level spoof detector decrements the ref count.
228 : // If the data is owned by us, it will be deleted when count goes to zero.
229 : SpoofData *addReference();
230 : void removeReference();
231 :
232 : // Reset all fields to an initial state.
233 : // Called from the top of all constructors.
234 : void reset();
235 :
236 : // Copy this instance's raw data buffer to the specified address.
237 : int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const;
238 :
239 : // Get the total number of bytes of data backed by this SpoofData.
240 : // Not to be confused with length, which returns the number of confusable entries.
241 : int32_t size() const;
242 :
243 : // Get the confusable skeleton transform for a single code point.
244 : // The result is a string with a length between 1 and 18 as of Unicode 9.
245 : // This is the main public endpoint for this class.
246 : // @return The length in UTF-16 code units of the substition string.
247 : int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const;
248 :
249 : // Get the number of confusable entries in this SpoofData.
250 : int32_t length() const;
251 :
252 : // Get the code point (key) at the specified index.
253 : UChar32 codePointAt(int32_t index) const;
254 :
255 : // Get the confusable skeleton (value) at the specified index.
256 : // Append it to the specified UnicodeString&.
257 : // @return The length in UTF-16 code units of the skeleton string.
258 : int32_t appendValueTo(int32_t index, UnicodeString& dest) const;
259 :
260 : private:
261 : // Reserve space in the raw data. For use by builder when putting together a
262 : // new set of data. Init the new storage to zero, to prevent inconsistent
263 : // results if it is not all otherwise set by the requester.
264 : // Return:
265 : // pointer to the new space that was added by this function.
266 : void *reserveSpace(int32_t numBytes, UErrorCode &status);
267 :
268 : // initialize the pointers from this object to the raw data.
269 : void initPtrs(UErrorCode &status);
270 :
271 : SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data
272 : UBool fDataOwned; // True if the raw data is owned, and needs
273 : // to be deleted when refcount goes to zero.
274 : UDataMemory *fUDM; // If not NULL, our data came from a
275 : // UDataMemory, which we must close when
276 : // we are done.
277 :
278 : uint32_t fMemLimit; // Limit of available raw data space
279 : u_atomic_int32_t fRefCount;
280 :
281 : // Confusable data
282 : int32_t *fCFUKeys;
283 : uint16_t *fCFUValues;
284 : UChar *fCFUStrings;
285 :
286 : friend class ConfusabledataBuilder;
287 : };
288 :
289 : //---------------------------------------------------------------------------------------
290 : //
291 : // Raw Binary Data Formats, as loaded from the ICU data file,
292 : // or as built by the builder.
293 : //
294 : //---------------------------------------------------------------------------------------
295 : struct SpoofDataHeader {
296 : int32_t fMagic; // (0x3845fdef)
297 : uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo
298 : // if there is one associated with this data.
299 : int32_t fLength; // Total lenght in bytes of this spoof data,
300 : // including all sections, not just the header.
301 :
302 : // The following four sections refer to data representing the confusable data
303 : // from the Unicode.org data from "confusables.txt"
304 :
305 : int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *)
306 : int32_t fCFUKeysSize; // number of entries in keys table (32 bits each)
307 :
308 : // TODO: change name to fCFUValues, for consistency.
309 : int32_t fCFUStringIndex; // byte offset to String Indexes table
310 : int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each)
311 : // (number of entries must be same as in Keys table
312 :
313 : int32_t fCFUStringTable; // byte offset of String table
314 : int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars)
315 :
316 : // The following sections are for data from xidmodifications.txt
317 :
318 : int32_t unused[15]; // Padding, Room for Expansion
319 : };
320 :
321 :
322 :
323 : U_NAMESPACE_END
324 : #endif /* __cplusplus */
325 :
326 : /**
327 : * Endianness swap function for binary spoof data.
328 : * @internal
329 : */
330 : U_CAPI int32_t U_EXPORT2
331 : uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
332 : UErrorCode *status);
333 :
334 :
335 : #endif
336 :
337 : #endif /* USPOOFIM_H */
338 :
|
