1 /*===--- ConvertUTF.c - Universal Character Names conversions ---------------===
2  *
3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  * See https://llvm.org/LICENSE.txt for license information.
5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  *
7  *===------------------------------------------------------------------------=*/
8 /*
9  * Copyright 2001-2004 Unicode, Inc.
10  *
11  * Disclaimer
12  *
13  * This source code is provided as is by Unicode, Inc. No claims are
14  * made as to fitness for any particular purpose. No warranties of any
15  * kind are expressed or implied. The recipient agrees to determine
16  * applicability of information provided. If this file has been
17  * purchased on magnetic or optical media from Unicode, Inc., the
18  * sole remedy for any claim will be exchange of defective media
19  * within 90 days of receipt.
20  *
21  * Limitations on Rights to Redistribute This Code
22  *
23  * Unicode, Inc. hereby grants the right to freely use the information
24  * supplied in this file in the creation of products supporting the
25  * Unicode Standard, and to make copies of this file in any form
26  * for internal or external distribution as long as this notice
27  * remains attached.
28  */
29 
30 /* ---------------------------------------------------------------------
31 
32     Conversions between UTF32, UTF-16, and UTF-8. Source code file.
33     Author: Mark E. Davis, 1994.
34     Rev History: Rick McGowan, fixes & updates May 2001.
35     Sept 2001: fixed const & error conditions per
36         mods suggested by S. Parent & A. Lillich.
37     June 2002: Tim Dodd added detection and handling of incomplete
38         source sequences, enhanced error detection, added casts
39         to eliminate compiler warnings.
40     July 2003: slight mods to back out aggressive FFFE detection.
41     Jan 2004: updated switches in from-UTF8 conversions.
42     Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
43 
44     See the header file "ConvertUTF.h" for complete documentation.
45 
46 ------------------------------------------------------------------------ */
47 
48 #include "llvm/Support/ConvertUTF.h"
49 #ifdef CVTUTF_DEBUG
50 #include <stdio.h>
51 #endif
52 #include <assert.h>
53 
54 /*
55  * This code extensively uses fall-through switches.
56  * Keep the compiler from warning about that.
57  */
58 #if defined(__clang__) && defined(__has_warning)
59 # if __has_warning("-Wimplicit-fallthrough")
60 #  define ConvertUTF_DISABLE_WARNINGS \
61     _Pragma("clang diagnostic push")  \
62     _Pragma("clang diagnostic ignored \"-Wimplicit-fallthrough\"")
63 #  define ConvertUTF_RESTORE_WARNINGS \
64     _Pragma("clang diagnostic pop")
65 # endif
66 #elif defined(__GNUC__) && __GNUC__ > 6
67 # define ConvertUTF_DISABLE_WARNINGS \
68    _Pragma("GCC diagnostic push")    \
69    _Pragma("GCC diagnostic ignored \"-Wimplicit-fallthrough\"")
70 # define ConvertUTF_RESTORE_WARNINGS \
71    _Pragma("GCC diagnostic pop")
72 #endif
73 #ifndef ConvertUTF_DISABLE_WARNINGS
74 # define ConvertUTF_DISABLE_WARNINGS
75 #endif
76 #ifndef ConvertUTF_RESTORE_WARNINGS
77 # define ConvertUTF_RESTORE_WARNINGS
78 #endif
79 
80 ConvertUTF_DISABLE_WARNINGS
81 
82 namespace llvm {
83 
84 static const int halfShift  = 10; /* used for shifting by 10 bits */
85 
86 static const UTF32 halfBase = 0x0010000UL;
87 static const UTF32 halfMask = 0x3FFUL;
88 
89 #define UNI_SUR_HIGH_START  (UTF32)0xD800
90 #define UNI_SUR_HIGH_END    (UTF32)0xDBFF
91 #define UNI_SUR_LOW_START   (UTF32)0xDC00
92 #define UNI_SUR_LOW_END     (UTF32)0xDFFF
93 
94 /* --------------------------------------------------------------------- */
95 
96 /*
97  * Index into the table below with the first byte of a UTF-8 sequence to
98  * get the number of trailing bytes that are supposed to follow it.
99  * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
100  * left as-is for anyone who may want to do such conversion, which was
101  * allowed in earlier algorithms.
102  */
103 static const char trailingBytesForUTF8[256] = {
104     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
105     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
106     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
107     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
108     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
109     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
110     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
111     2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
112 };
113 
114 /*
115  * Magic values subtracted from a buffer value during UTF8 conversion.
116  * This table contains as many values as there might be trailing bytes
117  * in a UTF-8 sequence.
118  */
119 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
120                      0x03C82080UL, 0xFA082080UL, 0x82082080UL };
121 
122 /*
123  * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
124  * into the first byte, depending on how many bytes follow.  There are
125  * as many entries in this table as there are UTF-8 sequence types.
126  * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
127  * for *legal* UTF-8 will be 4 or fewer bytes total.
128  */
129 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
130 
131 /* --------------------------------------------------------------------- */
132 
133 /* The interface converts a whole buffer to avoid function-call overhead.
134  * Constants have been gathered. Loops & conditionals have been removed as
135  * much as possible for efficiency, in favor of drop-through switches.
136  * (See "Note A" at the bottom of the file for equivalent code.)
137  * If your compiler supports it, the "isLegalUTF8" call can be turned
138  * into an inline function.
139  */
140 
141 
142 /* --------------------------------------------------------------------- */
143 
ConvertUTF32toUTF16(const UTF32 ** sourceStart,const UTF32 * sourceEnd,UTF16 ** targetStart,UTF16 * targetEnd,ConversionFlags flags)144 ConversionResult ConvertUTF32toUTF16 (
145         const UTF32** sourceStart, const UTF32* sourceEnd,
146         UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
147     ConversionResult result = conversionOK;
148     const UTF32* source = *sourceStart;
149     UTF16* target = *targetStart;
150     while (source < sourceEnd) {
151         UTF32 ch;
152         if (target >= targetEnd) {
153             result = targetExhausted; break;
154         }
155         ch = *source++;
156         if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
157             /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
158             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
159                 if (flags == strictConversion) {
160                     --source; /* return to the illegal value itself */
161                     result = sourceIllegal;
162                     break;
163                 } else {
164                     *target++ = UNI_REPLACEMENT_CHAR;
165                 }
166             } else {
167                 *target++ = (UTF16)ch; /* normal case */
168             }
169         } else if (ch > UNI_MAX_LEGAL_UTF32) {
170             if (flags == strictConversion) {
171                 result = sourceIllegal;
172             } else {
173                 *target++ = UNI_REPLACEMENT_CHAR;
174             }
175         } else {
176             /* target is a character in range 0xFFFF - 0x10FFFF. */
177             if (target + 1 >= targetEnd) {
178                 --source; /* Back up source pointer! */
179                 result = targetExhausted; break;
180             }
181             ch -= halfBase;
182             *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
183             *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
184         }
185     }
186     *sourceStart = source;
187     *targetStart = target;
188     return result;
189 }
190 
191 /* --------------------------------------------------------------------- */
192 
ConvertUTF16toUTF32(const UTF16 ** sourceStart,const UTF16 * sourceEnd,UTF32 ** targetStart,UTF32 * targetEnd,ConversionFlags flags)193 ConversionResult ConvertUTF16toUTF32 (
194         const UTF16** sourceStart, const UTF16* sourceEnd,
195         UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
196     ConversionResult result = conversionOK;
197     const UTF16* source = *sourceStart;
198     UTF32* target = *targetStart;
199     UTF32 ch, ch2;
200     while (source < sourceEnd) {
201         const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
202         ch = *source++;
203         /* If we have a surrogate pair, convert to UTF32 first. */
204         if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
205             /* If the 16 bits following the high surrogate are in the source buffer... */
206             if (source < sourceEnd) {
207                 ch2 = *source;
208                 /* If it's a low surrogate, convert to UTF32. */
209                 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
210                     ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
211                         + (ch2 - UNI_SUR_LOW_START) + halfBase;
212                     ++source;
213                 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
214                     --source; /* return to the illegal value itself */
215                     result = sourceIllegal;
216                     break;
217                 }
218             } else { /* We don't have the 16 bits following the high surrogate. */
219                 --source; /* return to the high surrogate */
220                 result = sourceExhausted;
221                 break;
222             }
223         } else if (flags == strictConversion) {
224             /* UTF-16 surrogate values are illegal in UTF-32 */
225             if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
226                 --source; /* return to the illegal value itself */
227                 result = sourceIllegal;
228                 break;
229             }
230         }
231         if (target >= targetEnd) {
232             source = oldSource; /* Back up source pointer! */
233             result = targetExhausted; break;
234         }
235         *target++ = ch;
236     }
237     *sourceStart = source;
238     *targetStart = target;
239 #ifdef CVTUTF_DEBUG
240 if (result == sourceIllegal) {
241     fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
242     fflush(stderr);
243 }
244 #endif
245     return result;
246 }
ConvertUTF16toUTF8(const UTF16 ** sourceStart,const UTF16 * sourceEnd,UTF8 ** targetStart,UTF8 * targetEnd,ConversionFlags flags)247 ConversionResult ConvertUTF16toUTF8 (
248         const UTF16** sourceStart, const UTF16* sourceEnd,
249         UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
250     ConversionResult result = conversionOK;
251     const UTF16* source = *sourceStart;
252     UTF8* target = *targetStart;
253     while (source < sourceEnd) {
254         UTF32 ch;
255         unsigned short bytesToWrite = 0;
256         const UTF32 byteMask = 0xBF;
257         const UTF32 byteMark = 0x80;
258         const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
259         ch = *source++;
260         /* If we have a surrogate pair, convert to UTF32 first. */
261         if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
262             /* If the 16 bits following the high surrogate are in the source buffer... */
263             if (source < sourceEnd) {
264                 UTF32 ch2 = *source;
265                 /* If it's a low surrogate, convert to UTF32. */
266                 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
267                     ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
268                         + (ch2 - UNI_SUR_LOW_START) + halfBase;
269                     ++source;
270                 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
271                     --source; /* return to the illegal value itself */
272                     result = sourceIllegal;
273                     break;
274                 }
275             } else { /* We don't have the 16 bits following the high surrogate. */
276                 --source; /* return to the high surrogate */
277                 result = sourceExhausted;
278                 break;
279             }
280         } else if (flags == strictConversion) {
281             /* UTF-16 surrogate values are illegal in UTF-32 */
282             if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
283                 --source; /* return to the illegal value itself */
284                 result = sourceIllegal;
285                 break;
286             }
287         }
288         /* Figure out how many bytes the result will require */
289         if (ch < (UTF32)0x80) {      bytesToWrite = 1;
290         } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
291         } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
292         } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
293         } else {                            bytesToWrite = 3;
294                                             ch = UNI_REPLACEMENT_CHAR;
295         }
296 
297         target += bytesToWrite;
298         if (target > targetEnd) {
299             source = oldSource; /* Back up source pointer! */
300             target -= bytesToWrite; result = targetExhausted; break;
301         }
302         switch (bytesToWrite) { /* note: everything falls through. */
303             case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
304             case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
305             case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
306             case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
307         }
308         target += bytesToWrite;
309     }
310     *sourceStart = source;
311     *targetStart = target;
312     return result;
313 }
314 
315 /* --------------------------------------------------------------------- */
316 
ConvertUTF32toUTF8(const UTF32 ** sourceStart,const UTF32 * sourceEnd,UTF8 ** targetStart,UTF8 * targetEnd,ConversionFlags flags)317 ConversionResult ConvertUTF32toUTF8 (
318         const UTF32** sourceStart, const UTF32* sourceEnd,
319         UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
320     ConversionResult result = conversionOK;
321     const UTF32* source = *sourceStart;
322     UTF8* target = *targetStart;
323     while (source < sourceEnd) {
324         UTF32 ch;
325         unsigned short bytesToWrite = 0;
326         const UTF32 byteMask = 0xBF;
327         const UTF32 byteMark = 0x80;
328         ch = *source++;
329         if (flags == strictConversion ) {
330             /* UTF-16 surrogate values are illegal in UTF-32 */
331             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
332                 --source; /* return to the illegal value itself */
333                 result = sourceIllegal;
334                 break;
335             }
336         }
337         /*
338          * Figure out how many bytes the result will require. Turn any
339          * illegally large UTF32 things (> Plane 17) into replacement chars.
340          */
341         if (ch < (UTF32)0x80) {      bytesToWrite = 1;
342         } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
343         } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
344         } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
345         } else {                            bytesToWrite = 3;
346                                             ch = UNI_REPLACEMENT_CHAR;
347                                             result = sourceIllegal;
348         }
349 
350         target += bytesToWrite;
351         if (target > targetEnd) {
352             --source; /* Back up source pointer! */
353             target -= bytesToWrite; result = targetExhausted; break;
354         }
355         switch (bytesToWrite) { /* note: everything falls through. */
356             case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
357             case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
358             case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
359             case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
360         }
361         target += bytesToWrite;
362     }
363     *sourceStart = source;
364     *targetStart = target;
365     return result;
366 }
367 
368 /* --------------------------------------------------------------------- */
369 
370 /*
371  * Utility routine to tell whether a sequence of bytes is legal UTF-8.
372  * This must be called with the length pre-determined by the first byte.
373  * If not calling this from ConvertUTF8to*, then the length can be set by:
374  *  length = trailingBytesForUTF8[*source]+1;
375  * and the sequence is illegal right away if there aren't that many bytes
376  * available.
377  * If presented with a length > 4, this returns false.  The Unicode
378  * definition of UTF-8 goes up to 4-byte sequences.
379  */
380 
isLegalUTF8(const UTF8 * source,int length)381 static Boolean isLegalUTF8(const UTF8 *source, int length) {
382     UTF8 a;
383     const UTF8 *srcptr = source+length;
384     switch (length) {
385     default: return false;
386         /* Everything else falls through when "true"... */
387     case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
388     case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
389     case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
390 
391         switch (*source) {
392             /* no fall-through in this inner switch */
393             case 0xE0: if (a < 0xA0) return false; break;
394             case 0xED: if (a > 0x9F) return false; break;
395             case 0xF0: if (a < 0x90) return false; break;
396             case 0xF4: if (a > 0x8F) return false; break;
397             default:   if (a < 0x80) return false;
398         }
399 
400     case 1: if (*source >= 0x80 && *source < 0xC2) return false;
401     }
402     if (*source > 0xF4) return false;
403     return true;
404 }
405 
406 /* --------------------------------------------------------------------- */
407 
408 /*
409  * Exported function to return whether a UTF-8 sequence is legal or not.
410  * This is not used here; it's just exported.
411  */
isLegalUTF8Sequence(const UTF8 * source,const UTF8 * sourceEnd)412 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
413     int length = trailingBytesForUTF8[*source]+1;
414     if (length > sourceEnd - source) {
415         return false;
416     }
417     return isLegalUTF8(source, length);
418 }
419 
420 /*
421  * Exported function to return the size of the first utf-8 code unit sequence,
422  * Or 0 if the sequence is not valid;
423  */
getUTF8SequenceSize(const UTF8 * source,const UTF8 * sourceEnd)424 unsigned getUTF8SequenceSize(const UTF8 *source, const UTF8 *sourceEnd) {
425   int length = trailingBytesForUTF8[*source] + 1;
426   return (length <= sourceEnd - source && isLegalUTF8(source, length)) ? length
427                                                                        : 0;
428 }
429 
430 /* --------------------------------------------------------------------- */
431 
432 static unsigned
findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 * source,const UTF8 * sourceEnd)433 findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source,
434                                           const UTF8 *sourceEnd) {
435   UTF8 b1, b2, b3;
436 
437   assert(!isLegalUTF8Sequence(source, sourceEnd));
438 
439   /*
440    * Unicode 6.3.0, D93b:
441    *
442    *   Maximal subpart of an ill-formed subsequence: The longest code unit
443    *   subsequence starting at an unconvertible offset that is either:
444    *   a. the initial subsequence of a well-formed code unit sequence, or
445    *   b. a subsequence of length one.
446    */
447 
448   if (source == sourceEnd)
449     return 0;
450 
451   /*
452    * Perform case analysis.  See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8
453    * Byte Sequences.
454    */
455 
456   b1 = *source;
457   ++source;
458   if (b1 >= 0xC2 && b1 <= 0xDF) {
459     /*
460      * First byte is valid, but we know that this code unit sequence is
461      * invalid, so the maximal subpart has to end after the first byte.
462      */
463     return 1;
464   }
465 
466   if (source == sourceEnd)
467     return 1;
468 
469   b2 = *source;
470   ++source;
471 
472   if (b1 == 0xE0) {
473     return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1;
474   }
475   if (b1 >= 0xE1 && b1 <= 0xEC) {
476     return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
477   }
478   if (b1 == 0xED) {
479     return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1;
480   }
481   if (b1 >= 0xEE && b1 <= 0xEF) {
482     return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
483   }
484   if (b1 == 0xF0) {
485     if (b2 >= 0x90 && b2 <= 0xBF) {
486       if (source == sourceEnd)
487         return 2;
488 
489       b3 = *source;
490       return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
491     }
492     return 1;
493   }
494   if (b1 >= 0xF1 && b1 <= 0xF3) {
495     if (b2 >= 0x80 && b2 <= 0xBF) {
496       if (source == sourceEnd)
497         return 2;
498 
499       b3 = *source;
500       return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
501     }
502     return 1;
503   }
504   if (b1 == 0xF4) {
505     if (b2 >= 0x80 && b2 <= 0x8F) {
506       if (source == sourceEnd)
507         return 2;
508 
509       b3 = *source;
510       return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
511     }
512     return 1;
513   }
514 
515   assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5);
516   /*
517    * There are no valid sequences that start with these bytes.  Maximal subpart
518    * is defined to have length 1 in these cases.
519    */
520   return 1;
521 }
522 
523 /* --------------------------------------------------------------------- */
524 
525 /*
526  * Exported function to return the total number of bytes in a codepoint
527  * represented in UTF-8, given the value of the first byte.
528  */
getNumBytesForUTF8(UTF8 first)529 unsigned getNumBytesForUTF8(UTF8 first) {
530   return trailingBytesForUTF8[first] + 1;
531 }
532 
533 /* --------------------------------------------------------------------- */
534 
535 /*
536  * Exported function to return whether a UTF-8 string is legal or not.
537  * This is not used here; it's just exported.
538  */
isLegalUTF8String(const UTF8 ** source,const UTF8 * sourceEnd)539 Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) {
540     while (*source != sourceEnd) {
541         int length = trailingBytesForUTF8[**source] + 1;
542         if (length > sourceEnd - *source || !isLegalUTF8(*source, length))
543             return false;
544         *source += length;
545     }
546     return true;
547 }
548 
549 /* --------------------------------------------------------------------- */
550 
ConvertUTF8toUTF16(const UTF8 ** sourceStart,const UTF8 * sourceEnd,UTF16 ** targetStart,UTF16 * targetEnd,ConversionFlags flags)551 ConversionResult ConvertUTF8toUTF16 (
552         const UTF8** sourceStart, const UTF8* sourceEnd,
553         UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
554     ConversionResult result = conversionOK;
555     const UTF8* source = *sourceStart;
556     UTF16* target = *targetStart;
557     while (source < sourceEnd) {
558         UTF32 ch = 0;
559         unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
560         if (extraBytesToRead >= sourceEnd - source) {
561             result = sourceExhausted; break;
562         }
563         /* Do this check whether lenient or strict */
564         if (!isLegalUTF8(source, extraBytesToRead+1)) {
565             result = sourceIllegal;
566             break;
567         }
568         /*
569          * The cases all fall through. See "Note A" below.
570          */
571         switch (extraBytesToRead) {
572             case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
573             case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
574             case 3: ch += *source++; ch <<= 6;
575             case 2: ch += *source++; ch <<= 6;
576             case 1: ch += *source++; ch <<= 6;
577             case 0: ch += *source++;
578         }
579         ch -= offsetsFromUTF8[extraBytesToRead];
580 
581         if (target >= targetEnd) {
582             source -= (extraBytesToRead+1); /* Back up source pointer! */
583             result = targetExhausted; break;
584         }
585         if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
586             /* UTF-16 surrogate values are illegal in UTF-32 */
587             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
588                 if (flags == strictConversion) {
589                     source -= (extraBytesToRead+1); /* return to the illegal value itself */
590                     result = sourceIllegal;
591                     break;
592                 } else {
593                     *target++ = UNI_REPLACEMENT_CHAR;
594                 }
595             } else {
596                 *target++ = (UTF16)ch; /* normal case */
597             }
598         } else if (ch > UNI_MAX_UTF16) {
599             if (flags == strictConversion) {
600                 result = sourceIllegal;
601                 source -= (extraBytesToRead+1); /* return to the start */
602                 break; /* Bail out; shouldn't continue */
603             } else {
604                 *target++ = UNI_REPLACEMENT_CHAR;
605             }
606         } else {
607             /* target is a character in range 0xFFFF - 0x10FFFF. */
608             if (target + 1 >= targetEnd) {
609                 source -= (extraBytesToRead+1); /* Back up source pointer! */
610                 result = targetExhausted; break;
611             }
612             ch -= halfBase;
613             *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
614             *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
615         }
616     }
617     *sourceStart = source;
618     *targetStart = target;
619     return result;
620 }
621 
622 /* --------------------------------------------------------------------- */
623 
ConvertUTF8toUTF32Impl(const UTF8 ** sourceStart,const UTF8 * sourceEnd,UTF32 ** targetStart,UTF32 * targetEnd,ConversionFlags flags,Boolean InputIsPartial)624 static ConversionResult ConvertUTF8toUTF32Impl(
625         const UTF8** sourceStart, const UTF8* sourceEnd,
626         UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags,
627         Boolean InputIsPartial) {
628     ConversionResult result = conversionOK;
629     const UTF8* source = *sourceStart;
630     UTF32* target = *targetStart;
631     while (source < sourceEnd) {
632         UTF32 ch = 0;
633         unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
634         if (extraBytesToRead >= sourceEnd - source) {
635             if (flags == strictConversion || InputIsPartial) {
636                 result = sourceExhausted;
637                 break;
638             } else {
639                 result = sourceIllegal;
640 
641                 /*
642                  * Replace the maximal subpart of ill-formed sequence with
643                  * replacement character.
644                  */
645                 source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
646                                                                     sourceEnd);
647                 *target++ = UNI_REPLACEMENT_CHAR;
648                 continue;
649             }
650         }
651         if (target >= targetEnd) {
652             result = targetExhausted; break;
653         }
654 
655         /* Do this check whether lenient or strict */
656         if (!isLegalUTF8(source, extraBytesToRead+1)) {
657             result = sourceIllegal;
658             if (flags == strictConversion) {
659                 /* Abort conversion. */
660                 break;
661             } else {
662                 /*
663                  * Replace the maximal subpart of ill-formed sequence with
664                  * replacement character.
665                  */
666                 source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
667                                                                     sourceEnd);
668                 *target++ = UNI_REPLACEMENT_CHAR;
669                 continue;
670             }
671         }
672         /*
673          * The cases all fall through. See "Note A" below.
674          */
675         switch (extraBytesToRead) {
676             case 5: ch += *source++; ch <<= 6;
677             case 4: ch += *source++; ch <<= 6;
678             case 3: ch += *source++; ch <<= 6;
679             case 2: ch += *source++; ch <<= 6;
680             case 1: ch += *source++; ch <<= 6;
681             case 0: ch += *source++;
682         }
683         ch -= offsetsFromUTF8[extraBytesToRead];
684 
685         if (ch <= UNI_MAX_LEGAL_UTF32) {
686             /*
687              * UTF-16 surrogate values are illegal in UTF-32, and anything
688              * over Plane 17 (> 0x10FFFF) is illegal.
689              */
690             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
691                 if (flags == strictConversion) {
692                     source -= (extraBytesToRead+1); /* return to the illegal value itself */
693                     result = sourceIllegal;
694                     break;
695                 } else {
696                     *target++ = UNI_REPLACEMENT_CHAR;
697                 }
698             } else {
699                 *target++ = ch;
700             }
701         } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
702             result = sourceIllegal;
703             *target++ = UNI_REPLACEMENT_CHAR;
704         }
705     }
706     *sourceStart = source;
707     *targetStart = target;
708     return result;
709 }
710 
ConvertUTF8toUTF32Partial(const UTF8 ** sourceStart,const UTF8 * sourceEnd,UTF32 ** targetStart,UTF32 * targetEnd,ConversionFlags flags)711 ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart,
712                                            const UTF8 *sourceEnd,
713                                            UTF32 **targetStart,
714                                            UTF32 *targetEnd,
715                                            ConversionFlags flags) {
716   return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
717                                 flags, /*InputIsPartial=*/true);
718 }
719 
ConvertUTF8toUTF32(const UTF8 ** sourceStart,const UTF8 * sourceEnd,UTF32 ** targetStart,UTF32 * targetEnd,ConversionFlags flags)720 ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart,
721                                     const UTF8 *sourceEnd, UTF32 **targetStart,
722                                     UTF32 *targetEnd, ConversionFlags flags) {
723   return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
724                                 flags, /*InputIsPartial=*/false);
725 }
726 
727 /* ---------------------------------------------------------------------
728 
729     Note A.
730     The fall-through switches in UTF-8 reading code save a
731     temp variable, some decrements & conditionals.  The switches
732     are equivalent to the following loop:
733         {
734             int tmpBytesToRead = extraBytesToRead+1;
735             do {
736                 ch += *source++;
737                 --tmpBytesToRead;
738                 if (tmpBytesToRead) ch <<= 6;
739             } while (tmpBytesToRead > 0);
740         }
741     In UTF-8 writing code, the switches on "bytesToWrite" are
742     similarly unrolled loops.
743 
744    --------------------------------------------------------------------- */
745 
746 } // namespace llvm
747 
748 ConvertUTF_RESTORE_WARNINGS
749