|
873.2.27
by Monty Taylor
Added in utf8cpp. We want this to be an external lib, but it needs packaged. |
1 |
// Copyright 2006 Nemanja Trifunovic
|
2 |
||
3 |
/*
|
|
4 |
Permission is hereby granted, free of charge, to any person or organization
|
|
5 |
obtaining a copy of the software and accompanying documentation covered by
|
|
6 |
this license (the "Software") to use, reproduce, display, distribute,
|
|
7 |
execute, and transmit the Software, and to prepare derivative works of the
|
|
8 |
Software, and to permit third-parties to whom the Software is furnished to
|
|
9 |
do so, all subject to the following:
|
|
10 |
||
11 |
The copyright notices in the Software and this entire statement, including
|
|
12 |
the above license grant, this restriction and the following disclaimer,
|
|
13 |
must be included in all copies of the Software, in whole or in part, and
|
|
14 |
all derivative works of the Software, unless such copies or derivative
|
|
15 |
works are solely in the form of machine-executable object code generated by
|
|
16 |
a source language processor.
|
|
17 |
||
18 |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
19 |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
20 |
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
|
21 |
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
|
22 |
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
|
23 |
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
24 |
DEALINGS IN THE SOFTWARE.
|
|
25 |
*/
|
|
26 |
||
27 |
||
|
1122.2.10
by Monty Taylor
Fixed all of the include guards. |
28 |
#ifndef DRIZZLED_UTF8_CORE_H
|
29 |
#define DRIZZLED_UTF8_CORE_H
|
|
|
873.2.27
by Monty Taylor
Added in utf8cpp. We want this to be an external lib, but it needs packaged. |
30 |
|
31 |
#include <iterator> |
|
32 |
#include CSTDINT_H
|
|
33 |
||
34 |
namespace utf8 |
|
35 |
{
|
|
36 |
||
37 |
// Helper code - not intended to be directly called by the library users. May be changed at any time
|
|
38 |
namespace internal |
|
39 |
{
|
|
40 |
// Unicode constants
|
|
41 |
// Leading (high) surrogates: 0xd800 - 0xdbff
|
|
42 |
// Trailing (low) surrogates: 0xdc00 - 0xdfff
|
|
43 |
const uint16_t LEAD_SURROGATE_MIN = 0xd800u; |
|
44 |
const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; |
|
45 |
const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; |
|
46 |
const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; |
|
47 |
const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); |
|
48 |
const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; |
|
49 |
||
50 |
// Maximum valid value for a Unicode code point
|
|
51 |
const uint32_t CODE_POINT_MAX = 0x0010ffffu; |
|
52 |
||
53 |
template<typename octet_type> |
|
54 |
inline uint8_t mask8(octet_type oc) |
|
55 |
{
|
|
56 |
return static_cast<uint8_t>(0xff & oc); |
|
57 |
}
|
|
58 |
template<typename u16_type> |
|
59 |
inline uint16_t mask16(u16_type oc) |
|
60 |
{
|
|
61 |
return static_cast<uint16_t>(0xffff & oc); |
|
62 |
}
|
|
63 |
template<typename octet_type> |
|
64 |
inline bool is_trail(octet_type oc) |
|
65 |
{
|
|
66 |
return ((mask8(oc) >> 6) == 0x2); |
|
67 |
}
|
|
68 |
||
69 |
template <typename u16> |
|
70 |
inline bool is_surrogate(u16 cp) |
|
71 |
{
|
|
72 |
return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); |
|
73 |
}
|
|
74 |
||
75 |
template <typename u32> |
|
76 |
inline bool is_code_point_valid(u32 cp) |
|
77 |
{
|
|
78 |
return (cp <= CODE_POINT_MAX && !is_surrogate(cp) && cp != 0xfffe && cp != 0xffff); |
|
79 |
}
|
|
80 |
||
81 |
template <typename octet_iterator> |
|
82 |
inline typename std::iterator_traits<octet_iterator>::difference_type |
|
83 |
sequence_length(octet_iterator lead_it) |
|
84 |
{
|
|
85 |
uint8_t lead = mask8(*lead_it); |
|
86 |
if (lead < 0x80) |
|
87 |
return 1; |
|
88 |
else if ((lead >> 5) == 0x6) |
|
89 |
return 2; |
|
90 |
else if ((lead >> 4) == 0xe) |
|
91 |
return 3; |
|
92 |
else if ((lead >> 3) == 0x1e) |
|
93 |
return 4; |
|
94 |
else
|
|
95 |
return 0; |
|
96 |
}
|
|
97 |
||
98 |
enum utf_error {OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; |
|
99 |
||
100 |
template <typename octet_iterator> |
|
101 |
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point) |
|
102 |
{
|
|
103 |
uint32_t cp = mask8(*it); |
|
104 |
// Check the lead octet
|
|
105 |
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type; |
|
106 |
octet_difference_type length = sequence_length(it); |
|
107 |
||
108 |
// "Shortcut" for ASCII characters
|
|
109 |
if (length == 1) { |
|
110 |
if (end - it > 0) { |
|
111 |
if (code_point) |
|
112 |
*code_point = cp; |
|
113 |
++it; |
|
114 |
return OK; |
|
115 |
}
|
|
116 |
else
|
|
117 |
return NOT_ENOUGH_ROOM; |
|
118 |
}
|
|
119 |
||
120 |
// Do we have enough memory?
|
|
121 |
if (std::distance(it, end) < length) |
|
122 |
return NOT_ENOUGH_ROOM; |
|
123 |
||
124 |
// Check trail octets and calculate the code point
|
|
125 |
switch (length) { |
|
126 |
case 0: |
|
127 |
return INVALID_LEAD; |
|
128 |
break; |
|
129 |
case 2: |
|
130 |
if (is_trail(*(++it))) { |
|
131 |
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); |
|
132 |
}
|
|
133 |
else { |
|
134 |
--it; |
|
135 |
return INCOMPLETE_SEQUENCE; |
|
136 |
}
|
|
137 |
break; |
|
138 |
case 3: |
|
139 |
if (is_trail(*(++it))) { |
|
140 |
cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff); |
|
141 |
if (is_trail(*(++it))) { |
|
142 |
cp += (*it) & 0x3f; |
|
143 |
}
|
|
144 |
else { |
|
145 |
std::advance(it, -2); |
|
146 |
return INCOMPLETE_SEQUENCE; |
|
147 |
}
|
|
148 |
}
|
|
149 |
else { |
|
150 |
--it; |
|
151 |
return INCOMPLETE_SEQUENCE; |
|
152 |
}
|
|
153 |
break; |
|
154 |
case 4: |
|
155 |
if (is_trail(*(++it))) { |
|
156 |
cp = ((cp << 18) & 0x1fffff) + ((mask8(*it) << 12) & 0x3ffff); |
|
157 |
if (is_trail(*(++it))) { |
|
158 |
cp += (mask8(*it) << 6) & 0xfff; |
|
159 |
if (is_trail(*(++it))) { |
|
160 |
cp += (*it) & 0x3f; |
|
161 |
}
|
|
162 |
else { |
|
163 |
std::advance(it, -3); |
|
164 |
return INCOMPLETE_SEQUENCE; |
|
165 |
}
|
|
166 |
}
|
|
167 |
else { |
|
168 |
std::advance(it, -2); |
|
169 |
return INCOMPLETE_SEQUENCE; |
|
170 |
}
|
|
171 |
}
|
|
172 |
else { |
|
173 |
--it; |
|
174 |
return INCOMPLETE_SEQUENCE; |
|
175 |
}
|
|
176 |
break; |
|
177 |
}
|
|
178 |
// Is the code point valid?
|
|
179 |
if (!is_code_point_valid(cp)) { |
|
180 |
for (octet_difference_type i = 0; i < length - 1; ++i) |
|
181 |
--it; |
|
182 |
return INVALID_CODE_POINT; |
|
183 |
}
|
|
184 |
||
185 |
if (code_point) |
|
186 |
*code_point = cp; |
|
187 |
||
188 |
if (cp < 0x80) { |
|
189 |
if (length != 1) { |
|
190 |
std::advance(it, -(length-1)); |
|
191 |
return OVERLONG_SEQUENCE; |
|
192 |
}
|
|
193 |
}
|
|
194 |
else if (cp < 0x800) { |
|
195 |
if (length != 2) { |
|
196 |
std::advance(it, -(length-1)); |
|
197 |
return OVERLONG_SEQUENCE; |
|
198 |
}
|
|
199 |
}
|
|
200 |
else if (cp < 0x10000) { |
|
201 |
if (length != 3) { |
|
202 |
std::advance(it, -(length-1)); |
|
203 |
return OVERLONG_SEQUENCE; |
|
204 |
}
|
|
205 |
}
|
|
206 |
||
207 |
++it; |
|
208 |
return OK; |
|
209 |
}
|
|
210 |
||
211 |
template <typename octet_iterator> |
|
212 |
inline utf_error validate_next(octet_iterator& it, octet_iterator end) { |
|
213 |
return validate_next(it, end, 0); |
|
214 |
}
|
|
215 |
||
216 |
} // namespace internal |
|
217 |
||
218 |
/// The library API - functions intended to be called by the users
|
|
219 |
||
220 |
// Byte order mark
|
|
221 |
const uint8_t bom[] = {0xef, 0xbb, 0xbf}; |
|
222 |
||
223 |
template <typename octet_iterator> |
|
224 |
octet_iterator find_invalid(octet_iterator start, octet_iterator end) |
|
225 |
{
|
|
226 |
octet_iterator result = start; |
|
227 |
while (result != end) { |
|
228 |
internal::utf_error err_code = internal::validate_next(result, end); |
|
229 |
if (err_code != internal::OK) |
|
230 |
return result; |
|
231 |
}
|
|
232 |
return result; |
|
233 |
}
|
|
234 |
||
235 |
template <typename octet_iterator> |
|
236 |
inline bool is_valid(octet_iterator start, octet_iterator end) |
|
237 |
{
|
|
238 |
return (find_invalid(start, end) == end); |
|
239 |
}
|
|
240 |
||
241 |
template <typename octet_iterator> |
|
242 |
inline bool is_bom (octet_iterator it) |
|
243 |
{
|
|
244 |
return ( |
|
245 |
(internal::mask8(*it++)) == bom[0] && |
|
246 |
(internal::mask8(*it++)) == bom[1] && |
|
247 |
(internal::mask8(*it)) == bom[2] |
|
248 |
);
|
|
249 |
}
|
|
250 |
} // namespace utf8 |
|
251 |
||
|
1122.2.10
by Monty Taylor
Fixed all of the include guards. |
252 |
#endif /* DRIZZLED_UTF8_CORE_H */ |
|
873.2.27
by Monty Taylor
Added in utf8cpp. We want this to be an external lib, but it needs packaged. |
253 |
|
254 |