upgrade utfcpp to v2.3.5
From https://github.com/nemtrif/utfcpp/releases/tag/v2.3.5 * utf8/utf8.h, utf8/utf8/checked.h, utf8/utf8/core.h, utf8/utf8/unchecked.h: Update. * utf8/doc/utf8cpp.html, utf8/doc/ReleaseNotes: Delete. * utf8/README.md: New file. * Makefile.am: Adjust.
This commit is contained in:
Alexandre Duret-Lutz
parent
b81d7e5839
commit
07e714e14e
8 changed files with 1320 additions and 2061 deletions
216
utf8/utf8/core.h
216
utf8/utf8/core.h
|
|
@ -68,7 +68,7 @@ namespace internal
|
|||
template<typename octet_type>
|
||||
inline bool is_trail(octet_type oc)
|
||||
{
|
||||
return ((mask8(oc) >> 6) == 0x2);
|
||||
return ((utf8::internal::mask8(oc) >> 6) == 0x2);
|
||||
}
|
||||
|
||||
template <typename u16>
|
||||
|
|
@ -92,14 +92,14 @@ namespace internal
|
|||
template <typename u32>
|
||||
inline bool is_code_point_valid(u32 cp)
|
||||
{
|
||||
return (cp <= CODE_POINT_MAX && !is_surrogate(cp));
|
||||
return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline typename std::iterator_traits<octet_iterator>::difference_type
|
||||
sequence_length(octet_iterator lead_it)
|
||||
{
|
||||
uint8_t lead = mask8(*lead_it);
|
||||
uint8_t lead = utf8::internal::mask8(*lead_it);
|
||||
if (lead < 0x80)
|
||||
return 1;
|
||||
else if ((lead >> 5) == 0x6)
|
||||
|
|
@ -116,15 +116,15 @@ namespace internal
|
|||
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
|
||||
{
|
||||
if (cp < 0x80) {
|
||||
if (length != 1)
|
||||
if (length != 1)
|
||||
return true;
|
||||
}
|
||||
else if (cp < 0x800) {
|
||||
if (length != 2)
|
||||
if (length != 2)
|
||||
return true;
|
||||
}
|
||||
else if (cp < 0x10000) {
|
||||
if (length != 3)
|
||||
if (length != 3)
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -133,124 +133,98 @@ namespace internal
|
|||
|
||||
enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
|
||||
|
||||
/// Helper for get_sequence_x
|
||||
template <typename octet_iterator>
|
||||
utf_error increase_safely(octet_iterator& it, octet_iterator end)
|
||||
{
|
||||
if (++it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
if (!utf8::internal::is_trail(*it))
|
||||
return INCOMPLETE_SEQUENCE;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
|
||||
|
||||
/// get_sequence_x functions decode utf-8 sequences of the length x
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t* code_point)
|
||||
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it != end) {
|
||||
if (code_point)
|
||||
*code_point = mask8(*it);
|
||||
return UTF8_OK;
|
||||
}
|
||||
return NOT_ENOUGH_ROOM;
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t* code_point)
|
||||
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
utf_error ret_code = NOT_ENOUGH_ROOM;
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
if (it != end) {
|
||||
uint32_t cp = mask8(*it);
|
||||
if (++it != end) {
|
||||
if (is_trail(*it)) {
|
||||
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
if (code_point)
|
||||
*code_point = cp;
|
||||
ret_code = UTF8_OK;
|
||||
}
|
||||
else
|
||||
ret_code = INCOMPLETE_SEQUENCE;
|
||||
}
|
||||
else
|
||||
ret_code = NOT_ENOUGH_ROOM;
|
||||
}
|
||||
code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
|
||||
|
||||
return ret_code;
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t* code_point)
|
||||
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
utf_error ret_code = NOT_ENOUGH_ROOM;
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
if (it != end) {
|
||||
uint32_t cp = mask8(*it);
|
||||
if (++it != end) {
|
||||
if (is_trail(*it)) {
|
||||
cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff);
|
||||
if (++it != end) {
|
||||
if (is_trail(*it)) {
|
||||
cp += (*it) & 0x3f;
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
if (code_point)
|
||||
*code_point = cp;
|
||||
ret_code = UTF8_OK;
|
||||
}
|
||||
else
|
||||
ret_code = INCOMPLETE_SEQUENCE;
|
||||
}
|
||||
else
|
||||
ret_code = NOT_ENOUGH_ROOM;
|
||||
}
|
||||
else
|
||||
ret_code = INCOMPLETE_SEQUENCE;
|
||||
}
|
||||
else
|
||||
ret_code = NOT_ENOUGH_ROOM;
|
||||
}
|
||||
code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
|
||||
|
||||
return ret_code;
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point += (*it) & 0x3f;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t* code_point)
|
||||
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
utf_error ret_code = NOT_ENOUGH_ROOM;
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
if (it != end) {
|
||||
uint32_t cp = mask8(*it);
|
||||
if (++it != end) {
|
||||
if (is_trail(*it)) {
|
||||
cp = ((cp << 18) & 0x1fffff) + ((mask8(*it) << 12) & 0x3ffff);
|
||||
if (++it != end) {
|
||||
if (is_trail(*it)) {
|
||||
cp += (mask8(*it) << 6) & 0xfff;
|
||||
if (++it != end) {
|
||||
if (is_trail(*it)) {
|
||||
cp += (*it) & 0x3f;
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
if (code_point)
|
||||
*code_point = cp;
|
||||
ret_code = UTF8_OK;
|
||||
}
|
||||
else
|
||||
ret_code = INCOMPLETE_SEQUENCE;
|
||||
}
|
||||
else
|
||||
ret_code = NOT_ENOUGH_ROOM;
|
||||
}
|
||||
else
|
||||
ret_code = INCOMPLETE_SEQUENCE;
|
||||
}
|
||||
else
|
||||
ret_code = NOT_ENOUGH_ROOM;
|
||||
}
|
||||
else
|
||||
ret_code = INCOMPLETE_SEQUENCE;
|
||||
}
|
||||
else
|
||||
ret_code = NOT_ENOUGH_ROOM;
|
||||
}
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
return ret_code;
|
||||
code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point += (*it) & 0x3f;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point)
|
||||
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
// Save the original value of it so we can go back in case of failure
|
||||
// Of course, it does not make much sense with i.e. stream iterators
|
||||
octet_iterator original_it = it;
|
||||
|
|
@ -258,41 +232,40 @@ namespace internal
|
|||
uint32_t cp = 0;
|
||||
// Determine the sequence length based on the lead octet
|
||||
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
|
||||
octet_difference_type length = sequence_length(it);
|
||||
if (length == 0)
|
||||
return INVALID_LEAD;
|
||||
const octet_difference_type length = utf8::internal::sequence_length(it);
|
||||
|
||||
// Now that we have a valid sequence length, get trail octets and calculate the code point
|
||||
// Get trail octets and calculate the code point
|
||||
utf_error err = UTF8_OK;
|
||||
switch (length) {
|
||||
case 0:
|
||||
return INVALID_LEAD;
|
||||
case 1:
|
||||
err = get_sequence_1(it, end, &cp);
|
||||
err = utf8::internal::get_sequence_1(it, end, cp);
|
||||
break;
|
||||
case 2:
|
||||
err = get_sequence_2(it, end, &cp);
|
||||
err = utf8::internal::get_sequence_2(it, end, cp);
|
||||
break;
|
||||
case 3:
|
||||
err = get_sequence_3(it, end, &cp);
|
||||
err = utf8::internal::get_sequence_3(it, end, cp);
|
||||
break;
|
||||
case 4:
|
||||
err = get_sequence_4(it, end, &cp);
|
||||
err = utf8::internal::get_sequence_4(it, end, cp);
|
||||
break;
|
||||
}
|
||||
|
||||
if (err == UTF8_OK) {
|
||||
// Decoding succeeded. Now, security checks...
|
||||
if (is_code_point_valid(cp)) {
|
||||
if (!is_overlong_sequence(cp, length)){
|
||||
if (utf8::internal::is_code_point_valid(cp)) {
|
||||
if (!utf8::internal::is_overlong_sequence(cp, length)){
|
||||
// Passed! Return here.
|
||||
if (code_point)
|
||||
*code_point = cp;
|
||||
code_point = cp;
|
||||
++it;
|
||||
return UTF8_OK;
|
||||
}
|
||||
else
|
||||
err = OVERLONG_SEQUENCE;
|
||||
}
|
||||
else
|
||||
else
|
||||
err = INVALID_CODE_POINT;
|
||||
}
|
||||
|
||||
|
|
@ -303,7 +276,8 @@ namespace internal
|
|||
|
||||
template <typename octet_iterator>
|
||||
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
|
||||
return validate_next(it, end, nullptr);
|
||||
uint32_t ignored;
|
||||
return utf8::internal::validate_next(it, end, ignored);
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
|
@ -318,7 +292,7 @@ namespace internal
|
|||
{
|
||||
octet_iterator result = start;
|
||||
while (result != end) {
|
||||
internal::utf_error err_code = internal::validate_next(result, end);
|
||||
utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
|
||||
if (err_code != internal::UTF8_OK)
|
||||
return result;
|
||||
}
|
||||
|
|
@ -328,29 +302,31 @@ namespace internal
|
|||
template <typename octet_iterator>
|
||||
inline bool is_valid(octet_iterator start, octet_iterator end)
|
||||
{
|
||||
return (find_invalid(start, end) == end);
|
||||
return (utf8::find_invalid(start, end) == end);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline bool starts_with_bom (octet_iterator it, octet_iterator end)
|
||||
{
|
||||
return (
|
||||
((it != end) && (internal::mask8(*it++)) == bom[0]) &&
|
||||
((it != end) && (internal::mask8(*it++)) == bom[1]) &&
|
||||
((it != end) && (internal::mask8(*it)) == bom[2])
|
||||
((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
|
||||
((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
|
||||
((it != end) && (utf8::internal::mask8(*it)) == bom[2])
|
||||
);
|
||||
}
|
||||
|
||||
//Deprecated in release 2.3
|
||||
|
||||
//Deprecated in release 2.3
|
||||
template <typename octet_iterator>
|
||||
inline bool is_bom (octet_iterator it)
|
||||
{
|
||||
return (
|
||||
(internal::mask8(*it++)) == bom[0] &&
|
||||
(internal::mask8(*it++)) == bom[1] &&
|
||||
(internal::mask8(*it)) == bom[2]
|
||||
(utf8::internal::mask8(*it++)) == bom[0] &&
|
||||
(utf8::internal::mask8(*it++)) == bom[1] &&
|
||||
(utf8::internal::mask8(*it)) == bom[2]
|
||||
);
|
||||
}
|
||||
} // namespace utf8
|
||||
|
||||
#endif // header guard
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue