mirror of
https://github.com/cocos/cocos4.git
synced 2026-11-20 23:52:40 +08:00
570 lines
19 KiB
C++
Executable File
570 lines
19 KiB
C++
Executable File
#include "UTFString.h"
|
|
#include "memory/Memory.h"
|
|
|
|
namespace cc {
|
|
|
|
UTFString::unicode_char UTFString::_base_iterator::_getCharacter() const {
|
|
return string_->getChar(_get_index());
|
|
}
|
|
|
|
int UTFString::_base_iterator::_setCharacter(unicode_char uc) {
|
|
size_type current_index = _get_index();
|
|
int change = string_->setChar(current_index, uc);
|
|
_jump_to(current_index);
|
|
return change;
|
|
}
|
|
|
|
void UTFString::_base_iterator::_moveNext() {
|
|
_seekFwd(1); // move 1 code point forward
|
|
if (_test_end()) return; // exit if we hit the end
|
|
if (_utf16_surrogate_follow(iter_[0])) {
|
|
// landing on a follow code point means we might be part of a bigger character
|
|
// so we test for that
|
|
code_point lead_half = 0;
|
|
//NB: we can't possibly be at the beginning here, so no need to test
|
|
lead_half = iter_[-1]; // check the previous code point to see if we're part of a surrogate pair
|
|
if (_utf16_surrogate_lead(lead_half)) {
|
|
_seekFwd(1); // if so, then advance 1 more code point
|
|
}
|
|
}
|
|
}
|
|
|
|
void UTFString::_base_iterator::_movePrev() {
|
|
_seekRev(1); // move 1 code point backwards
|
|
if (_test_begin()) return; // exit if we hit the beginning
|
|
if (_utf16_surrogate_follow(iter_[0])) {
|
|
// landing on a follow code point means we might be part of a bigger character
|
|
// so we test for that
|
|
code_point lead_half = 0;
|
|
lead_half = iter_[-1]; // check the previous character to see if we're part of a surrogate pair
|
|
if (_utf16_surrogate_lead(lead_half)) {
|
|
_seekRev(1); // if so, then rewind 1 more code point
|
|
}
|
|
}
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
|
|
UTFString::unicode_char UTFString::getChar(size_type loc) const {
|
|
const code_point *ptr = c_str();
|
|
unicode_char uc;
|
|
size_t l = _utf16_char_length(ptr[loc]);
|
|
code_point cp[2] = {/* blame the code beautifier */
|
|
0, 0};
|
|
cp[0] = ptr[loc];
|
|
|
|
if (l == 2 && (loc + 1) < m_data.length()) {
|
|
cp[1] = ptr[loc + 1];
|
|
}
|
|
_utf16_to_utf32(cp, uc);
|
|
return uc;
|
|
}
|
|
|
|
int UTFString::setChar(size_type loc, unicode_char ch) {
|
|
code_point cp[2] = {/* blame the code beautifier */
|
|
0, 0};
|
|
size_t l = _utf32_to_utf16(ch, cp);
|
|
unicode_char existingChar = getChar(loc);
|
|
size_t existingSize = _utf16_char_length(existingChar);
|
|
size_t newSize = _utf16_char_length(ch);
|
|
|
|
if (newSize > existingSize) {
|
|
at(loc) = cp[0];
|
|
insert(loc + 1, 1, cp[1]);
|
|
return 1;
|
|
}
|
|
if (newSize < existingSize) {
|
|
erase(loc, 1);
|
|
at(loc) = cp[0];
|
|
return -1;
|
|
}
|
|
|
|
// newSize == existingSize
|
|
at(loc) = cp[0];
|
|
if (l == 2) at(loc + 1) = cp[1];
|
|
return 0;
|
|
}
|
|
|
|
UTFString &UTFString::assign(const String &str) {
|
|
size_type len = _verifyUTF8(str);
|
|
clear(); // empty our contents, if there are any
|
|
if (len == 0) {
|
|
return *this;
|
|
}
|
|
|
|
reserve(len); // best guess bulk capacity growth
|
|
|
|
// This is a 3 step process, converting each byte in the UTF-8 stream to UTF-32,
|
|
// then converting it to UTF-16, then finally appending the data buffer
|
|
|
|
unicode_char uc = 0; // temporary Unicode character buffer
|
|
unsigned char utf8buf[7]; // temporary UTF-8 buffer
|
|
utf8buf[6] = 0;
|
|
size_t utf8len; // UTF-8 length
|
|
code_point utf16buff[3]; // temporary UTF-16 buffer
|
|
utf16buff[2] = 0;
|
|
size_t utf16len; // UTF-16 length
|
|
|
|
String::const_iterator i, ie = str.end();
|
|
for (i = str.begin(); i != ie; i++) {
|
|
utf8len = _utf8_char_length(static_cast<unsigned char>(*i)); // estimate bytes to load
|
|
for (size_t j = 0; j < utf8len; j++) { // load the needed UTF-8 bytes
|
|
utf8buf[j] = (static_cast<unsigned char>(*(i + j))); // we don't increment 'i' here just in case the estimate is wrong (shouldn't happen, but we're being careful)
|
|
}
|
|
utf8buf[utf8len] = 0; // nul terminate so we throw an exception before running off the end of the buffer
|
|
utf8len = _utf8_to_utf32(utf8buf, uc); // do the UTF-8 -> UTF-32 conversion
|
|
i += utf8len - 1; // we subtract 1 for the increment of the 'for' loop
|
|
|
|
utf16len = _utf32_to_utf16(uc, utf16buff); // UTF-32 -> UTF-16 conversion
|
|
append(utf16buff, utf16len); // append the characters to the string
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
UTFString &UTFString::assign(const WString &wstr) {
|
|
m_data.clear();
|
|
m_data.reserve(wstr.length()); // best guess bulk allocate
|
|
#ifdef WCHAR_UTF16 // if we're already working in UTF-16, this is easy
|
|
code_point tmp;
|
|
WString::const_iterator i, ie = wstr.end();
|
|
for (i = wstr.begin(); i != ie; i++) {
|
|
tmp = static_cast<code_point>(*i);
|
|
m_data.push_back(tmp);
|
|
}
|
|
#else // otherwise we do it the safe way (which is still 100% safe to pass UTF-16 through, just slower)
|
|
code_point cp[3] = {0, 0, 0};
|
|
unicode_char tmp;
|
|
WString::const_iterator i, ie = wstr.end();
|
|
for (i = wstr.begin(); i != ie; i++) {
|
|
tmp = static_cast<unicode_char>(*i);
|
|
size_t l = _utf32_to_utf16(tmp, cp);
|
|
if (l > 0) m_data.push_back(cp[0]);
|
|
if (l > 1) m_data.push_back(cp[1]);
|
|
}
|
|
#endif
|
|
return *this;
|
|
}
|
|
|
|
UTFString &UTFString::append(size_type num, unicode_char ch) {
|
|
code_point cp[2] = {0, 0};
|
|
if (_utf32_to_utf16(ch, cp) == 2) {
|
|
for (size_type i = 0; i < num; i++) {
|
|
append(1, cp[0]);
|
|
append(1, cp[1]);
|
|
}
|
|
} else {
|
|
for (size_type i = 0; i < num; i++) {
|
|
append(1, cp[0]);
|
|
}
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
UTFString &UTFString::insert(size_type index, size_type num, unicode_char ch) {
|
|
code_point cp[3] = {0, 0, 0};
|
|
size_t l = _utf32_to_utf16(ch, cp);
|
|
if (l == 1) {
|
|
return insert(index, num, cp[0]);
|
|
}
|
|
for (size_type c = 0; c < num; c++) {
|
|
// insert in reverse order to preserve ordering after insert
|
|
insert(index, 1, cp[1]);
|
|
insert(index, 1, cp[0]);
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
void UTFString::insert(iterator i, size_type num, const unicode_char &ch) {
|
|
code_point cp[3] = {0, 0, 0};
|
|
size_t l = _utf32_to_utf16(ch, cp);
|
|
if (l == 1) {
|
|
insert(i, num, cp[0]);
|
|
} else {
|
|
for (size_type c = 0; c < num; c++) {
|
|
// insert in reverse order to preserve ordering after insert
|
|
insert(i, 1, cp[1]);
|
|
insert(i, 1, cp[0]);
|
|
}
|
|
}
|
|
}
|
|
|
|
UTFString::size_type UTFString::find_first_of(const UTFString &str, size_type index /*= 0*/, size_type num /*= npos */) const {
|
|
size_type i = 0;
|
|
const size_type len = length();
|
|
while (i < num && (index + i) < len) {
|
|
unicode_char ch = getChar(index + i);
|
|
if (str.inString(ch))
|
|
return index + i;
|
|
i += _utf16_char_length(ch); // increment by the Unicode character length
|
|
}
|
|
return npos;
|
|
}
|
|
|
|
UTFString::size_type UTFString::find_first_not_of(const UTFString &str, size_type index /*= 0*/, size_type num /*= npos */) const {
|
|
size_type i = 0;
|
|
const size_type len = length();
|
|
while (i < num && (index + i) < len) {
|
|
unicode_char ch = getChar(index + i);
|
|
if (!str.inString(ch))
|
|
return index + i;
|
|
i += _utf16_char_length(ch); // increment by the Unicode character length
|
|
}
|
|
return npos;
|
|
}
|
|
|
|
UTFString::size_type UTFString::find_last_of(const UTFString &str, size_type index /*= npos*/, size_type num /*= npos */) const {
|
|
size_type i = 0;
|
|
const size_type len = length();
|
|
if (index > len) index = len - 1;
|
|
|
|
while (i < num && (index - i) != npos) {
|
|
size_type j = index - i;
|
|
// careful to step full Unicode characters
|
|
if (j != 0 && _utf16_surrogate_follow(at(j)) && _utf16_surrogate_lead(at(j - 1))) {
|
|
j = index - ++i;
|
|
}
|
|
// and back to the usual dull test
|
|
unicode_char ch = getChar(j);
|
|
if (str.inString(ch))
|
|
return j;
|
|
i++;
|
|
}
|
|
return npos;
|
|
}
|
|
|
|
UTFString::size_type UTFString::find_last_not_of(const UTFString &str, size_type index /*= npos*/, size_type num /*= npos */) const {
|
|
size_type i = 0;
|
|
const size_type len = length();
|
|
if (index > len) index = len - 1;
|
|
|
|
while (i < num && (index - i) != npos) {
|
|
size_type j = index - i;
|
|
// careful to step full Unicode characters
|
|
if (j != 0 && _utf16_surrogate_follow(at(j)) && _utf16_surrogate_lead(at(j - 1))) {
|
|
j = index - ++i;
|
|
}
|
|
// and back to the usual dull test
|
|
unicode_char ch = getChar(j);
|
|
if (!str.inString(ch))
|
|
return j;
|
|
i++;
|
|
}
|
|
return npos;
|
|
}
|
|
|
|
size_t UTFString::_utf16_to_utf32(const code_point in_cp[2], unicode_char &out_uc) {
|
|
const code_point &cp1 = in_cp[0];
|
|
const code_point &cp2 = in_cp[1];
|
|
bool wordPair = false;
|
|
|
|
// does it look like a surrogate pair?
|
|
if (0xD800 <= cp1 && cp1 <= 0xDBFF) {
|
|
// looks like one, but does the other half match the algorithm as well?
|
|
if (0xDC00 <= cp2 && cp2 <= 0xDFFF)
|
|
wordPair = true; // yep!
|
|
}
|
|
|
|
if (!wordPair) { // if we aren't a 100% authentic surrogate pair, then just copy the value
|
|
out_uc = cp1;
|
|
return 1;
|
|
}
|
|
|
|
unsigned short cU = cp1, cL = cp2; // copy upper and lower words of surrogate pair to writable buffers
|
|
cU -= 0xD800; // remove the encoding markers
|
|
cL -= 0xDC00;
|
|
|
|
out_uc = (cU & 0x03FF) << 10; // grab the 10 upper bits and set them in their proper location
|
|
out_uc |= (cL & 0x03FF); // combine in the lower 10 bits
|
|
out_uc += 0x10000; // add back in the value offset
|
|
|
|
return 2; // this whole operation takes to words, so that's what we'll return
|
|
}
|
|
|
|
size_t UTFString::_utf32_to_utf16(const unicode_char &in_uc, code_point out_cp[2]) {
|
|
if (in_uc <= 0xFFFF) { // we blindly preserve sentinel values because our decoder understands them
|
|
out_cp[0] = static_cast<code_point>(in_uc);
|
|
return 1;
|
|
}
|
|
unicode_char uc = in_uc; // copy to writable buffer
|
|
unsigned short tmp; // single code point buffer
|
|
uc -= 0x10000; // subtract value offset
|
|
|
|
//process upper word
|
|
tmp = static_cast<unsigned short>((uc >> 10) & 0x03FF); // grab the upper 10 bits
|
|
tmp += 0xD800; // add encoding offset
|
|
out_cp[0] = tmp; // write
|
|
|
|
// process lower word
|
|
tmp = static_cast<unsigned short>(uc & 0x03FF); // grab the lower 10 bits
|
|
tmp += 0xDC00; // add encoding offset
|
|
out_cp[1] = tmp; // write
|
|
|
|
return 2; // return used word count (2 for surrogate pairs)
|
|
}
|
|
|
|
size_t UTFString::_utf8_char_length(unsigned char cp) {
|
|
if (!(cp & 0x80)) return 1;
|
|
if ((cp & ~_lead1_mask) == _lead1) return 2;
|
|
if ((cp & ~_lead2_mask) == _lead2) return 3;
|
|
if ((cp & ~_lead3_mask) == _lead3) return 4;
|
|
if ((cp & ~_lead4_mask) == _lead4) return 5;
|
|
if ((cp & ~_lead5_mask) == _lead5) return 6;
|
|
|
|
CCASSERT(0, "invalid UTF-8 sequence header value");
|
|
return 1;
|
|
}
|
|
|
|
size_t UTFString::_utf8_char_length(unicode_char uc) {
|
|
/*
|
|
7 bit: U-00000000 - U-0000007F: 0xxxxxxx
|
|
11 bit: U-00000080 - U-000007FF: 110xxxxx 10xxxxxx
|
|
16 bit: U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
|
|
21 bit: U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
26 bit: U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
31 bit: U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
*/
|
|
if (!(uc & ~0x0000007F)) return 1;
|
|
if (!(uc & ~0x000007FF)) return 2;
|
|
if (!(uc & ~0x0000FFFF)) return 3;
|
|
if (!(uc & ~0x001FFFFF)) return 4;
|
|
if (!(uc & ~0x03FFFFFF)) return 5;
|
|
if (!(uc & ~0x7FFFFFFF)) return 6;
|
|
|
|
CCASSERT(0, "invalid UTF-32 value");
|
|
return 1;
|
|
}
|
|
|
|
size_t UTFString::_utf8_to_utf32(const unsigned char in_cp[6], unicode_char &out_uc) {
|
|
size_t len = _utf8_char_length(in_cp[0]);
|
|
if (len == 1) { // if we are only 1 byte long, then just grab it and exit
|
|
out_uc = in_cp[0];
|
|
return 1;
|
|
}
|
|
|
|
unicode_char c = 0; // temporary buffer
|
|
size_t i = 0;
|
|
switch (len) { // load header byte
|
|
case 6:
|
|
c = in_cp[i] & _lead5_mask;
|
|
break;
|
|
case 5:
|
|
c = in_cp[i] & _lead4_mask;
|
|
break;
|
|
case 4:
|
|
c = in_cp[i] & _lead3_mask;
|
|
break;
|
|
case 3:
|
|
c = in_cp[i] & _lead2_mask;
|
|
break;
|
|
case 2:
|
|
c = in_cp[i] & _lead1_mask;
|
|
break;
|
|
}
|
|
|
|
for (++i; i < len; i++) { // load each continuation byte
|
|
if ((in_cp[i] & ~_cont_mask) != _cont) {
|
|
CCASSERT(0, "bad UTF-8 continuation byte");
|
|
break;
|
|
}
|
|
c <<= 6;
|
|
c |= (in_cp[i] & _cont_mask);
|
|
}
|
|
|
|
out_uc = c; // write the final value and return the used byte length
|
|
return len;
|
|
}
|
|
|
|
size_t UTFString::_utf32_to_utf8(const unicode_char &in_uc, unsigned char out_cp[6]) {
|
|
size_t len = _utf8_char_length(in_uc); // predict byte length of sequence
|
|
unicode_char c = in_uc; // copy to temp buffer
|
|
|
|
//stuff all of the lower bits
|
|
for (size_t i = len - 1; i > 0; i--) {
|
|
out_cp[i] = static_cast<unsigned char>(((c)&_cont_mask) | _cont);
|
|
c >>= 6;
|
|
}
|
|
|
|
//now write the header byte
|
|
switch (len) {
|
|
case 6:
|
|
out_cp[0] = static_cast<unsigned char>(((c)&_lead5_mask) | _lead5);
|
|
break;
|
|
case 5:
|
|
out_cp[0] = static_cast<unsigned char>(((c)&_lead4_mask) | _lead4);
|
|
break;
|
|
case 4:
|
|
out_cp[0] = static_cast<unsigned char>(((c)&_lead3_mask) | _lead3);
|
|
break;
|
|
case 3:
|
|
out_cp[0] = static_cast<unsigned char>(((c)&_lead2_mask) | _lead2);
|
|
break;
|
|
case 2:
|
|
out_cp[0] = static_cast<unsigned char>(((c)&_lead1_mask) | _lead1);
|
|
break;
|
|
case 1:
|
|
default:
|
|
out_cp[0] = static_cast<unsigned char>((c)&0x7F);
|
|
break;
|
|
}
|
|
|
|
// return the byte length of the sequence
|
|
return len;
|
|
}
|
|
|
|
UTFString::size_type UTFString::_verifyUTF8(const String &str) {
|
|
String::const_iterator i, ie = str.end();
|
|
i = str.begin();
|
|
size_type length = 0;
|
|
|
|
while (i != ie) {
|
|
// characters pass until we find an extended sequence
|
|
if ((*i) & 0x80) {
|
|
unsigned char c = (*i);
|
|
size_t contBytes = 0;
|
|
|
|
// get continuation byte count and test for overlong sequences
|
|
if ((c & ~_lead1_mask) == _lead1) { // 1 additional byte
|
|
if (c == _lead1) {
|
|
CCASSERT(0, "overlong UTF-8 sequence");
|
|
return 0;
|
|
}
|
|
contBytes = 1;
|
|
|
|
} else if ((c & ~_lead2_mask) == _lead2) { // 2 additional bytes
|
|
contBytes = 2;
|
|
if (c == _lead2) { // possible overlong UTF-8 sequence
|
|
c = (*(i + 1)); // look ahead to next byte in sequence
|
|
if ((c & _lead2) == _cont) {
|
|
CCASSERT(0, "overlong UTF-8 sequence");
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
} else if ((c & ~_lead3_mask) == _lead3) { // 3 additional bytes
|
|
contBytes = 3;
|
|
if (c == _lead3) { // possible overlong UTF-8 sequence
|
|
c = (*(i + 1)); // look ahead to next byte in sequence
|
|
if ((c & _lead3) == _cont) {
|
|
CCASSERT(0, "overlong UTF-8 sequence");
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
} else if ((c & ~_lead4_mask) == _lead4) { // 4 additional bytes
|
|
contBytes = 4;
|
|
if (c == _lead4) { // possible overlong UTF-8 sequence
|
|
c = (*(i + 1)); // look ahead to next byte in sequence
|
|
if ((c & _lead4) == _cont) {
|
|
CCASSERT(0, "overlong UTF-8 sequence");
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
} else if ((c & ~_lead5_mask) == _lead5) { // 5 additional bytes
|
|
contBytes = 5;
|
|
if (c == _lead5) { // possible overlong UTF-8 sequence
|
|
c = (*(i + 1)); // look ahead to next byte in sequence
|
|
if ((c & _lead5) == _cont) {
|
|
CCASSERT(0, "overlong UTF-8 sequence");
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
// check remaining continuation bytes for
|
|
while (contBytes--) {
|
|
c = (*(++i)); // get next byte in sequence
|
|
if ((c & ~_cont_mask) != _cont) {
|
|
CCASSERT(0, "bad UTF-8 continuation byte");
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
length++;
|
|
i++;
|
|
}
|
|
return length;
|
|
}
|
|
|
|
void UTFString::_cleanBuffer() const {
|
|
if (m_buff.m_voidBuff != 0) {
|
|
switch (m_buffType) {
|
|
case BT_STR:
|
|
CC_DELETE_T(m_buff.m_strBuff, String);
|
|
break;
|
|
case BT_WSTR:
|
|
CC_DELETE_T(m_buff.m_wstrBuff, WString);
|
|
break;
|
|
case BT_NONE: // under the worse of circumstances, this is all we can do, and hope it works out
|
|
default:
|
|
//delete mBuffer.mVoidBuffer;
|
|
// delete void* is undefined, don't do that
|
|
CCASSERT(0, "This should never happen - mVoidBuffer should never contain something if we "
|
|
"don't know the type");
|
|
break;
|
|
}
|
|
m_buff.m_voidBuff = 0;
|
|
m_buffSize = 0;
|
|
m_buffType = BT_NONE;
|
|
}
|
|
}
|
|
|
|
void UTFString::_getBufferStr() const {
|
|
if (m_buffType != BT_STR) {
|
|
_cleanBuffer();
|
|
m_buff.m_strBuff = CC_NEW_T(String);
|
|
m_buffType = BT_STR;
|
|
}
|
|
m_buff.m_strBuff->clear();
|
|
}
|
|
|
|
void UTFString::_getBufferWStr() const {
|
|
if (m_buffType != BT_WSTR) {
|
|
_cleanBuffer();
|
|
m_buff.m_wstrBuff = CC_NEW_T(WString);
|
|
m_buffType = BT_WSTR;
|
|
}
|
|
m_buff.m_wstrBuff->clear();
|
|
}
|
|
|
|
void UTFString::_load_buffer_UTF8() const {
|
|
_getBufferStr();
|
|
String &buffer = (*m_buff.m_strBuff);
|
|
buffer.reserve(length());
|
|
|
|
unsigned char utf8buf[6] = "";
|
|
char *charbuf = (char *)utf8buf;
|
|
unicode_char c;
|
|
size_t len;
|
|
|
|
const_iterator i, ie = end();
|
|
for (i = begin(); i != ie; i.moveNext()) {
|
|
c = i.getCharacter();
|
|
len = _utf32_to_utf8(c, utf8buf);
|
|
size_t j = 0;
|
|
while (j < len)
|
|
buffer.push_back(charbuf[j++]);
|
|
}
|
|
}
|
|
|
|
void UTFString::_load_buffer_WStr() const {
|
|
_getBufferWStr();
|
|
WString &buffer = (*m_buff.m_wstrBuff);
|
|
buffer.reserve(length()); // may over reserve, but should be close enough
|
|
#ifdef WCHAR_UTF16 // wchar_t matches UTF-16
|
|
const_iterator i, ie = end();
|
|
for (i = begin(); i != ie; ++i) {
|
|
buffer.push_back((wchar_t)(*i));
|
|
}
|
|
#else // wchar_t fits UTF-32
|
|
unicode_char c;
|
|
const_iterator i, ie = end();
|
|
for (i = begin(); i != ie; i.moveNext()) {
|
|
c = i.getCharacter();
|
|
buffer.push_back((wchar_t)c);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
} // namespace cc
|