Files
cocos4/native/cocos/renderer/core/kernel/UTFString.cc
T
2019-10-17 14:50:54 +08:00

592 lines
18 KiB
C++
Executable File

#include "CoreStd.h"
#include "UTFString.h"
CC_NAMESPACE_BEGIN
UTFString::unicode_char UTFString::_base_iterator::_getCharacter() const {
return string_->getChar(_get_index());
}
int UTFString::_base_iterator::_setCharacter(unicode_char uc) {
size_type current_index = _get_index();
int change = string_->setChar( current_index, uc );
_jump_to( current_index );
return change;
}
void UTFString::_base_iterator::_moveNext() {
_seekFwd( 1 ); // move 1 code point forward
if ( _test_end() ) return; // exit if we hit the end
if ( _utf16_surrogate_follow( iter_[0] ) ) {
// landing on a follow code point means we might be part of a bigger character
// so we test for that
code_point lead_half = 0;
//NB: we can't possibly be at the beginning here, so no need to test
lead_half = iter_[-1]; // check the previous code point to see if we're part of a surrogate pair
if ( _utf16_surrogate_lead( lead_half ) ) {
_seekFwd( 1 ); // if so, then advance 1 more code point
}
}
}
void UTFString::_base_iterator::_movePrev() {
_seekRev( 1 ); // move 1 code point backwards
if ( _test_begin() ) return; // exit if we hit the beginning
if ( _utf16_surrogate_follow( iter_[0] ) ) {
// landing on a follow code point means we might be part of a bigger character
// so we test for that
code_point lead_half = 0;
lead_half = iter_[-1]; // check the previous character to see if we're part of a surrogate pair
if ( _utf16_surrogate_lead( lead_half ) ) {
_seekRev( 1 ); // if so, then rewind 1 more code point
}
}
}
//////////////////////////////////////////////////////////////////////////
UTFString::unicode_char UTFString::getChar( size_type loc ) const {
const code_point* ptr = c_str();
unicode_char uc;
size_t l = _utf16_char_length( ptr[loc] );
code_point cp[2] = { /* blame the code beautifier */
0, 0
};
cp[0] = ptr[loc];
if ( l == 2 && ( loc + 1 ) < m_data.length() ) {
cp[1] = ptr[loc+1];
}
_utf16_to_utf32( cp, uc );
return uc;
}
int UTFString::setChar( size_type loc, unicode_char ch ) {
code_point cp[2] = { /* blame the code beautifier */
0, 0
};
size_t l = _utf32_to_utf16( ch, cp );
unicode_char existingChar = getChar( loc );
size_t existingSize = _utf16_char_length( existingChar );
size_t newSize = _utf16_char_length( ch );
if ( newSize > existingSize ) {
at( loc ) = cp[0];
insert( loc + 1, 1, cp[1] );
return 1;
}
if ( newSize < existingSize ) {
erase( loc, 1 );
at( loc ) = cp[0];
return -1;
}
// newSize == existingSize
at( loc ) = cp[0];
if ( l == 2 ) at( loc + 1 ) = cp[1];
return 0;
}
UTFString& UTFString::assign( const String& str ) {
size_type len = _verifyUTF8( str );
clear(); // empty our contents, if there are any
if (len == 0)
{
return *this;
}
reserve( len ); // best guess bulk capacity growth
// This is a 3 step process, converting each byte in the UTF-8 stream to UTF-32,
// then converting it to UTF-16, then finally appending the data buffer
unicode_char uc = 0; // temporary Unicode character buffer
unsigned char utf8buf[7]; // temporary UTF-8 buffer
utf8buf[6] = 0;
size_t utf8len; // UTF-8 length
code_point utf16buff[3]; // temporary UTF-16 buffer
utf16buff[2] = 0;
size_t utf16len; // UTF-16 length
String::const_iterator i, ie = str.end();
for ( i = str.begin(); i != ie; i++ ) {
utf8len = _utf8_char_length( static_cast<unsigned char>( *i ) ); // estimate bytes to load
for ( size_t j = 0; j < utf8len; j++ ) { // load the needed UTF-8 bytes
utf8buf[j] = ( static_cast<unsigned char>( *( i + j ) ) ); // we don't increment 'i' here just in case the estimate is wrong (shouldn't happen, but we're being careful)
}
utf8buf[utf8len] = 0; // nul terminate so we throw an exception before running off the end of the buffer
utf8len = _utf8_to_utf32( utf8buf, uc ); // do the UTF-8 -> UTF-32 conversion
i += utf8len - 1; // we subtract 1 for the increment of the 'for' loop
utf16len = _utf32_to_utf16( uc, utf16buff ); // UTF-32 -> UTF-16 conversion
append( utf16buff, utf16len ); // append the characters to the string
}
return *this;
}
UTFString& UTFString::assign( const WString& wstr )
{
m_data.clear();
m_data.reserve( wstr.length() ); // best guess bulk allocate
#ifdef WCHAR_UTF16 // if we're already working in UTF-16, this is easy
code_point tmp;
WString::const_iterator i, ie = wstr.end();
for ( i = wstr.begin(); i != ie; i++ ) {
tmp = static_cast<code_point>( *i );
m_data.push_back( tmp );
}
#else // otherwise we do it the safe way (which is still 100% safe to pass UTF-16 through, just slower)
code_point cp[3] = {0, 0, 0};
unicode_char tmp;
WString::const_iterator i, ie = wstr.end();
for ( i = wstr.begin(); i != ie; i++ ) {
tmp = static_cast<unicode_char>( *i );
size_t l = _utf32_to_utf16( tmp, cp );
if ( l > 0 ) m_data.push_back( cp[0] );
if ( l > 1 ) m_data.push_back( cp[1] );
}
#endif
return *this;
}
UTFString& UTFString::append( size_type num, unicode_char ch )
{
code_point cp[2] = {0, 0};
if ( _utf32_to_utf16( ch, cp ) == 2 ) {
for ( size_type i = 0; i < num; i++ ) {
append( 1, cp[0] );
append( 1, cp[1] );
}
} else {
for ( size_type i = 0; i < num; i++ ) {
append( 1, cp[0] );
}
}
return *this;
}
UTFString& UTFString::insert( size_type index, size_type num, unicode_char ch )
{
code_point cp[3] = {0, 0, 0};
size_t l = _utf32_to_utf16( ch, cp );
if ( l == 1 ) {
return insert( index, num, cp[0] );
}
for ( size_type c = 0; c < num; c++ ) {
// insert in reverse order to preserve ordering after insert
insert( index, 1, cp[1] );
insert( index, 1, cp[0] );
}
return *this;
}
void UTFString::insert( iterator i, size_type num, const unicode_char& ch )
{
code_point cp[3] = {0, 0, 0};
size_t l = _utf32_to_utf16( ch, cp );
if ( l == 1 ) {
insert( i, num, cp[0] );
} else {
for ( size_type c = 0; c < num; c++ ) {
// insert in reverse order to preserve ordering after insert
insert( i, 1, cp[1] );
insert( i, 1, cp[0] );
}
}
}
UTFString::size_type UTFString::find_first_of( const UTFString &str, size_type index /*= 0*/, size_type num /*= npos */ ) const
{
size_type i = 0;
const size_type len = length();
while ( i < num && ( index + i ) < len ) {
unicode_char ch = getChar( index + i );
if ( str.inString( ch ) )
return index + i;
i += _utf16_char_length( ch ); // increment by the Unicode character length
}
return npos;
}
UTFString::size_type UTFString::find_first_not_of( const UTFString& str, size_type index /*= 0*/, size_type num /*= npos */ ) const
{
size_type i = 0;
const size_type len = length();
while ( i < num && ( index + i ) < len ) {
unicode_char ch = getChar( index + i );
if ( !str.inString( ch ) )
return index + i;
i += _utf16_char_length( ch ); // increment by the Unicode character length
}
return npos;
}
UTFString::size_type UTFString::find_last_of( const UTFString& str, size_type index /*= npos*/, size_type num /*= npos */ ) const
{
size_type i = 0;
const size_type len = length();
if ( index > len ) index = len - 1;
while ( i < num && ( index - i ) != npos ) {
size_type j = index - i;
// careful to step full Unicode characters
if ( j != 0 && _utf16_surrogate_follow( at( j ) ) && _utf16_surrogate_lead( at( j - 1 ) ) ) {
j = index - ++i;
}
// and back to the usual dull test
unicode_char ch = getChar( j );
if ( str.inString( ch ) )
return j;
i++;
}
return npos;
}
UTFString::size_type UTFString::find_last_not_of( const UTFString& str, size_type index /*= npos*/, size_type num /*= npos */ ) const
{
size_type i = 0;
const size_type len = length();
if ( index > len ) index = len - 1;
while ( i < num && ( index - i ) != npos ) {
size_type j = index - i;
// careful to step full Unicode characters
if ( j != 0 && _utf16_surrogate_follow( at( j ) ) && _utf16_surrogate_lead( at( j - 1 ) ) ) {
j = index - ++i;
}
// and back to the usual dull test
unicode_char ch = getChar( j );
if ( !str.inString( ch ) )
return j;
i++;
}
return npos;
}
size_t UTFString::_utf16_to_utf32( const code_point in_cp[2], unicode_char& out_uc )
{
const code_point& cp1 = in_cp[0];
const code_point& cp2 = in_cp[1];
bool wordPair = false;
// does it look like a surrogate pair?
if ( 0xD800 <= cp1 && cp1 <= 0xDBFF ) {
// looks like one, but does the other half match the algorithm as well?
if ( 0xDC00 <= cp2 && cp2 <= 0xDFFF )
wordPair = true; // yep!
}
if ( !wordPair ) { // if we aren't a 100% authentic surrogate pair, then just copy the value
out_uc = cp1;
return 1;
}
unsigned short cU = cp1, cL = cp2; // copy upper and lower words of surrogate pair to writable buffers
cU -= 0xD800; // remove the encoding markers
cL -= 0xDC00;
out_uc = ( cU & 0x03FF ) << 10; // grab the 10 upper bits and set them in their proper location
out_uc |= ( cL & 0x03FF ); // combine in the lower 10 bits
out_uc += 0x10000; // add back in the value offset
return 2; // this whole operation takes to words, so that's what we'll return
}
size_t UTFString::_utf32_to_utf16( const unicode_char& in_uc, code_point out_cp[2] )
{
if ( in_uc <= 0xFFFF ) { // we blindly preserve sentinel values because our decoder understands them
out_cp[0] = static_cast<code_point>(in_uc);
return 1;
}
unicode_char uc = in_uc; // copy to writable buffer
unsigned short tmp; // single code point buffer
uc -= 0x10000; // subtract value offset
//process upper word
tmp = static_cast<unsigned short>(( uc >> 10 ) & 0x03FF); // grab the upper 10 bits
tmp += 0xD800; // add encoding offset
out_cp[0] = tmp; // write
// process lower word
tmp = static_cast<unsigned short>(uc & 0x03FF); // grab the lower 10 bits
tmp += 0xDC00; // add encoding offset
out_cp[1] = tmp; // write
return 2; // return used word count (2 for surrogate pairs)
}
size_t UTFString::_utf8_char_length( unsigned char cp )
{
if ( !( cp & 0x80 ) ) return 1;
if (( cp & ~_lead1_mask ) == _lead1 ) return 2;
if (( cp & ~_lead2_mask ) == _lead2 ) return 3;
if (( cp & ~_lead3_mask ) == _lead3 ) return 4;
if (( cp & ~_lead4_mask ) == _lead4 ) return 5;
if (( cp & ~_lead5_mask ) == _lead5 ) return 6;
CC_ASSERTS(0, "invalid UTF-8 sequence header value");
return 1;
}
size_t UTFString::_utf8_char_length( unicode_char uc ) {
/*
7 bit: U-00000000 - U-0000007F: 0xxxxxxx
11 bit: U-00000080 - U-000007FF: 110xxxxx 10xxxxxx
16 bit: U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
21 bit: U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
26 bit: U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
31 bit: U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
*/
if ( !( uc & ~0x0000007F ) ) return 1;
if ( !( uc & ~0x000007FF ) ) return 2;
if ( !( uc & ~0x0000FFFF ) ) return 3;
if ( !( uc & ~0x001FFFFF ) ) return 4;
if ( !( uc & ~0x03FFFFFF ) ) return 5;
if ( !( uc & ~0x7FFFFFFF ) ) return 6;
CC_ASSERTS(0, "invalid UTF-32 value");
return 1;
}
size_t UTFString::_utf8_to_utf32( const unsigned char in_cp[6], unicode_char& out_uc )
{
size_t len = _utf8_char_length( in_cp[0] );
if ( len == 1 ) { // if we are only 1 byte long, then just grab it and exit
out_uc = in_cp[0];
return 1;
}
unicode_char c = 0; // temporary buffer
size_t i = 0;
switch ( len ) { // load header byte
case 6:
c = in_cp[i] & _lead5_mask;
break;
case 5:
c = in_cp[i] & _lead4_mask;
break;
case 4:
c = in_cp[i] & _lead3_mask;
break;
case 3:
c = in_cp[i] & _lead2_mask;
break;
case 2:
c = in_cp[i] & _lead1_mask;
break;
}
for ( ++i; i < len; i++ ) { // load each continuation byte
if ((in_cp[i] & ~_cont_mask) != _cont) {
CC_ASSERTS(0, "bad UTF-8 continuation byte");
break;
}
c <<= 6;
c |= ( in_cp[i] & _cont_mask );
}
out_uc = c; // write the final value and return the used byte length
return len;
}
size_t UTFString::_utf32_to_utf8( const unicode_char& in_uc, unsigned char out_cp[6] ) {
size_t len = _utf8_char_length( in_uc ); // predict byte length of sequence
unicode_char c = in_uc; // copy to temp buffer
//stuff all of the lower bits
for ( size_t i = len - 1; i > 0; i-- ) {
out_cp[i] = static_cast<unsigned char>((( c ) & _cont_mask ) | _cont);
c >>= 6;
}
//now write the header byte
switch ( len ) {
case 6:
out_cp[0] = static_cast<unsigned char>((( c ) & _lead5_mask ) | _lead5);
break;
case 5:
out_cp[0] = static_cast<unsigned char>((( c ) & _lead4_mask ) | _lead4);
break;
case 4:
out_cp[0] = static_cast<unsigned char>((( c ) & _lead3_mask ) | _lead3);
break;
case 3:
out_cp[0] = static_cast<unsigned char>((( c ) & _lead2_mask ) | _lead2);
break;
case 2:
out_cp[0] = static_cast<unsigned char>((( c ) & _lead1_mask ) | _lead1);
break;
case 1:
default:
out_cp[0] = static_cast<unsigned char>(( c ) & 0x7F);
break;
}
// return the byte length of the sequence
return len;
}
UTFString::size_type UTFString::_verifyUTF8(const String& str) {
String::const_iterator i, ie = str.end();
i = str.begin();
size_type length = 0;
while ( i != ie ) {
// characters pass until we find an extended sequence
if (( *i ) & 0x80 ) {
unsigned char c = ( *i );
size_t contBytes = 0;
// get continuation byte count and test for overlong sequences
if (( c & ~_lead1_mask ) == _lead1 ) { // 1 additional byte
if (c == _lead1)
{
CC_ASSERTS(0, "overlong UTF-8 sequence");
return 0;
}
contBytes = 1;
} else if (( c & ~_lead2_mask ) == _lead2 ) { // 2 additional bytes
contBytes = 2;
if ( c == _lead2 ) { // possible overlong UTF-8 sequence
c = ( *( i + 1 ) ); // look ahead to next byte in sequence
if (( c & _lead2 ) == _cont )
{
CC_ASSERTS(0, "overlong UTF-8 sequence");
return 0;
}
}
} else if (( c & ~_lead3_mask ) == _lead3 ) { // 3 additional bytes
contBytes = 3;
if ( c == _lead3 ) { // possible overlong UTF-8 sequence
c = ( *( i + 1 ) ); // look ahead to next byte in sequence
if (( c & _lead3 ) == _cont )
{
CC_ASSERTS(0, "overlong UTF-8 sequence");
return 0;
}
}
} else if (( c & ~_lead4_mask ) == _lead4 ) { // 4 additional bytes
contBytes = 4;
if ( c == _lead4 ) { // possible overlong UTF-8 sequence
c = ( *( i + 1 ) ); // look ahead to next byte in sequence
if (( c & _lead4 ) == _cont )
{
CC_ASSERTS(0, "overlong UTF-8 sequence");
return 0;
}
}
} else if (( c & ~_lead5_mask ) == _lead5 ) { // 5 additional bytes
contBytes = 5;
if ( c == _lead5 ) { // possible overlong UTF-8 sequence
c = ( *( i + 1 ) ); // look ahead to next byte in sequence
if (( c & _lead5 ) == _cont )
{
CC_ASSERTS(0, "overlong UTF-8 sequence");
return 0;
}
}
}
// check remaining continuation bytes for
while ( contBytes-- ) {
c = ( *( ++i ) ); // get next byte in sequence
if (( c & ~_cont_mask ) != _cont ) {
CC_ASSERTS(0, "bad UTF-8 continuation byte");
return 0;
}
}
}
length++;
i++;
}
return length;
}
void UTFString::_cleanBuffer() const {
if ( m_buff.m_voidBuff != 0 ) {
switch ( m_buffType ) {
case BT_STR:
CC_DELETE_T(m_buff.m_strBuff, String);
break;
case BT_WSTR:
CC_DELETE_T(m_buff.m_wstrBuff, WString);
break;
case BT_NONE: // under the worse of circumstances, this is all we can do, and hope it works out
default:
//delete mBuffer.mVoidBuffer;
// delete void* is undefined, don't do that
CC_ASSERTS(0, "This should never happen - mVoidBuffer should never contain something if we "
"don't know the type");
break;
}
m_buff.m_voidBuff = 0;
m_buffSize = 0;
m_buffType = BT_NONE;
}
}
void UTFString::_getBufferStr() const {
if ( m_buffType != BT_STR ) {
_cleanBuffer();
m_buff.m_strBuff = CC_NEW_T(String);
m_buffType = BT_STR;
}
m_buff.m_strBuff->clear();
}
void UTFString::_getBufferWStr() const {
if ( m_buffType != BT_WSTR ) {
_cleanBuffer();
m_buff.m_wstrBuff = CC_NEW_T(WString);
m_buffType = BT_WSTR;
}
m_buff.m_wstrBuff->clear();
}
void UTFString::_load_buffer_UTF8() const {
_getBufferStr();
String& buffer = ( *m_buff.m_strBuff );
buffer.reserve( length() );
unsigned char utf8buf[6] = "";
char* charbuf = ( char* )utf8buf;
unicode_char c;
size_t len;
const_iterator i, ie = end();
for ( i = begin(); i != ie; i.moveNext() )
{
c = i.getCharacter();
len = _utf32_to_utf8( c, utf8buf );
size_t j = 0;
while ( j < len )
buffer.push_back( charbuf[j++] );
}
}
void UTFString::_load_buffer_WStr() const {
_getBufferWStr();
WString& buffer = ( *m_buff.m_wstrBuff );
buffer.reserve( length() ); // may over reserve, but should be close enough
#ifdef WCHAR_UTF16 // wchar_t matches UTF-16
const_iterator i, ie = end();
for ( i = begin(); i != ie; ++i ) {
buffer.push_back(( wchar_t )( *i ) );
}
#else // wchar_t fits UTF-32
unicode_char c;
const_iterator i, ie = end();
for ( i = begin(); i != ie; i.moveNext() ) {
c = i.getCharacter();
buffer.push_back(( wchar_t )c );
}
#endif
}
CC_NAMESPACE_END