#include "CoreStd.h" #include "UTFString.h" NS_CC_BEGIN UTFString::unicode_char UTFString::_base_iterator::_getCharacter() const { return string_->getChar(_get_index()); } int UTFString::_base_iterator::_setCharacter(unicode_char uc) { size_type current_index = _get_index(); int change = string_->setChar( current_index, uc ); _jump_to( current_index ); return change; } void UTFString::_base_iterator::_moveNext() { _seekFwd( 1 ); // move 1 code point forward if ( _test_end() ) return; // exit if we hit the end if ( _utf16_surrogate_follow( iter_[0] ) ) { // landing on a follow code point means we might be part of a bigger character // so we test for that code_point lead_half = 0; //NB: we can't possibly be at the beginning here, so no need to test lead_half = iter_[-1]; // check the previous code point to see if we're part of a surrogate pair if ( _utf16_surrogate_lead( lead_half ) ) { _seekFwd( 1 ); // if so, then advance 1 more code point } } } void UTFString::_base_iterator::_movePrev() { _seekRev( 1 ); // move 1 code point backwards if ( _test_begin() ) return; // exit if we hit the beginning if ( _utf16_surrogate_follow( iter_[0] ) ) { // landing on a follow code point means we might be part of a bigger character // so we test for that code_point lead_half = 0; lead_half = iter_[-1]; // check the previous character to see if we're part of a surrogate pair if ( _utf16_surrogate_lead( lead_half ) ) { _seekRev( 1 ); // if so, then rewind 1 more code point } } } ////////////////////////////////////////////////////////////////////////// UTFString::unicode_char UTFString::getChar( size_type loc ) const { const code_point* ptr = c_str(); unicode_char uc; size_t l = _utf16_char_length( ptr[loc] ); code_point cp[2] = { /* blame the code beautifier */ 0, 0 }; cp[0] = ptr[loc]; if ( l == 2 && ( loc + 1 ) < m_data.length() ) { cp[1] = ptr[loc+1]; } _utf16_to_utf32( cp, uc ); return uc; } int UTFString::setChar( size_type loc, unicode_char ch ) { code_point cp[2] = { /* blame the code beautifier */ 0, 0 }; size_t l = _utf32_to_utf16( ch, cp ); unicode_char existingChar = getChar( loc ); size_t existingSize = _utf16_char_length( existingChar ); size_t newSize = _utf16_char_length( ch ); if ( newSize > existingSize ) { at( loc ) = cp[0]; insert( loc + 1, 1, cp[1] ); return 1; } if ( newSize < existingSize ) { erase( loc, 1 ); at( loc ) = cp[0]; return -1; } // newSize == existingSize at( loc ) = cp[0]; if ( l == 2 ) at( loc + 1 ) = cp[1]; return 0; } UTFString& UTFString::assign( const String& str ) { size_type len = _verifyUTF8( str ); clear(); // empty our contents, if there are any if (len == 0) { return *this; } reserve( len ); // best guess bulk capacity growth // This is a 3 step process, converting each byte in the UTF-8 stream to UTF-32, // then converting it to UTF-16, then finally appending the data buffer unicode_char uc = 0; // temporary Unicode character buffer unsigned char utf8buf[7]; // temporary UTF-8 buffer utf8buf[6] = 0; size_t utf8len; // UTF-8 length code_point utf16buff[3]; // temporary UTF-16 buffer utf16buff[2] = 0; size_t utf16len; // UTF-16 length String::const_iterator i, ie = str.end(); for ( i = str.begin(); i != ie; i++ ) { utf8len = _utf8_char_length( static_cast( *i ) ); // estimate bytes to load for ( size_t j = 0; j < utf8len; j++ ) { // load the needed UTF-8 bytes utf8buf[j] = ( static_cast( *( i + j ) ) ); // we don't increment 'i' here just in case the estimate is wrong (shouldn't happen, but we're being careful) } utf8buf[utf8len] = 0; // nul terminate so we throw an exception before running off the end of the buffer utf8len = _utf8_to_utf32( utf8buf, uc ); // do the UTF-8 -> UTF-32 conversion i += utf8len - 1; // we subtract 1 for the increment of the 'for' loop utf16len = _utf32_to_utf16( uc, utf16buff ); // UTF-32 -> UTF-16 conversion append( utf16buff, utf16len ); // append the characters to the string } return *this; } UTFString& UTFString::assign( const WString& wstr ) { m_data.clear(); m_data.reserve( wstr.length() ); // best guess bulk allocate #ifdef WCHAR_UTF16 // if we're already working in UTF-16, this is easy code_point tmp; WString::const_iterator i, ie = wstr.end(); for ( i = wstr.begin(); i != ie; i++ ) { tmp = static_cast( *i ); m_data.push_back( tmp ); } #else // otherwise we do it the safe way (which is still 100% safe to pass UTF-16 through, just slower) code_point cp[3] = {0, 0, 0}; unicode_char tmp; WString::const_iterator i, ie = wstr.end(); for ( i = wstr.begin(); i != ie; i++ ) { tmp = static_cast( *i ); size_t l = _utf32_to_utf16( tmp, cp ); if ( l > 0 ) m_data.push_back( cp[0] ); if ( l > 1 ) m_data.push_back( cp[1] ); } #endif return *this; } UTFString& UTFString::append( size_type num, unicode_char ch ) { code_point cp[2] = {0, 0}; if ( _utf32_to_utf16( ch, cp ) == 2 ) { for ( size_type i = 0; i < num; i++ ) { append( 1, cp[0] ); append( 1, cp[1] ); } } else { for ( size_type i = 0; i < num; i++ ) { append( 1, cp[0] ); } } return *this; } UTFString& UTFString::insert( size_type index, size_type num, unicode_char ch ) { code_point cp[3] = {0, 0, 0}; size_t l = _utf32_to_utf16( ch, cp ); if ( l == 1 ) { return insert( index, num, cp[0] ); } for ( size_type c = 0; c < num; c++ ) { // insert in reverse order to preserve ordering after insert insert( index, 1, cp[1] ); insert( index, 1, cp[0] ); } return *this; } void UTFString::insert( iterator i, size_type num, const unicode_char& ch ) { code_point cp[3] = {0, 0, 0}; size_t l = _utf32_to_utf16( ch, cp ); if ( l == 1 ) { insert( i, num, cp[0] ); } else { for ( size_type c = 0; c < num; c++ ) { // insert in reverse order to preserve ordering after insert insert( i, 1, cp[1] ); insert( i, 1, cp[0] ); } } } UTFString::size_type UTFString::find_first_of( const UTFString &str, size_type index /*= 0*/, size_type num /*= npos */ ) const { size_type i = 0; const size_type len = length(); while ( i < num && ( index + i ) < len ) { unicode_char ch = getChar( index + i ); if ( str.inString( ch ) ) return index + i; i += _utf16_char_length( ch ); // increment by the Unicode character length } return npos; } UTFString::size_type UTFString::find_first_not_of( const UTFString& str, size_type index /*= 0*/, size_type num /*= npos */ ) const { size_type i = 0; const size_type len = length(); while ( i < num && ( index + i ) < len ) { unicode_char ch = getChar( index + i ); if ( !str.inString( ch ) ) return index + i; i += _utf16_char_length( ch ); // increment by the Unicode character length } return npos; } UTFString::size_type UTFString::find_last_of( const UTFString& str, size_type index /*= npos*/, size_type num /*= npos */ ) const { size_type i = 0; const size_type len = length(); if ( index > len ) index = len - 1; while ( i < num && ( index - i ) != npos ) { size_type j = index - i; // careful to step full Unicode characters if ( j != 0 && _utf16_surrogate_follow( at( j ) ) && _utf16_surrogate_lead( at( j - 1 ) ) ) { j = index - ++i; } // and back to the usual dull test unicode_char ch = getChar( j ); if ( str.inString( ch ) ) return j; i++; } return npos; } UTFString::size_type UTFString::find_last_not_of( const UTFString& str, size_type index /*= npos*/, size_type num /*= npos */ ) const { size_type i = 0; const size_type len = length(); if ( index > len ) index = len - 1; while ( i < num && ( index - i ) != npos ) { size_type j = index - i; // careful to step full Unicode characters if ( j != 0 && _utf16_surrogate_follow( at( j ) ) && _utf16_surrogate_lead( at( j - 1 ) ) ) { j = index - ++i; } // and back to the usual dull test unicode_char ch = getChar( j ); if ( !str.inString( ch ) ) return j; i++; } return npos; } size_t UTFString::_utf16_to_utf32( const code_point in_cp[2], unicode_char& out_uc ) { const code_point& cp1 = in_cp[0]; const code_point& cp2 = in_cp[1]; bool wordPair = false; // does it look like a surrogate pair? if ( 0xD800 <= cp1 && cp1 <= 0xDBFF ) { // looks like one, but does the other half match the algorithm as well? if ( 0xDC00 <= cp2 && cp2 <= 0xDFFF ) wordPair = true; // yep! } if ( !wordPair ) { // if we aren't a 100% authentic surrogate pair, then just copy the value out_uc = cp1; return 1; } unsigned short cU = cp1, cL = cp2; // copy upper and lower words of surrogate pair to writable buffers cU -= 0xD800; // remove the encoding markers cL -= 0xDC00; out_uc = ( cU & 0x03FF ) << 10; // grab the 10 upper bits and set them in their proper location out_uc |= ( cL & 0x03FF ); // combine in the lower 10 bits out_uc += 0x10000; // add back in the value offset return 2; // this whole operation takes to words, so that's what we'll return } size_t UTFString::_utf32_to_utf16( const unicode_char& in_uc, code_point out_cp[2] ) { if ( in_uc <= 0xFFFF ) { // we blindly preserve sentinel values because our decoder understands them out_cp[0] = static_cast(in_uc); return 1; } unicode_char uc = in_uc; // copy to writable buffer unsigned short tmp; // single code point buffer uc -= 0x10000; // subtract value offset //process upper word tmp = static_cast(( uc >> 10 ) & 0x03FF); // grab the upper 10 bits tmp += 0xD800; // add encoding offset out_cp[0] = tmp; // write // process lower word tmp = static_cast(uc & 0x03FF); // grab the lower 10 bits tmp += 0xDC00; // add encoding offset out_cp[1] = tmp; // write return 2; // return used word count (2 for surrogate pairs) } size_t UTFString::_utf8_char_length( unsigned char cp ) { if ( !( cp & 0x80 ) ) return 1; if (( cp & ~_lead1_mask ) == _lead1 ) return 2; if (( cp & ~_lead2_mask ) == _lead2 ) return 3; if (( cp & ~_lead3_mask ) == _lead3 ) return 4; if (( cp & ~_lead4_mask ) == _lead4 ) return 5; if (( cp & ~_lead5_mask ) == _lead5 ) return 6; CCASSERT(0, "invalid UTF-8 sequence header value"); return 1; } size_t UTFString::_utf8_char_length( unicode_char uc ) { /* 7 bit: U-00000000 - U-0000007F: 0xxxxxxx 11 bit: U-00000080 - U-000007FF: 110xxxxx 10xxxxxx 16 bit: U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx 21 bit: U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 26 bit: U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 31 bit: U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */ if ( !( uc & ~0x0000007F ) ) return 1; if ( !( uc & ~0x000007FF ) ) return 2; if ( !( uc & ~0x0000FFFF ) ) return 3; if ( !( uc & ~0x001FFFFF ) ) return 4; if ( !( uc & ~0x03FFFFFF ) ) return 5; if ( !( uc & ~0x7FFFFFFF ) ) return 6; CCASSERT(0, "invalid UTF-32 value"); return 1; } size_t UTFString::_utf8_to_utf32( const unsigned char in_cp[6], unicode_char& out_uc ) { size_t len = _utf8_char_length( in_cp[0] ); if ( len == 1 ) { // if we are only 1 byte long, then just grab it and exit out_uc = in_cp[0]; return 1; } unicode_char c = 0; // temporary buffer size_t i = 0; switch ( len ) { // load header byte case 6: c = in_cp[i] & _lead5_mask; break; case 5: c = in_cp[i] & _lead4_mask; break; case 4: c = in_cp[i] & _lead3_mask; break; case 3: c = in_cp[i] & _lead2_mask; break; case 2: c = in_cp[i] & _lead1_mask; break; } for ( ++i; i < len; i++ ) { // load each continuation byte if ((in_cp[i] & ~_cont_mask) != _cont) { CCASSERT(0, "bad UTF-8 continuation byte"); break; } c <<= 6; c |= ( in_cp[i] & _cont_mask ); } out_uc = c; // write the final value and return the used byte length return len; } size_t UTFString::_utf32_to_utf8( const unicode_char& in_uc, unsigned char out_cp[6] ) { size_t len = _utf8_char_length( in_uc ); // predict byte length of sequence unicode_char c = in_uc; // copy to temp buffer //stuff all of the lower bits for ( size_t i = len - 1; i > 0; i-- ) { out_cp[i] = static_cast((( c ) & _cont_mask ) | _cont); c >>= 6; } //now write the header byte switch ( len ) { case 6: out_cp[0] = static_cast((( c ) & _lead5_mask ) | _lead5); break; case 5: out_cp[0] = static_cast((( c ) & _lead4_mask ) | _lead4); break; case 4: out_cp[0] = static_cast((( c ) & _lead3_mask ) | _lead3); break; case 3: out_cp[0] = static_cast((( c ) & _lead2_mask ) | _lead2); break; case 2: out_cp[0] = static_cast((( c ) & _lead1_mask ) | _lead1); break; case 1: default: out_cp[0] = static_cast(( c ) & 0x7F); break; } // return the byte length of the sequence return len; } UTFString::size_type UTFString::_verifyUTF8(const String& str) { String::const_iterator i, ie = str.end(); i = str.begin(); size_type length = 0; while ( i != ie ) { // characters pass until we find an extended sequence if (( *i ) & 0x80 ) { unsigned char c = ( *i ); size_t contBytes = 0; // get continuation byte count and test for overlong sequences if (( c & ~_lead1_mask ) == _lead1 ) { // 1 additional byte if (c == _lead1) { CCASSERT(0, "overlong UTF-8 sequence"); return 0; } contBytes = 1; } else if (( c & ~_lead2_mask ) == _lead2 ) { // 2 additional bytes contBytes = 2; if ( c == _lead2 ) { // possible overlong UTF-8 sequence c = ( *( i + 1 ) ); // look ahead to next byte in sequence if (( c & _lead2 ) == _cont ) { CCASSERT(0, "overlong UTF-8 sequence"); return 0; } } } else if (( c & ~_lead3_mask ) == _lead3 ) { // 3 additional bytes contBytes = 3; if ( c == _lead3 ) { // possible overlong UTF-8 sequence c = ( *( i + 1 ) ); // look ahead to next byte in sequence if (( c & _lead3 ) == _cont ) { CCASSERT(0, "overlong UTF-8 sequence"); return 0; } } } else if (( c & ~_lead4_mask ) == _lead4 ) { // 4 additional bytes contBytes = 4; if ( c == _lead4 ) { // possible overlong UTF-8 sequence c = ( *( i + 1 ) ); // look ahead to next byte in sequence if (( c & _lead4 ) == _cont ) { CCASSERT(0, "overlong UTF-8 sequence"); return 0; } } } else if (( c & ~_lead5_mask ) == _lead5 ) { // 5 additional bytes contBytes = 5; if ( c == _lead5 ) { // possible overlong UTF-8 sequence c = ( *( i + 1 ) ); // look ahead to next byte in sequence if (( c & _lead5 ) == _cont ) { CCASSERT(0, "overlong UTF-8 sequence"); return 0; } } } // check remaining continuation bytes for while ( contBytes-- ) { c = ( *( ++i ) ); // get next byte in sequence if (( c & ~_cont_mask ) != _cont ) { CCASSERT(0, "bad UTF-8 continuation byte"); return 0; } } } length++; i++; } return length; } void UTFString::_cleanBuffer() const { if ( m_buff.m_voidBuff != 0 ) { switch ( m_buffType ) { case BT_STR: CC_DELETE_T(m_buff.m_strBuff, String); break; case BT_WSTR: CC_DELETE_T(m_buff.m_wstrBuff, WString); break; case BT_NONE: // under the worse of circumstances, this is all we can do, and hope it works out default: //delete mBuffer.mVoidBuffer; // delete void* is undefined, don't do that CCASSERT(0, "This should never happen - mVoidBuffer should never contain something if we " "don't know the type"); break; } m_buff.m_voidBuff = 0; m_buffSize = 0; m_buffType = BT_NONE; } } void UTFString::_getBufferStr() const { if ( m_buffType != BT_STR ) { _cleanBuffer(); m_buff.m_strBuff = CC_NEW_T(String); m_buffType = BT_STR; } m_buff.m_strBuff->clear(); } void UTFString::_getBufferWStr() const { if ( m_buffType != BT_WSTR ) { _cleanBuffer(); m_buff.m_wstrBuff = CC_NEW_T(WString); m_buffType = BT_WSTR; } m_buff.m_wstrBuff->clear(); } void UTFString::_load_buffer_UTF8() const { _getBufferStr(); String& buffer = ( *m_buff.m_strBuff ); buffer.reserve( length() ); unsigned char utf8buf[6] = ""; char* charbuf = ( char* )utf8buf; unicode_char c; size_t len; const_iterator i, ie = end(); for ( i = begin(); i != ie; i.moveNext() ) { c = i.getCharacter(); len = _utf32_to_utf8( c, utf8buf ); size_t j = 0; while ( j < len ) buffer.push_back( charbuf[j++] ); } } void UTFString::_load_buffer_WStr() const { _getBufferWStr(); WString& buffer = ( *m_buff.m_wstrBuff ); buffer.reserve( length() ); // may over reserve, but should be close enough #ifdef WCHAR_UTF16 // wchar_t matches UTF-16 const_iterator i, ie = end(); for ( i = begin(); i != ie; ++i ) { buffer.push_back(( wchar_t )( *i ) ); } #else // wchar_t fits UTF-32 unicode_char c; const_iterator i, ie = end(); for ( i = begin(); i != ie; i.moveNext() ) { c = i.getCharacter(); buffer.push_back(( wchar_t )c ); } #endif } NS_CC_END