1091 lines
34 KiB
C++
1091 lines
34 KiB
C++
//---------------------------------------------------------------------------------------------------------------------------------
|
|
// File: localizationimpl.cpp
|
|
//
|
|
// Contents: Contains non-inline code for the SystemLocale class
|
|
// Must be included in one c/cpp file per binary
|
|
// A build error will occur if this inclusion policy is not followed
|
|
//
|
|
// Microsoft Drivers 5.10 for PHP for SQL Server
|
|
// Copyright(c) Microsoft Corporation
|
|
// All rights reserved.
|
|
// MIT License
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files(the ""Software""),
|
|
// to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions :
|
|
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
// IN THE SOFTWARE.
|
|
//---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
#include "localization.hpp"
|
|
|
|
#include "globalization.h"
|
|
#include "StringFunctions.h"
|
|
|
|
struct cp_iconv
|
|
{
|
|
UINT CodePage;
|
|
const char * IConvEncoding;
|
|
|
|
static const cp_iconv g_cp_iconv[];
|
|
static const size_t g_cp_iconv_count;
|
|
|
|
static int GetIndex( UINT codepage )
|
|
{
|
|
for ( size_t idx = 0; idx < g_cp_iconv_count; ++idx )
|
|
{
|
|
if ( g_cp_iconv[idx].CodePage == codepage )
|
|
return (int)idx;
|
|
}
|
|
// Should never be an unknown code page
|
|
assert( false );
|
|
return -1;
|
|
}
|
|
};
|
|
|
|
// Array of CodePage-to-IConvEncoding mappings
|
|
// First few elements are most commonly used
|
|
// CodePage 2 corresponds to binary. If the attribute PDO::SQLSRV_ENCODING_BINARY
|
|
// is set, GetIndex() above hits the assert(false) directive unless we include
|
|
// CodePage 2 below and assign an empty string to it.
|
|
#ifdef __MUSL__
|
|
#define TRANSLIT ""
|
|
#else
|
|
#define TRANSLIT "//TRANSLIT"
|
|
#endif
|
|
|
|
const cp_iconv cp_iconv::g_cp_iconv[] = {
|
|
{ 65001, "UTF-8" },
|
|
{ 1200, "UTF-16LE" },
|
|
{ 3, "UTF-8" },
|
|
{ 2, "" },
|
|
{ 1252, "CP1252" TRANSLIT },
|
|
{ 850, "CP850" TRANSLIT },
|
|
{ 437, "CP437" TRANSLIT },
|
|
{ 874, "CP874" TRANSLIT },
|
|
{ 932, "CP932" TRANSLIT },
|
|
{ 936, "CP936" TRANSLIT },
|
|
{ 949, "CP949" TRANSLIT },
|
|
{ 950, "CP950" TRANSLIT },
|
|
{ 1250, "CP1250" TRANSLIT },
|
|
{ 1251, "CP1251" TRANSLIT },
|
|
{ 1253, "CP1253" TRANSLIT },
|
|
{ 1254, "CP1254" TRANSLIT },
|
|
{ 1255, "CP1255" TRANSLIT },
|
|
{ 1256, "CP1256" TRANSLIT },
|
|
{ 1257, "CP1257" TRANSLIT },
|
|
{ 1258, "CP1258" TRANSLIT },
|
|
{ 54936, "GB18030" TRANSLIT},
|
|
{ CP_ISO8859_1, "ISO8859-1" TRANSLIT },
|
|
{ CP_ISO8859_2, "ISO8859-2" TRANSLIT },
|
|
{ CP_ISO8859_3, "ISO8859-3" TRANSLIT },
|
|
{ CP_ISO8859_4, "ISO8859-4" TRANSLIT },
|
|
{ CP_ISO8859_5, "ISO8859-5" TRANSLIT },
|
|
{ CP_ISO8859_6, "ISO8859-6" TRANSLIT },
|
|
{ CP_ISO8859_7, "ISO8859-7" TRANSLIT },
|
|
{ CP_ISO8859_8, "ISO8859-8" TRANSLIT },
|
|
{ CP_ISO8859_9, "ISO8859-9" TRANSLIT },
|
|
{ CP_ISO8859_13, "ISO8859-13" TRANSLIT },
|
|
{ CP_ISO8859_15, "ISO8859-15" TRANSLIT },
|
|
{ 12000, "UTF-32LE" }
|
|
};
|
|
const size_t cp_iconv::g_cp_iconv_count = ARRAYSIZE(cp_iconv::g_cp_iconv);
|
|
|
|
class IConvCachePool
|
|
{
|
|
SLIST_HEADER m_Pool[cp_iconv::g_cp_iconv_count][cp_iconv::g_cp_iconv_count];
|
|
|
|
IConvCachePool( const IConvCachePool & );
|
|
IConvCachePool & operator=( const IConvCachePool & );
|
|
|
|
// This bool indicates that the iconv pool is no longer available.
|
|
// For the driver,lis flag indicates the pool can no longer be used.
|
|
// Global destructors will be called by a single thread so this flag does not
|
|
// need thread synch protection.
|
|
static bool s_PoolDestroyed;
|
|
|
|
IConvCachePool()
|
|
{
|
|
for ( int dstIdx = 0; dstIdx < cp_iconv::g_cp_iconv_count; ++dstIdx )
|
|
{
|
|
for ( int srcIdx = 0; srcIdx < cp_iconv::g_cp_iconv_count; ++srcIdx )
|
|
{
|
|
InitializeSListHead( &m_Pool[dstIdx][srcIdx] );
|
|
}
|
|
}
|
|
}
|
|
|
|
~IConvCachePool()
|
|
{
|
|
IConvCachePool::s_PoolDestroyed = true;
|
|
|
|
// Clean up remaining nodes
|
|
for ( int dstIdx = 0; dstIdx < cp_iconv::g_cp_iconv_count; ++dstIdx )
|
|
{
|
|
for ( int srcIdx = 0; srcIdx < cp_iconv::g_cp_iconv_count; ++srcIdx )
|
|
{
|
|
IConvCache * pNode = static_cast<IConvCache*>( InterlockedFlushSList(&m_Pool[dstIdx][srcIdx]) );
|
|
while ( NULL != pNode )
|
|
{
|
|
IConvCache * pNext = static_cast<IConvCache*>( pNode->Next );
|
|
delete pNode;
|
|
pNode = pNext;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
USHORT Depth( int dstIdx, int srcIdx )
|
|
{
|
|
assert( 0 <= dstIdx && dstIdx < cp_iconv::g_cp_iconv_count );
|
|
assert( 0 <= srcIdx && srcIdx < cp_iconv::g_cp_iconv_count );
|
|
return QueryDepthSList( &m_Pool[dstIdx][srcIdx] );
|
|
}
|
|
|
|
// If this returns NULL, then caller must allocate their own iconv_t.
|
|
// It will return NULL if allocation for a new instance failed (out of memory).
|
|
const IConvCache * Borrow( int dstIdx, int srcIdx )
|
|
{
|
|
assert( 0 <= dstIdx && dstIdx < cp_iconv::g_cp_iconv_count );
|
|
assert( 0 <= srcIdx && srcIdx < cp_iconv::g_cp_iconv_count );
|
|
|
|
const IConvCache * pCache = static_cast<const IConvCache*>( InterlockedPopEntrySList(&m_Pool[dstIdx][srcIdx]) );
|
|
if ( NULL == pCache )
|
|
{
|
|
const IConvCache * pNewCache = new IConvCache( dstIdx, srcIdx );
|
|
if ( NULL != pNewCache )
|
|
{
|
|
if ( INVALID_ICONV != pNewCache->GetIConv() )
|
|
pCache = pNewCache;
|
|
else
|
|
delete pNewCache;
|
|
}
|
|
}
|
|
return pCache;
|
|
}
|
|
|
|
void Return( const IConvCache * pCache, int dstIdx, int srcIdx )
|
|
{
|
|
assert( pCache );
|
|
assert( 0 <= dstIdx && dstIdx < cp_iconv::g_cp_iconv_count );
|
|
assert( 0 <= srcIdx && srcIdx < cp_iconv::g_cp_iconv_count );
|
|
|
|
// Setting an arbitrary limit to prevent unbounded memory use by the pool.
|
|
// Want this to be large enough for a substantial number of concurrent threads.
|
|
const USHORT MAX_POOL_SIZE = 1024;
|
|
|
|
if ( INVALID_ICONV != pCache->GetIConv() && Depth(dstIdx, srcIdx) < MAX_POOL_SIZE )
|
|
{
|
|
SLIST_ENTRY * pNode = const_cast<IConvCache*>( pCache );
|
|
InterlockedPushEntrySList( &m_Pool[dstIdx][srcIdx], pNode );
|
|
}
|
|
else
|
|
{
|
|
delete pCache;
|
|
}
|
|
}
|
|
|
|
static IConvCachePool & Singleton()
|
|
{
|
|
// GCC ensures that function scoped static initializers are threadsafe
|
|
// We must not use the -fno-threadsafe-statics compiler option
|
|
#if !defined(__GNUC__) || defined(NO_THREADSAFE_STATICS)
|
|
#error "Relying on GCC's threadsafe initialization of local statics."
|
|
#endif
|
|
static IConvCachePool s_Pool;
|
|
return s_Pool;
|
|
}
|
|
|
|
public:
|
|
static const IConvCache * BorrowCache( UINT dstCP, UINT srcCP )
|
|
{
|
|
int dstIdx = cp_iconv::GetIndex(dstCP);
|
|
int srcIdx = cp_iconv::GetIndex(srcCP);
|
|
|
|
if ( -1 == dstIdx || -1 == srcIdx )
|
|
return NULL;
|
|
else if ( !s_PoolDestroyed )
|
|
return Singleton().Borrow( dstIdx, srcIdx );
|
|
else
|
|
return new IConvCache( dstIdx, srcIdx );
|
|
}
|
|
|
|
static void ReturnCache( const IConvCache * pCache, UINT dstCP, UINT srcCP )
|
|
{
|
|
int dstIdx = cp_iconv::GetIndex(dstCP);
|
|
int srcIdx = cp_iconv::GetIndex(srcCP);
|
|
|
|
if ( -1 != dstIdx && -1 != srcIdx && !s_PoolDestroyed )
|
|
Singleton().Return( pCache, dstIdx, srcIdx );
|
|
else
|
|
delete pCache;
|
|
}
|
|
|
|
static USHORT Depth( UINT dstCP, UINT srcCP )
|
|
{
|
|
if ( IConvCachePool::s_PoolDestroyed )
|
|
return 0;
|
|
else
|
|
{
|
|
int dstIdx = cp_iconv::GetIndex(dstCP);
|
|
int srcIdx = cp_iconv::GetIndex(srcCP);
|
|
|
|
if ( -1 == dstIdx || -1 == srcIdx )
|
|
return 0;
|
|
else
|
|
return Singleton().Depth( dstIdx, srcIdx );
|
|
}
|
|
}
|
|
};
|
|
|
|
|
|
bool IConvCachePool::s_PoolDestroyed = false;
|
|
|
|
IConvCache::IConvCache( int dstIdx, int srcIdx )
|
|
: m_iconv( iconv_open(
|
|
cp_iconv::g_cp_iconv[dstIdx].IConvEncoding,
|
|
cp_iconv::g_cp_iconv[srcIdx].IConvEncoding) )
|
|
{
|
|
}
|
|
|
|
IConvCache::~IConvCache()
|
|
{
|
|
if ( INVALID_ICONV != m_iconv )
|
|
iconv_close( m_iconv );
|
|
}
|
|
|
|
|
|
EncodingConverter::EncodingConverter( UINT dstCodePage, UINT srcCodePage )
|
|
: m_dstCodePage( dstCodePage ),
|
|
m_srcCodePage( srcCodePage ),
|
|
m_pCvtCache( NULL )
|
|
{
|
|
}
|
|
|
|
EncodingConverter::~EncodingConverter()
|
|
{
|
|
if ( NULL != m_pCvtCache )
|
|
{
|
|
IConvCachePool::ReturnCache( m_pCvtCache, m_dstCodePage, m_srcCodePage );
|
|
}
|
|
}
|
|
|
|
bool EncodingConverter::Initialize()
|
|
{
|
|
if ( !IsValidIConv() )
|
|
{
|
|
m_pCvtCache = IConvCachePool::BorrowCache( m_dstCodePage, m_srcCodePage );
|
|
}
|
|
return IsValidIConv();
|
|
}
|
|
|
|
|
|
#include <locale>
|
|
|
|
using namespace std;
|
|
|
|
#ifndef _countof
|
|
#define _countof(obj) (sizeof(obj)/sizeof(obj[0]))
|
|
#endif
|
|
|
|
const char* DEFAULT_LOCALES[] = {"en_US.UTF-8", "C"};
|
|
|
|
bool _setLocale(const char * localeName, std::locale ** pLocale)
|
|
{
|
|
try
|
|
{
|
|
*pLocale = new std::locale(localeName);
|
|
}
|
|
catch(const std::exception& e)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void setDefaultLocale(const char ** localeName, std::locale ** pLocale)
|
|
{
|
|
if(!localeName || !_setLocale(*localeName, pLocale))
|
|
{
|
|
int count = 0;
|
|
while(!_setLocale(DEFAULT_LOCALES[count], pLocale) && count < _countof(DEFAULT_LOCALES))
|
|
{
|
|
count++;
|
|
}
|
|
|
|
if(localeName)
|
|
*localeName = count < _countof(DEFAULT_LOCALES)?DEFAULT_LOCALES[count]:NULL;
|
|
}
|
|
}
|
|
|
|
SystemLocale::SystemLocale( const char * localeName )
|
|
: m_uAnsiCP(CP_UTF8)
|
|
, m_pLocale(NULL)
|
|
{
|
|
setDefaultLocale(&localeName, &m_pLocale);
|
|
|
|
// Mapping from locale charset to codepage
|
|
struct LocaleCP
|
|
{
|
|
const char* localeName;
|
|
UINT codePage;
|
|
};
|
|
#define CPxxx(cp) { "CP" #cp, cp }
|
|
#define ISO8859(n) { "ISO-8859-" #n, CP_ISO8859_ ## n }, \
|
|
{ "8859_" #n, CP_ISO8859_ ## n }, \
|
|
{ "ISO8859-" #n, CP_ISO8859_ ## n }, \
|
|
{ "ISO8859" #n, CP_ISO8859_ ## n }, \
|
|
{ "ISO_8859-" #n, CP_ISO8859_ ## n }, \
|
|
{ "ISO_8859_" #n, CP_ISO8859_ ## n }
|
|
const LocaleCP lcpTable[] = {
|
|
{ "utf8", CP_UTF8 },
|
|
{ "UTF-8", CP_UTF8 },
|
|
{ "BIG5", 950 },
|
|
{ "BIG5-HKSCS", 950 },
|
|
{ "gb18030", 54936 },
|
|
{ "gb2312", 936 },
|
|
{ "gbk", 936 },
|
|
CPxxx(1252), CPxxx(850), CPxxx(437), CPxxx(874), CPxxx(932), CPxxx(936), CPxxx(949), CPxxx(950),
|
|
CPxxx(1250), CPxxx(1251), CPxxx(1253), CPxxx(1254), CPxxx(1255), CPxxx(1256), CPxxx(1257), CPxxx(1258),
|
|
ISO8859(1), ISO8859(2), ISO8859(3), ISO8859(4), ISO8859(5), ISO8859(6),
|
|
ISO8859(7), ISO8859(8), ISO8859(9), ISO8859(13), ISO8859(15),
|
|
{ "UTF-32LE", 12000 }
|
|
};
|
|
if (localeName)
|
|
{
|
|
const char *charsetName = strchr(localeName, '.');
|
|
charsetName = charsetName ? charsetName + 1 : localeName;
|
|
for (const LocaleCP& lcp : lcpTable)
|
|
{
|
|
if (!strncasecmp(lcp.localeName, charsetName, strnlen_s(lcp.localeName)))
|
|
{
|
|
m_uAnsiCP = lcp.codePage;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
SystemLocale::~SystemLocale()
|
|
{
|
|
delete m_pLocale;
|
|
}
|
|
|
|
const SystemLocale & SystemLocale::Singleton()
|
|
{
|
|
// GCC ensures that function scoped static initializers are threadsafe
|
|
// We must not use the -fno-threadsafe-statics compiler option
|
|
#if !defined(__GNUC__) || defined(NO_THREADSAFE_STATICS)
|
|
#error "Relying on GCC's threadsafe initialization of local statics."
|
|
#endif
|
|
static const SystemLocale s_Default(setlocale(LC_CTYPE, NULL));
|
|
return s_Default;
|
|
}
|
|
|
|
|
|
// Convert CP1252 to UTF-16 without requiring iconv or taking a lock.
|
|
// This is trivial because, except for the 80-9F range, CP1252 bytes
|
|
// directly map to the corresponding UTF-16 codepoint.
|
|
size_t SystemLocale::CP1252ToUtf16( const char *src, SSIZE_T cchSrc, WCHAR *dest, size_t cchDest, DWORD *pErrorCode )
|
|
{
|
|
const static WCHAR s_1252Map[] =
|
|
{
|
|
0x20AC, 0x003F, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x003F, 0x017D, 0x003F,
|
|
0x003F, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x003F, 0x017E, 0x0178
|
|
};
|
|
const unsigned char *usrc = reinterpret_cast<const unsigned char*>(src);
|
|
const unsigned char *srcEnd = usrc + cchSrc;
|
|
const WCHAR *destEnd = dest + cchDest;
|
|
|
|
while(usrc < srcEnd && dest < destEnd)
|
|
{
|
|
DWORD ucode = *usrc++;
|
|
*dest++ = (ucode <= 127 || ucode >= 160) ? ucode : s_1252Map[ucode - 128];
|
|
}
|
|
pErrorCode && (*pErrorCode = (dest == destEnd && usrc != srcEnd) ? ERROR_INSUFFICIENT_BUFFER : ERROR_SUCCESS);
|
|
return cchDest - (destEnd - dest);
|
|
}
|
|
|
|
// Convert UTF-8 to UTF-16 without requiring iconv or taking a lock.
|
|
// 0abcdefg -> 0abcdefg 00000000
|
|
// 110abcde 10fghijk -> defghijk 00000abc
|
|
// 1110abcd 10efghij 10klmnop -> ijklmnop abcdefgh
|
|
// 11110abc 10defghi 10jklmno 10pqrstu -> cdfghijk 110110ab nopqrstu 11011lm
|
|
size_t SystemLocale::Utf8To16( const char *src, SSIZE_T cchSrc, WCHAR *dest, size_t cchDest, DWORD *pErrorCode )
|
|
{
|
|
const unsigned char *usrc = reinterpret_cast<const unsigned char*>(src);
|
|
const unsigned char *srcEnd = usrc + cchSrc;
|
|
const WCHAR *destEnd = dest + cchDest;
|
|
DWORD dummyError;
|
|
if (!pErrorCode)
|
|
{
|
|
pErrorCode = &dummyError;
|
|
}
|
|
*pErrorCode = 0;
|
|
|
|
while(usrc < srcEnd && dest < destEnd)
|
|
{
|
|
DWORD ucode = *usrc++;
|
|
if(ucode <= 127) // Most common case for ASCII
|
|
{
|
|
*dest++ = ucode;
|
|
}
|
|
else if(ucode < 0xC0) // unexpected trailing byte 10xxxxxx
|
|
{
|
|
goto Invalid;
|
|
}
|
|
else if(ucode < 0xE0) // 110abcde 10fghijk
|
|
{
|
|
if (usrc >= srcEnd || *usrc < 0x80 || *usrc > 0xBF ||
|
|
(*dest = (ucode & 0x1F)<<6 | (*usrc++ & 0x3F)) < 0x80)
|
|
{
|
|
*dest = 0xFFFD;
|
|
}
|
|
dest++;
|
|
}
|
|
else if(ucode < 0xF0) // 1110abcd 10efghij 10klmnop
|
|
{
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c1 = *usrc;
|
|
if (c1 < 0x80 || c1 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c2 = *usrc;
|
|
if (c2 < 0x80 || c2 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
ucode = (ucode&15)<<12 | (c1&0x3F)<<6 | (c2&0x3F);
|
|
if (ucode < 0x800 || (ucode >= 0xD800 && ucode <= 0xDFFF))
|
|
{
|
|
goto Invalid;
|
|
}
|
|
*dest++ = ucode;
|
|
}
|
|
else if(ucode < 0xF8) // 11110abc 10defghi 10jklmno 10pqrstu
|
|
{
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c1 = *usrc;
|
|
if (c1 < 0x80 || c1 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c2 = *usrc;
|
|
if (c2 < 0x80 || c2 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c3 = *usrc;
|
|
if (c3 < 0x80 || c3 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
ucode = (ucode&7)<<18 | (c1&0x3F)<<12 | (c2&0x3F)<<6 | (c3&0x3F);
|
|
|
|
if (ucode < 0x10000 // overlong encoding
|
|
|| ucode > 0x10FFFF // exceeds Unicode range
|
|
|| (ucode >= 0xD800 && ucode <= 0xDFFF)) // surrogate pairs
|
|
{
|
|
goto Invalid;
|
|
}
|
|
if (dest >= destEnd - 1)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return cchDest - (destEnd - dest);
|
|
}
|
|
ucode -= 0x10000;
|
|
// Lead surrogate
|
|
*dest++ = 0xD800 + (ucode >> 10);
|
|
// Trail surrogate
|
|
*dest++ = 0xDC00 + (ucode & 0x3FF);
|
|
}
|
|
else // invalid
|
|
{
|
|
Invalid:
|
|
*dest++ = 0xFFFD;
|
|
}
|
|
}
|
|
if (!*pErrorCode)
|
|
{
|
|
*pErrorCode = (dest == destEnd && usrc != srcEnd) ? ERROR_INSUFFICIENT_BUFFER : ERROR_SUCCESS;
|
|
}
|
|
return cchDest - (destEnd - dest);
|
|
}
|
|
|
|
size_t SystemLocale::Utf8To16Strict( const char *src, SSIZE_T cchSrc, WCHAR *dest, size_t cchDest, DWORD *pErrorCode )
|
|
{
|
|
const unsigned char *usrc = reinterpret_cast<const unsigned char*>(src);
|
|
const unsigned char *srcEnd = usrc + cchSrc;
|
|
const WCHAR *destEnd = dest + cchDest;
|
|
DWORD dummyError;
|
|
if (!pErrorCode)
|
|
{
|
|
pErrorCode = &dummyError;
|
|
}
|
|
*pErrorCode = 0;
|
|
|
|
while(usrc < srcEnd && dest < destEnd)
|
|
{
|
|
DWORD ucode = *usrc++;
|
|
if(ucode <= 127) // Most common case for ASCII
|
|
{
|
|
*dest++ = ucode;
|
|
}
|
|
else if(ucode < 0xC0) // unexpected trailing byte 10xxxxxx
|
|
{
|
|
goto Invalid;
|
|
}
|
|
else if(ucode < 0xE0) // 110abcde 10fghijk
|
|
{
|
|
if (usrc >= srcEnd || *usrc < 0x80 || *usrc > 0xBF ||
|
|
(*dest = (ucode & 0x1F)<<6 | (*usrc++ & 0x3F)) < 0x80)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
dest++;
|
|
}
|
|
else if(ucode < 0xF0) // 1110abcd 10efghij 10klmnop
|
|
{
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c1 = *usrc;
|
|
if (c1 < 0x80 || c1 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c2 = *usrc;
|
|
if (c2 < 0x80 || c2 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
ucode = (ucode&15)<<12 | (c1&0x3F)<<6 | (c2&0x3F);
|
|
if (ucode < 0x800 || (ucode >= 0xD800 && ucode <= 0xDFFF))
|
|
{
|
|
goto Invalid;
|
|
}
|
|
*dest++ = ucode;
|
|
}
|
|
else if(ucode < 0xF8) // 11110abc 10defghi 10jklmno 10pqrstu
|
|
{
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c1 = *usrc;
|
|
if (c1 < 0x80 || c1 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c2 = *usrc;
|
|
if (c2 < 0x80 || c2 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
if (usrc >= srcEnd)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
DWORD c3 = *usrc;
|
|
if (c3 < 0x80 || c3 > 0xBF)
|
|
{
|
|
goto Invalid;
|
|
}
|
|
usrc++;
|
|
ucode = (ucode&7)<<18 | (c1&0x3F)<<12 | (c2&0x3F)<<6 | (c3&0x3F);
|
|
|
|
if (ucode < 0x10000 // overlong encoding
|
|
|| ucode > 0x10FFFF // exceeds Unicode range
|
|
|| (ucode >= 0xD800 && ucode <= 0xDFFF)) // surrogate pairs
|
|
{
|
|
goto Invalid;
|
|
}
|
|
if (dest >= destEnd - 1)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return cchDest - (destEnd - dest);
|
|
}
|
|
ucode -= 0x10000;
|
|
// Lead surrogate
|
|
*dest++ = 0xD800 + (ucode >> 10);
|
|
// Trail surrogate
|
|
*dest++ = 0xDC00 + (ucode & 0x3FF);
|
|
}
|
|
else // invalid
|
|
{
|
|
Invalid:
|
|
*pErrorCode = ERROR_NO_UNICODE_TRANSLATION;
|
|
return 0 ;
|
|
}
|
|
}
|
|
if (!*pErrorCode)
|
|
{
|
|
*pErrorCode = (dest == destEnd && usrc != srcEnd) ? ERROR_INSUFFICIENT_BUFFER : ERROR_SUCCESS;
|
|
}
|
|
return cchDest - (destEnd - dest);
|
|
}
|
|
|
|
size_t SystemLocale::ToUtf16( UINT srcCodePage, const char * src, SSIZE_T cchSrc, WCHAR * dest, size_t cchDest, DWORD * pErrorCode )
|
|
{
|
|
if (cchSrc < 0) {
|
|
if (NULL != pErrorCode)
|
|
*pErrorCode = ERROR_INVALID_PARAMETER;
|
|
return 0;
|
|
}
|
|
|
|
srcCodePage = ExpandSpecialCP( srcCodePage );
|
|
if ( dest )
|
|
{
|
|
if ( srcCodePage == CP_UTF8 )
|
|
{
|
|
return SystemLocale::Utf8To16( src, cchSrc, dest, cchDest, pErrorCode );
|
|
}
|
|
else if ( srcCodePage == 1252 )
|
|
{
|
|
return SystemLocale::CP1252ToUtf16( src, cchSrc, dest, cchDest, pErrorCode );
|
|
}
|
|
}
|
|
EncodingConverter cvt( CP_UTF16, srcCodePage );
|
|
if ( !cvt.Initialize() )
|
|
{
|
|
if ( NULL != pErrorCode )
|
|
*pErrorCode = ERROR_INVALID_PARAMETER;
|
|
return 0;
|
|
}
|
|
size_t cchSrcActual = (cchSrc < 0 ? (1+strnlen_s(src)) : cchSrc);
|
|
bool hasLoss = false;
|
|
return cvt.Convert( dest, cchDest, src, cchSrcActual, false, &hasLoss, pErrorCode );
|
|
}
|
|
|
|
size_t SystemLocale::ToUtf16Strict( UINT srcCodePage, const char * src, SSIZE_T cchSrc, WCHAR * dest, size_t cchDest, DWORD * pErrorCode )
|
|
{
|
|
if (cchSrc < 0) {
|
|
if (NULL != pErrorCode)
|
|
*pErrorCode = ERROR_INVALID_PARAMETER;
|
|
return 0;
|
|
}
|
|
srcCodePage = ExpandSpecialCP( srcCodePage );
|
|
if ( dest )
|
|
{
|
|
if ( srcCodePage == CP_UTF8 )
|
|
{
|
|
return SystemLocale::Utf8To16Strict( src, cchSrc, dest, cchDest, pErrorCode );
|
|
}
|
|
else if ( srcCodePage == 1252 )
|
|
{
|
|
return SystemLocale::CP1252ToUtf16( src, cchSrc, dest, cchDest, pErrorCode );
|
|
}
|
|
}
|
|
EncodingConverter cvt( CP_UTF16, srcCodePage );
|
|
if ( !cvt.Initialize() )
|
|
{
|
|
if ( NULL != pErrorCode )
|
|
*pErrorCode = ERROR_INVALID_PARAMETER;
|
|
return 0;
|
|
}
|
|
size_t cchSrcActual = (cchSrc < 0 ? (1+strnlen_s(src)) : cchSrc);
|
|
bool hasLoss = false;
|
|
return cvt.Convert( dest, cchDest, src, cchSrcActual, true, &hasLoss, pErrorCode );
|
|
}
|
|
|
|
size_t SystemLocale::Utf8From16( const WCHAR *src, SSIZE_T cchSrc, char *dest, size_t cchDest, DWORD *pErrorCode )
|
|
{
|
|
const WCHAR *srcEnd = src + cchSrc;
|
|
char *destEnd = dest + cchDest;
|
|
DWORD dummyError;
|
|
if (!pErrorCode)
|
|
{
|
|
pErrorCode = &dummyError;
|
|
}
|
|
*pErrorCode = 0;
|
|
|
|
// null dest is a special mode to calculate the output size required.
|
|
if (!dest)
|
|
{
|
|
size_t cbOut = 0;
|
|
while (src < srcEnd)
|
|
{
|
|
DWORD wch = *src++;
|
|
if (wch < 128) // most common case.
|
|
{
|
|
cbOut++;
|
|
}
|
|
else if (wch < 0x800) // 127 to 2047: 2 bytes
|
|
{
|
|
cbOut += 2;
|
|
}
|
|
else if (wch < 0xD800 || wch > 0xDFFF) // 2048 to 55295 and 57344 to 65535: 3 bytes
|
|
{
|
|
cbOut += 3;
|
|
}
|
|
else if (wch < 0xDC00) // 65536 to end of Unicode: 4 bytes
|
|
{
|
|
if (src >= srcEnd)
|
|
{
|
|
cbOut += 3; // lone surrogate at end
|
|
}
|
|
else if (*src < 0xDC00 || *src > 0xDFFF)
|
|
{
|
|
cbOut += 3; // low surrogate not followed by high
|
|
}
|
|
else
|
|
{
|
|
cbOut += 4;
|
|
}
|
|
}
|
|
else // unexpected trail surrogate
|
|
{
|
|
cbOut += 3;
|
|
}
|
|
}
|
|
return cbOut;
|
|
}
|
|
while ( src < srcEnd && dest < destEnd )
|
|
{
|
|
DWORD wch = *src++;
|
|
if (wch < 128) // most common case.
|
|
{
|
|
*dest++ = wch;
|
|
}
|
|
else if (wch < 0x800) // 127 to 2047: 2 bytes
|
|
{
|
|
if (destEnd - dest < 2)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xC0 | (wch >> 6);
|
|
*dest++ = 0x80 | (wch & 0x3F);
|
|
}
|
|
else if (wch < 0xD800 || wch > 0xDFFF) // 2048 to 55295 and 57344 to 65535: 3 bytes
|
|
{
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xE0 | (wch >> 12);
|
|
*dest++ = 0x80 | ((wch >> 6)&0x3F);
|
|
*dest++ = 0x80 | (wch &0x3F);
|
|
}
|
|
else if (wch < 0xDC00) // 65536 to end of Unicode: 4 bytes
|
|
{
|
|
if (src >= srcEnd)
|
|
{
|
|
*pErrorCode = ERROR_NO_UNICODE_TRANSLATION; // lone surrogate at end
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xEF;
|
|
*dest++ = 0xBF;
|
|
*dest++ = 0xBD;
|
|
continue;
|
|
}
|
|
if (*src < 0xDC00 || *src > 0xDFFF)
|
|
{
|
|
// low surrogate not followed by high
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xEF;
|
|
*dest++ = 0xBF;
|
|
*dest++ = 0xBD;
|
|
continue;
|
|
}
|
|
wch = 0x10000 + ((wch - 0xD800)<<10) + *src++ - 0xDC00;
|
|
if (destEnd - dest < 4)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xF0 | (wch >> 18);
|
|
*dest++ = 0x80 | ((wch >>12)&0x3F);
|
|
*dest++ = 0x80 | ((wch >> 6)&0x3F);
|
|
*dest++ = 0x80 | (wch&0x3F);
|
|
}
|
|
else // unexpected trail surrogate
|
|
{
|
|
*pErrorCode = ERROR_NO_UNICODE_TRANSLATION; // lone surrogate at end
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xEF;
|
|
*dest++ = 0xBF;
|
|
*dest++ = 0xBD;
|
|
}
|
|
}
|
|
if (!*pErrorCode)
|
|
{
|
|
*pErrorCode = (dest == destEnd && src != srcEnd) ? ERROR_INSUFFICIENT_BUFFER : ERROR_SUCCESS;
|
|
}
|
|
return *pErrorCode == ERROR_INSUFFICIENT_BUFFER ? 0 : cchDest - (destEnd - dest);
|
|
}
|
|
|
|
size_t SystemLocale::Utf8From16Strict( const WCHAR *src, SSIZE_T cchSrc, char *dest, size_t cchDest, DWORD *pErrorCode )
|
|
{
|
|
const WCHAR *srcEnd = src + cchSrc;
|
|
char *destEnd = dest + cchDest;
|
|
DWORD dummyError;
|
|
if (!pErrorCode)
|
|
{
|
|
pErrorCode = &dummyError;
|
|
}
|
|
*pErrorCode = 0;
|
|
|
|
// null dest is a special mode to calculate the output size required.
|
|
if (!dest)
|
|
{
|
|
size_t cbOut = 0;
|
|
while (src < srcEnd)
|
|
{
|
|
DWORD wch = *src++;
|
|
if (wch < 128) // most common case.
|
|
{
|
|
cbOut++;
|
|
}
|
|
else if (wch < 0x800) // 127 to 2047: 2 bytes
|
|
{
|
|
cbOut += 2;
|
|
}
|
|
else if (wch < 0xD800 || wch > 0xDFFF) // 2048 to 55295 and 57344 to 65535: 3 bytes
|
|
{
|
|
cbOut += 3;
|
|
}
|
|
else if (wch < 0xDC00) // 65536 to end of Unicode: 4 bytes
|
|
{
|
|
if (src >= srcEnd)
|
|
{
|
|
cbOut += 3; // lone surrogate at end
|
|
}
|
|
else if (*src < 0xDC00 || *src > 0xDFFF)
|
|
{
|
|
cbOut += 3; // low surrogate not followed by high
|
|
}
|
|
else
|
|
{
|
|
cbOut += 4;
|
|
}
|
|
}
|
|
else // unexpected trail surrogate
|
|
{
|
|
cbOut += 3;
|
|
}
|
|
}
|
|
return cbOut;
|
|
}
|
|
while ( src < srcEnd && dest < destEnd )
|
|
{
|
|
DWORD wch = *src++;
|
|
if (wch < 128) // most common case.
|
|
{
|
|
*dest++ = wch;
|
|
}
|
|
else if (wch < 0x800) // 127 to 2047: 2 bytes
|
|
{
|
|
if (destEnd - dest < 2)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xC0 | (wch >> 6);
|
|
*dest++ = 0x80 | (wch & 0x3F);
|
|
}
|
|
else if (wch < 0xD800 || wch > 0xDFFF) // 2048 to 55295 and 57344 to 65535: 3 bytes
|
|
{
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xE0 | (wch >> 12);
|
|
*dest++ = 0x80 | ((wch >> 6)&0x3F);
|
|
*dest++ = 0x80 | (wch &0x3F);
|
|
}
|
|
else if (wch < 0xDC00) // 65536 to end of Unicode: 4 bytes
|
|
{
|
|
if (src >= srcEnd)
|
|
{
|
|
*pErrorCode = ERROR_NO_UNICODE_TRANSLATION; // lone surrogate at end
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
if (*src < 0xDC00 || *src > 0xDFFF)
|
|
{
|
|
*pErrorCode = ERROR_NO_UNICODE_TRANSLATION; // low surrogate not followed by high
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
}
|
|
return 0;
|
|
}
|
|
wch = 0x10000 + ((wch - 0xD800)<<10) + *src++ - 0xDC00;
|
|
if (destEnd - dest < 4)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
return 0;
|
|
}
|
|
*dest++ = 0xF0 | (wch >> 18);
|
|
*dest++ = 0x80 | ((wch >>12)&0x3F);
|
|
*dest++ = 0x80 | ((wch >> 6)&0x3F);
|
|
*dest++ = 0x80 | (wch&0x3F);
|
|
}
|
|
else // unexpected trail surrogate
|
|
{
|
|
*pErrorCode = ERROR_NO_UNICODE_TRANSLATION; // lone surrogate at end
|
|
if (destEnd - dest < 3)
|
|
{
|
|
*pErrorCode = ERROR_INSUFFICIENT_BUFFER;
|
|
}
|
|
return 0;
|
|
}
|
|
}
|
|
if (!*pErrorCode)
|
|
{
|
|
*pErrorCode = (dest == destEnd && src != srcEnd) ? ERROR_INSUFFICIENT_BUFFER : ERROR_SUCCESS;
|
|
}
|
|
return *pErrorCode == ERROR_INSUFFICIENT_BUFFER ? 0 : cchDest - (destEnd - dest);
|
|
}
|
|
|
|
size_t SystemLocale::FromUtf16( UINT destCodePage, const WCHAR * src, SSIZE_T cchSrc, char * dest, size_t cchDest, bool * pHasDataLoss, DWORD * pErrorCode )
|
|
{
|
|
destCodePage = ExpandSpecialCP( destCodePage );
|
|
if ( destCodePage == CP_UTF8 )
|
|
{
|
|
pHasDataLoss && (*pHasDataLoss = 0);
|
|
return SystemLocale::Utf8From16( src, cchSrc < 0 ? 1+mplat_wcslen(src) : cchSrc, dest, cchDest, pErrorCode );
|
|
}
|
|
EncodingConverter cvt( destCodePage, CP_UTF16 );
|
|
if ( !cvt.Initialize() )
|
|
{
|
|
if ( NULL != pErrorCode )
|
|
*pErrorCode = ERROR_INVALID_PARAMETER;
|
|
return 0;
|
|
}
|
|
size_t cchSrcActual = (cchSrc < 0 ? (1+mplat_wcslen(src)) : cchSrc);
|
|
bool hasLoss = false;
|
|
return cvt.Convert( dest, cchDest, src, cchSrcActual, false, &hasLoss, pErrorCode );
|
|
}
|
|
|
|
size_t SystemLocale::FromUtf16Strict(UINT destCodePage, const WCHAR * src, SSIZE_T cchSrc, char * dest, size_t cchDest, bool * pHasDataLoss, DWORD * pErrorCode)
|
|
{
|
|
destCodePage = ExpandSpecialCP(destCodePage);
|
|
if ( destCodePage == CP_UTF8 )
|
|
{
|
|
pHasDataLoss && (*pHasDataLoss = 0);
|
|
return SystemLocale::Utf8From16Strict( src, cchSrc < 0 ? 1+mplat_wcslen(src) : cchSrc, dest, cchDest, pErrorCode );
|
|
}
|
|
EncodingConverter cvt(destCodePage, CP_UTF16);
|
|
if (!cvt.Initialize())
|
|
{
|
|
if (NULL != pErrorCode)
|
|
*pErrorCode = ERROR_INVALID_PARAMETER;
|
|
return 0;
|
|
}
|
|
size_t cchSrcActual = (cchSrc < 0 ? (1 + mplat_wcslen(src)) : cchSrc);
|
|
bool hasLoss = false;
|
|
return cvt.Convert(dest, cchDest, src, cchSrcActual, true, &hasLoss, pErrorCode);
|
|
}
|
|
|
|
char * SystemLocale::NextChar( UINT codepage, const char * start, size_t cchBytesLeft )
|
|
{
|
|
if ( NULL == start || '\0' == *start || 0 == cchBytesLeft )
|
|
return const_cast<char *>( start );
|
|
|
|
char first = *start;
|
|
codepage = ExpandSpecialCP( codepage );
|
|
if ( CP_UTF8 != codepage )
|
|
{
|
|
if ( !IsDBCSLeadByteEx(codepage, first) || '\0' == *(start+1) )
|
|
return const_cast<char *>( start+1 ); // single byte char or truncated double byte char
|
|
else
|
|
return const_cast<char *>( start+2 ); // double byte char
|
|
}
|
|
|
|
// CP_UTF8
|
|
// MB utf8 sequences have this format
|
|
// Lead byte starts with 2 set bits, '11'
|
|
// Rest of bytes start with one set and one not, '10'
|
|
|
|
// ASCII or not first of utf8 sequence
|
|
// If this isn't the first byte of a utf8 sequence, just move one byte at a time
|
|
// since we don't know where the correct boundary is located.
|
|
if ( (char)0 == (first & (char)0x80) || !SystemLocale::IsUtf8LeadByte((BYTE)first) )
|
|
return const_cast<char *>( start+1 );
|
|
else
|
|
{
|
|
// Initial char tells us how many bytes are supposed to be in this sequence
|
|
UINT cchExpectedSize = SystemLocale::CchUtf8CodePt( (BYTE)first );
|
|
|
|
// Skip lead bye
|
|
++start;
|
|
--cchExpectedSize;
|
|
--cchBytesLeft;
|
|
|
|
// Proceed to end of utf8 sequence, null term, or end of expected size
|
|
while ( 0 < cchExpectedSize && 0 < cchBytesLeft && (char)0x80 == (*start & (char)0xC0) )
|
|
{
|
|
++start;
|
|
--cchExpectedSize;
|
|
--cchBytesLeft;
|
|
}
|
|
return const_cast<char *>( start );
|
|
}
|
|
}
|
|
|
|
char * SystemLocale::NextChar( UINT codepage, const char * start )
|
|
{
|
|
// Just assume some large max buffer size since caller is saying
|
|
// start is null terminated.
|
|
return NextChar( codepage, start, DWORD_MAX );
|
|
}
|