W
So, für alle die, die's interessiert, hier der Code:
#include <string>
#include <limits>
typedef std::u32string str_t;
typedef char32_t char_t;
void Error(const char_t *);
size_t utils::binary::GetLeadingZerosCount(char_t in)
{
int bits = sizeof(in) * CHAR_BIT;
int bit_count = bits - 1; //-1 for shifting (32 bits -> shift by 31 to get the first one)
for(; bit_count >= 0; --bit_count) if((in & (1 << bit_count)) != 0) break;
//now bit_count is at the first binary 0
return bits - bit_count - 1;
}
size_t utils::binary::GetLeadingOnesCount(char in) //returns the number of leading ones in binary
{
int bits = sizeof(in) * CHAR_BIT;
int bit_count = bits - 1; //-1 for shifting (8 bits -> shift by 7 to get the first one)
for(; bit_count >= 0; --bit_count) if((in & (1 << bit_count)) == 0) break;
//now bit_count is at the first binary 0
return bits - bit_count - 1;
}
char_t encoding::MultiCharFromUTF16To32(char16_t high, char16_t low) //private
{
char_t high_decoded, low_decoded;
high_decoded = high - 0xD800;
low_decoded = low - 0xDC00;
return ((high_decoded << 10) + low_decoded) + 0x10000;
}
//corresponding from http://floern.com/webscripting/is-utf8-auf-utf-8-pr%C3%BCfen
bool encoding::IsUTF8(const std::string &in)
{
//check for UTF-8 Byte-Order-Mark (not always there
if(in.substr(0,3) == "\xEF\xBB\xBF") return true;
size_t length = in.length();
for(size_t i = 0; i < length; ++i)
{
char_t val = in[i];
size_t n = 0;
if(val < 0x80) continue; // 0bbbbbbb
else if((val & 0xE0) == 0xC0 && val > 0xC1) n = 1; // 110bbbbb (excl C0-C1)
else if((val & 0xF0) == 0xE0) n = 2; // 1110bbbb
else if((val & 0xF8) == 0xF0 && val < 0xF5) n = 3; // 11110bbb (excl F5-FF)
else return false; // invalid UTF-8-character
for(size_t c = 0; c < n; c++) // n following bytes? // 10bbbbbb
if(++i == length || (in[i] & 0xC0) != 0x80)
return false; // invalid UTF-8-character
}
return true; //no invalid UTF-8 sign found
}
encoding::Type encoding::GetEncoding(const std::string &in)
{
//check for UTF-32 before UTF-16 because UTF16_BE would match UTF32_BE but not reverse
if(in.substr(0, 4) == std::string("\xFF\xFE\x00\x00", 4)) return encoding::UTF32_LE;
else if(in.substr(0, 4) == std::string("\x00\x00\xFE\xFF", 4)) return encoding::UTF32_BE;
else if(in.substr(0, 2) == "\xFF\xFE") return encoding::UTF16_LE;
else if(in.substr(0, 2) == "\xFE\xFF") return encoding::UTF16_BE;
else if(encoding::IsUTF8(in)) return encoding::UTF8;
else return encoding::ASCII;
}
//you firstly have to get the encoding of the string and then convert case-specific
str_t encoding::FileStringToUTF(const std::string &str, const std::string &filename)
{
//All Bytes of UTF-16/32 are stored separately in characters of the str_t by ifstr_t
//unify them
encoding::Type str_encoding = encoding::GetEncoding(str);
str_t ret;
if(str_encoding == encoding::ASCII) //Error: we can not know/convert the codepage
{
C_OUT << tr(_("Fatal error in file")) << _(" \"") << filename << _(":\n");
C_OUT << tr(_("Please save your File in an UTF-Encoding or don't use special letters")) << _("\n");
return str_t();
}
else if(str_encoding == encoding::UTF32_BE)
{
encoding::UTF32BETo32(str, ret);
}
else if(str_encoding == encoding::UTF32_LE)
{
encoding::UTF32LETo32(str, ret);
}
else if(str_encoding == encoding::UTF16_BE)
{
encoding::UTF16BETo32(str, ret);
}
else if(str_encoding == encoding::UTF16_LE)
{
encoding::UTF16LETo32(str, ret);
}
else if(str_encoding == encoding::UTF8)
{
encoding::UTF8To32(str, ret);
}
return str_t(str.begin(), str.end());
}
void encoding::UTF16LETo32(const std::string &in, str_t &out)
{
//Encoding: 2 1 [4 3]
//Next Byte Detector: 0xD800 <= x <= 0xDBFF
//see http://en.wikipedia.org/wiki/UTF-16/UCS-2
if((((in.size()-2)) % 2) != 0)
Error(_("encoding::UTF16LETo32()->Wrong Byte number"));
out.reserve((in.size()-2) / 4); //minimum size, is probably higher, will be dynamically increased
size_t in_pos = 2; //after BOM
unsigned long tmp = 0;
char16_t first, second; //first and second 16-Bit-Char, second only sometimes needed
while(in_pos < in.size())
{
first = static_cast<unsigned char>(in[in_pos]);
first |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+1])) << CHAR_BIT;
if(first >= 0xD800 && first <= 0xDBFF) //a second 16-Bit-Character is needed for the one UTF-32 character
{
in_pos += 2;
if(in_pos >= in.size())
Error(_("encoding::UTF16LETo32()->No second 16-Bit char"));
second = static_cast<unsigned char>(in[in_pos]);
second |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+1])) << CHAR_BIT;
tmp = encoding::MultiCharFromUTF16To32(first, second);
}
else
{
tmp = first;
}
out += tmp;
in_pos += 2;
}
}
void encoding::UTF16BETo32(const std::string &in, str_t &out)
{
//Encoding: 1 2 [3 4]
//Next Byte Detector: 0xD800 <= x <= 0xDBFF
//see http://en.wikipedia.org/wiki/UTF-16/UCS-2
if((((in.size()-2)) % 2) != 0)
Error(_("encoding::UTF16BETo32()->Wrong Byte number"));
out.reserve((in.size()-2) / 4); //minimum size, is probably higher, will be dynamically increased
size_t in_pos = 2; //after BOM
unsigned long tmp = 0;
char16_t first, second; //first and second 16-Bit-Char, second only sometimes needed
while(in_pos < in.size())
{
first = static_cast<unsigned char>(in[in_pos+1]);
first |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos])) << CHAR_BIT;
if(first >= 0xD800 && first <= 0xDBFF) //a second 16-Bit-Character is needed for the one UTF-32 character
{
in_pos += 2;
if(in_pos >= in.size())
Error(_("encoding::UTF16BETo32()->No second 16-Bit char"));
second = static_cast<unsigned char>(in[in_pos+1]);
second |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos])) << CHAR_BIT;
tmp = encoding::MultiCharFromUTF16To32(first, second);
}
else
{
tmp = first;
}
out += tmp;
in_pos += 2;
}
}
void encoding::UTF32LETo32(const std::string &in, str_t &out)
{
//Encoding: 4 Bytes: 4 3 2 1
if((((in.size()-4)) % 4) != 0)
Error(_("encoding::UTF32LETo32()->UTF32_LE->Wrong Byte number"));
out.resize((in.size()-4) / 4);
size_t in_pos = 4; //after BOM
size_t out_pos = 0;
unsigned long tmp = 0;
while(in_pos < in.size())
{
tmp = static_cast<unsigned char>(in[in_pos]);
tmp |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+1])) << CHAR_BIT;
tmp |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+2])) << (CHAR_BIT * 2);
tmp |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+3])) << (CHAR_BIT * 3);
out[out_pos] = tmp;
++out_pos;
in_pos += 4;
}
}
void encoding::UTF32BETo32(const std::string &in, str_t &out)
{
//Encoding: 4 Bytes: 1 2 3 4
if((((in.size()-4)) % 4) != 0)
Error(_("encoding::UTF32BETo32()->Wrong Byte number"));
out.resize((in.size()-4) / 4);
size_t in_pos = 4; //after BOM
size_t out_pos = 0;
while(in_pos < in.size())
{
char_t tmp = 0;
tmp = static_cast<unsigned char>(in[in_pos+3]);
tmp |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+2])) << CHAR_BIT;
tmp |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos+1])) << (CHAR_BIT * 2);
tmp |= static_cast<char_t>(static_cast<unsigned char>(in[in_pos])) << (CHAR_BIT * 3);
out[out_pos] = tmp;
++out_pos;
in_pos += 4;
}
}
void encoding::UTF8To32(const std::string &in, str_t &out)
{
//see http://www.fileformat.info/info/unicode/utf8.htm
//and http://en.wikipedia.org/wiki/UTF-8
//example:
//00000000000000100011011010110011
//becomes
//11110000 10100011 10011010 10110011
//TODO
char_t tmp = 0;
size_t in_pos = 0;
int following_byte_num = 0, leading_ones = 0;
if(in.substr(0,3) == "\xEF\xBB\xBF") in_pos = 3; //BOM, optional
out.reserve((in.size() - in_pos) / 4);
while(in_pos < in.size())
{
leading_ones = utils::binary::GetLeadingOnesCount(in[in_pos]);
following_byte_num = leading_ones;
//UTF-8 uses e.g. 0b110xxxx for specifying that 1 Byte will follow
if(leading_ones > 5) Error(_("encoding::UTF8To32->leading_ones>5"));
else if(leading_ones > 1)
{
following_byte_num -= 1; //= leading_ones - 1
//every following byte can save 6 bits
tmp = (in[in_pos] & (0xFF >> (leading_ones+1))) << (following_byte_num * 6);
while(following_byte_num > 0)
{
++in_pos;
--following_byte_num;
tmp |= (in[in_pos] & 0x3F) << (following_byte_num * 6);
}
}
else if(leading_ones == 1) Error(_("encoding::UTF8To32->leading_ones==1"));
else //leading_ones == 0
{
tmp = in[in_pos];
}
out += tmp;
++in_pos;
}
}
std::string encoding::UTF32To8(const str_t &str)
{
std::string ret;
int used_bits = 0, needed_following_bytes = 0, used_firstbyte_bits = 0;
ret.reserve(str.size() + 3); //the minimum size of the UTF8-return-string + byte order mark
ret += "\xEF\xBB\xBF";
for(size_t str_pos = 0; str_pos < str.size(); ++str_pos)
{
if(str[str_pos] < 0x80)
{
ret.push_back(str[str_pos]);
}
else //more UTF8-Bytes are needed in order to encode the character
{
used_bits = (sizeof(char_t) * 8) - utils::binary::GetLeadingZerosCount(str[str_pos]);
needed_following_bytes = (used_bits - 2) / 5; //example: 12 unto 16 bits need 2 following bytes
used_firstbyte_bits = 6 - needed_following_bytes;
const char first_char_leading_ones = (static_cast<char>(0xFF) << (used_firstbyte_bits+1));
const char first_char_other_content = static_cast<unsigned char>(str[str_pos] >> (needed_following_bytes * 6));
ret.push_back(first_char_leading_ones | first_char_other_content);
for(int i = needed_following_bytes; i > 0; --i)
{
//get the next 6 bits from the left of the string
const char_t bits_for_char = ((0x3F << ((i - 1) * 6)) & str[str_pos]) >> ((i-1) * 6);
ret.push_back(0x80 | bits_for_char);
}
}
}
return ret;
}
//str is ASCII-Encoded
str_t encoding::ToUTF(const std::string &str)
{
return str_t(str.begin(), str.end());
}
std::string encoding::ToASCII(const str_t &str)
{
size_t length = str.length();
std::string ret;
ret.resize(length);
for(size_t i = 0; i < length; ++i)
{
if(str[i] > 0xFF) ret[i] = CHAR_MAX;
else ret[i] = str[i];
}
return ret;
}
