diff --git a/external/ours/library/CMakeLists.txt b/external/ours/library/CMakeLists.txt index d3037e4b..d24bef6d 100644 --- a/external/ours/library/CMakeLists.txt +++ b/external/ours/library/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(archive) +add_subdirectory(unicode) diff --git a/external/ours/library/unicode/CMakeLists.txt b/external/ours/library/unicode/CMakeLists.txt new file mode 100644 index 00000000..359e1774 --- /dev/null +++ b/external/ours/library/unicode/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 2.8) + +project(unicode) + +if(WIN32) + add_definitions(/D_CRT_SECURE_NO_WARNINGS) +endif() + +add_subdirectory(src) diff --git a/external/ours/library/unicode/include/FirstUnicode.h b/external/ours/library/unicode/include/FirstUnicode.h new file mode 100644 index 00000000..5ac25422 --- /dev/null +++ b/external/ours/library/unicode/include/FirstUnicode.h @@ -0,0 +1 @@ +#include "../src/shared/FirstUnicode.h" diff --git a/external/ours/library/unicode/include/Unicode.h b/external/ours/library/unicode/include/Unicode.h new file mode 100644 index 00000000..e8f284bb --- /dev/null +++ b/external/ours/library/unicode/include/Unicode.h @@ -0,0 +1 @@ +#include "../src/shared/Unicode.h" diff --git a/external/ours/library/unicode/include/UnicodeBlocks.h b/external/ours/library/unicode/include/UnicodeBlocks.h new file mode 100644 index 00000000..3e0b130f --- /dev/null +++ b/external/ours/library/unicode/include/UnicodeBlocks.h @@ -0,0 +1 @@ +#include "../src/shared/UnicodeBlocks.h" diff --git a/external/ours/library/unicode/include/UnicodeCharacterData.h b/external/ours/library/unicode/include/UnicodeCharacterData.h new file mode 100644 index 00000000..9ca95780 --- /dev/null +++ b/external/ours/library/unicode/include/UnicodeCharacterData.h @@ -0,0 +1 @@ +#include "../src/shared/UnicodeCharacterData.h" diff --git a/external/ours/library/unicode/include/UnicodeCharacterDataMap.h b/external/ours/library/unicode/include/UnicodeCharacterDataMap.h new file mode 100644 index 00000000..305854f6 --- /dev/null +++ b/external/ours/library/unicode/include/UnicodeCharacterDataMap.h @@ -0,0 +1 @@ +#include "../src/shared/UnicodeCharacterDataMap.h" diff --git a/external/ours/library/unicode/include/UnicodeUtils.h b/external/ours/library/unicode/include/UnicodeUtils.h new file mode 100644 index 00000000..ee603d0a --- /dev/null +++ b/external/ours/library/unicode/include/UnicodeUtils.h @@ -0,0 +1 @@ +#include "../src/shared/UnicodeUtils.h" diff --git a/external/ours/library/unicode/include/utf8.h b/external/ours/library/unicode/include/utf8.h new file mode 100644 index 00000000..be4628bb --- /dev/null +++ b/external/ours/library/unicode/include/utf8.h @@ -0,0 +1 @@ +#include "../src/shared/utf8.h" diff --git a/external/ours/library/unicode/src/CMakeLists.txt b/external/ours/library/unicode/src/CMakeLists.txt new file mode 100644 index 00000000..7c5d86e8 --- /dev/null +++ b/external/ours/library/unicode/src/CMakeLists.txt @@ -0,0 +1,33 @@ + + +set(SHARED_SOURCES + shared/FirstUnicode.h + shared/UnicodeBlocks.cpp + shared/UnicodeBlocks.h + shared/UnicodeCharacterData.cpp + shared/UnicodeCharacterData.h + shared/UnicodeCharacterDataMap.cpp + shared/UnicodeCharacterDataMap.h + shared/Unicode.cpp + shared/Unicode.h + shared/UnicodeUtils.cpp + shared/UnicodeUtils.h + shared/utf8.cpp +) + +if(WIN32) + set(PLATFORM_SOURCES + win32/FirstUnicode.cpp + ) + + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/win32) +else() + set(PLATFORM_SOURCES "") +endif() + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/shared) + +add_library(unicode + ${SHARED_SOURCES} + ${PLATFORM_SOURCES} +) diff --git a/external/ours/library/unicode/src/shared/FirstUnicode.h b/external/ours/library/unicode/src/shared/FirstUnicode.h new file mode 100644 index 00000000..cbd3cd84 --- /dev/null +++ b/external/ours/library/unicode/src/shared/FirstUnicode.h @@ -0,0 +1,23 @@ +// ====================================================================== +// +// FirstUnicode.h +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#ifndef INCLUDED_FirstUnicode_H +#define INCLUDED_FirstUnicode_H + +// ====================================================================== +// stl unref inline func removed + +#ifdef WIN32 +// C4514 unreferenced inline/local function has been removed +// C4786 'identifier' : identifier was truncated to 'number' characters in the debug information +// C4702 unreachable code +#pragma warning (disable:4514 4786 4702) +#endif + +// ====================================================================== + +#endif diff --git a/external/ours/library/unicode/src/shared/Unicode.cpp b/external/ours/library/unicode/src/shared/Unicode.cpp new file mode 100644 index 00000000..35323a9f --- /dev/null +++ b/external/ours/library/unicode/src/shared/Unicode.cpp @@ -0,0 +1,18 @@ +// ====================================================================== +// +// Unicode.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstUnicode.h" +#include "Unicode.h" + +// ====================================================================== + +namespace Unicode +{ + +} + +// ====================================================================== diff --git a/external/ours/library/unicode/src/shared/Unicode.h b/external/ours/library/unicode/src/shared/Unicode.h new file mode 100644 index 00000000..c20d6b78 --- /dev/null +++ b/external/ours/library/unicode/src/shared/Unicode.h @@ -0,0 +1,51 @@ +// ====================================================================== +// Unicode.h +// copyright (c) 2001 Sony Online Entertainment +// +// jwatson +// +// Basic Unicode primitives and string handling/manipulating functions +// ====================================================================== + +#ifndef INCLUDED_Unicode_H +#define INCLUDED_Unicode_H + +#if WIN32 +#pragma warning (disable:4710) +#pragma warning (disable:4786) +// stl warning func not inlined +#pragma warning (disable:4514) +#endif + +#include + +//----------------------------------------------------------------- + +namespace Unicode +{ + + typedef unsigned short unicode_char_t; + + /** + * Standard Unicode string is UTF-16 + */ + typedef std::basic_string , std::allocator > String; + + /** + * NarrowString is a ascii string. + */ + + typedef std::string NarrowString; + + const unicode_char_t whitespace [] = { ' ', '\n', '\r', '\t', 0x3000, 0 }; + const unicode_char_t endlines [] = { '\r', '\n', 0 }; + const char ascii_whitespace [] = { ' ', '\n', '\r', '\t', 0 }; + const char ascii_endlines [] = { '\r', '\n', 0 }; + + const String emptyString; +} + +//----------------------------------------------------------------- + +#endif + diff --git a/external/ours/library/unicode/src/shared/UnicodeBlocks.cpp b/external/ours/library/unicode/src/shared/UnicodeBlocks.cpp new file mode 100644 index 00000000..feb26ae3 --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeBlocks.cpp @@ -0,0 +1,261 @@ +// ====================================================================== +// +// UnicodeBlocks.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// This contains Unicode 3.0 compliant Blocks. +// +// ====================================================================== + +#include "FirstUnicode.h" +#include "UnicodeBlocks.h" + +// ====================================================================== + +using namespace Unicode::Blocks; +using Unicode::unicode_char_t; + +//----------------------------------------------------------------- + +const Unicode::Blocks::RawData Unicode::Blocks::Mapping::ms_defaultBlockData [] = +{ + { Basic_Latin, "Basic Latin",0x0000,0x007F }, + { Latin_1_Supplement,"Latin-1 Supplement",0x0080,0x00FF }, + { Latin_Extended_A, "Latin Extended-A",0x0100,0x017F }, + { Latin_Extended_B, "Latin Extended-B",0x0180,0x024F }, + { IPA_Extensions, "IPA Extensions",0x0250,0x02AF }, + { Spacing_Modifier_Letters, "Spacing Modifier Letters",0x02B0,0x02FF }, + { Combining_Diacritical_Marks, "Combining Diacritical Marks",0x0300,0x036F }, + { Greek, "Greek",0x0370,0x03FF }, + { Cyrillic, "Cyrillic",0x0400,0x04FF }, + { Armenian, "Armenian",0x0530,0x058F }, + { Hebrew, "Hebrew",0x0590,0x05FF }, + { Arabic, "Arabic",0x0600,0x06FF }, + { Syriac, "Syriac ",0x0700,0x074F }, + { Thaana, "Thaana",0x0780,0x07BF }, + { Devanagari, "Devanagari",0x0900,0x097F }, + { Bengali, "Bengali",0x0980,0x09FF }, + { Gurmukhi, "Gurmukhi",0x0A00,0x0A7F }, + { Gujarati, "Gujarati",0x0A80,0x0AFF }, + { Oriya, "Oriya",0x0B00,0x0B7F }, + { Tamil, "Tamil",0x0B80,0x0BFF }, + { Telugu, "Telugu",0x0C00,0x0C7F }, + { Kannada, "Kannada",0x0C80,0x0CFF }, + { Malayalam, "Malayalam",0x0D00,0x0D7F }, + { Sinhala, "Sinhala",0x0D80,0x0DFF }, + { Thai, "Thai",0x0E00,0x0E7F }, + { Lao, "Lao",0x0E80,0x0EFF }, + { Tibetan, "Tibetan",0x0F00,0x0FFF }, + { Myanmar, "Myanmar ",0x1000,0x109F }, + { Georgian, "Georgian",0x10A0,0x10FF }, + { Hangul_Jamo, "Hangul Jamo",0x1100,0x11FF }, + { Ethiopic, "Ethiopic",0x1200,0x137F }, + { Cherokee, "Cherokee",0x13A0,0x13FF }, + { Unified_Canadian_Aboriginal_Syllabics, "Unified Canadian Aboriginal Syllabics",0x1400,0x167F }, + { Ogham, "Ogham",0x1680,0x169F }, + { Runic, "Runic",0x16A0,0x16FF }, + { Khmer, "Khmer",0x1780,0x17FF }, + { Mongolian, "Mongolian",0x1800,0x18AF }, + { Latin_Extended_Additional, "Latin Extended Additional",0x1E00,0x1EFF }, + { Greek_Extended, "Greek Extended",0x1F00,0x1FFF }, + { General_Punctuation, "General Punctuation",0x2000,0x206F }, + { Superscripts_and_Subscripts, "Superscripts and Subscripts",0x2070,0x209F }, + { Currency_Symbols, "Currency Symbols",0x20A0,0x20CF }, + { Combining_Marks_for_Symbols, "Combining Marks for Symbols",0x20D0,0x20FF }, + { Letterlike_Symbols, "Letterlike Symbols",0x2100,0x214F }, + { Number_Forms, "Number Forms",0x2150,0x218F }, + { Arrows, "Arrows",0x2190,0x21FF }, + { Mathematical_Operators, "Mathematical Operators",0x2200,0x22FF }, + { Miscellaneous_Technical, "Miscellaneous Technical",0x2300,0x23FF }, + { Control_Pictures, "Control Pictures",0x2400,0x243F }, + { Optical_Character_Recognition, "Optical Character Recognition",0x2440,0x245F }, + { Enclosed_Alphanumerics, "Enclosed Alphanumerics",0x2460,0x24FF }, + { Box_Drawing, "Box Drawing",0x2500,0x257F }, + { Block_Elements, "Block Elements",0x2580,0x259F }, + { Geometric_Shapes, "Geometric Shapes",0x25A0,0x25FF }, + { Miscellaneous_Symbols, "Miscellaneous Symbols",0x2600,0x26FF }, + { Dingbats, "Dingbats",0x2700,0x27BF }, + { Braille_Patterns, "Braille Patterns",0x2800,0x28FF }, + { CJK_Radicals_Supplement, "CJK Radicals Supplement",0x2E80,0x2EFF }, + { Kangxi_Radicals, "Kangxi Radicals",0x2F00,0x2FDF }, + { Ideographic_Description_Characters, "Ideographic Description Characters",0x2FF0,0x2FFF }, + { CJK_Symbols_and_Punctuation, "CJK Symbols and Punctuation",0x3000,0x303F }, + { Hiragana, "Hiragana",0x3040,0x309F }, + { Katakana, "Katakana",0x30A0,0x30FF }, + { Bopomofo, "Bopomofo",0x3100,0x312F }, + { Hangul_Compatibility_Jamo, "Hangul Compatibility Jamo",0x3130,0x318F }, + { Kanbun, "Kanbun",0x3190,0x319F }, + { Bopomofo_Extended, "Bopomofo Extended",0x31A0,0x31BF }, + { Enclosed_CJK_Letters_and_Months, "Enclosed CJK Letters and Months",0x3200,0x32FF }, + { CJK_Compatibility, "CJK Compatibility",0x3300,0x33FF }, + { CJK_Unified_Ideographs_Extension_A, "CJK Unified Ideographs Extension A",0x3400,0x4DB5 }, + { CJK_Unified_Ideographs, "CJK Unified Ideographs",0x4E00,0x9FFF }, + { Yi_Syllables, "Yi Syllables",0xA000,0xA48F }, + { Yi_Radicals, "Yi Radicals",0xA490,0xA4CF }, + { Hangul_Syllables, "Hangul Syllables",0xAC00,0xD7A3 }, + { High_Surrogates, "High Surrogates",0xD800,0xDB7F }, + { High_Private_Use_Surrogates, "High Private Use Surrogates",0xDB80,0xDBFF }, + { Low_Surrogates, "Low Surrogates",0xDC00,0xDFFF }, + { Private_Use, "Private Use",0xE000,0xF8FF }, + { CJK_Compatibility_Ideographs, "CJK Compatibility Ideographs",0xF900,0xFAFF }, + { Alphabetic_Presentation_Forms, "Alphabetic Presentation Forms",0xFB00,0xFB4F }, + { Arabic_Presentation_Forms_A, "Arabic Presentation Forms-A",0xFB50,0xFDFF }, + { Combining_Half_Marks, "Combining Half Marks",0xFE20,0xFE2F }, + { CJK_Compatibility_Forms, "CJK Compatibility Forms",0xFE30,0xFE4F }, + { Small_Form_Variants, "Small Form Variants",0xFE50,0xFE6F }, + { Arabic_Presentation_Forms_B, "Arabic Presentation Forms-B",0xFE70,0xFEFE }, + { Specials_1, "Specials",0xFEFF,0xFEFF }, + { Halfwidth_and_Fullwidth_Forms, "Halfwidth and Fullwidth Forms",0xFF00,0xFFEF }, + { Specials_2, "Specials 2",0xFFF0,0xFFFD }, + { End_Block_Ids, "End Block Ids",0xFFFE,0xFFFF } +}; + +//----------------------------------------------------------------- + +Mapping * Mapping::ms_singleton = 0; + +//----------------------------------------------------------------- + +/** +* Initialize the singleton. This must be called before mapping is used. +*/ + +void Mapping::initSingleton () +{ + explicitDestroy (); + ms_singleton = new Mapping (ms_defaultBlockData); +} + +//----------------------------------------------------------------- + +/** +* Destroys the singleton +*/ + +void Mapping::explicitDestroy () +{ + if (ms_singleton) + delete ms_singleton; + + ms_singleton = 0; +} + +//----------------------------------------------------------------- + +/** +* Construct a Mapping from an array of RawData +*/ + +Mapping::Mapping (const RawData dataArray []) : +m_idMap (), +m_nameMap (), +m_errorData () +{ + + size_t i = 0; + for (; dataArray [i].m_id != End_Block_Ids; ++i) + { + m_idMap [dataArray[i].m_id] = Data (dataArray [i]); + m_nameMap [dataArray[i].m_name] = Data (dataArray [i]); + } + + m_errorData = Data (dataArray [i]); +} + +//---------------------------------------------------------------------- + +/** +* Construct a complete set of code points in this block. +* @param idset should enter this function empty +*/ + +const std::set & Data::generateFilteredIdSet (std::set & idset) const +{ + { + for (unicode_char_t i = m_start; i < m_end; ++i) + { + idset.insert (i); + } + } + + + for (RangeGroups_t::const_iterator iter = m_rangeGroups.begin (); iter != m_rangeGroups.end (); ++iter) + { + const RangeGroup & rg = *iter; + + if (rg.m_isInclusive) + { + for (RangeGroup::Ranges_t::const_iterator rg_iter = rg.m_ranges.begin (); rg_iter != rg.m_ranges.end (); ++rg_iter) + { + for (unicode_char_t i = (*rg_iter).m_low; i <= (*rg_iter).m_high; ++i) + { + idset.insert (i); + } + } + } + else + { + for (RangeGroup::Ranges_t::const_iterator rg_iter = rg.m_ranges.begin (); rg_iter != rg.m_ranges.end (); ++rg_iter) + { + for (unicode_char_t i = (*rg_iter).m_low; i <= (*rg_iter).m_high; ++i) + { + idset.erase (i); + } + } + } + } + + return idset; +} + +//----------------------------------------------------------------- + +/** +* Add a range group with a given string representation. Valid strings include hexadecimal numbers, dashes, and commas. +* @param inclusive indicates if this group should be additive or subtractive. It is an error to attempt to add code points outside the Block's gross range. +*/ + +bool Data::addRangeGroup (bool inclusive, const Unicode::NarrowString & str) +{ + RangeGroup rg; + rg.m_isInclusive = inclusive; + + size_t tokenStartPos = 0; + + while (tokenStartPos != Unicode::NarrowString::npos) + { + size_t tokenCommaPos = str.find_first_of (',', tokenStartPos); + + size_t tokenDashPos = str.find_first_of ('-', tokenStartPos); + + Range rng; + + // this is a dashed range + if (tokenDashPos < tokenCommaPos) + { + rng.m_low = static_cast (strtoul ( str.substr (tokenStartPos, tokenDashPos - tokenStartPos).c_str (), 0, 16)); + tokenStartPos = str.find_first_not_of ('-', tokenDashPos); + + if (tokenStartPos == Unicode::NarrowString::npos) + return false; + + rng.m_high = static_cast (strtoul (str.substr (tokenStartPos, tokenCommaPos - tokenStartPos).c_str (), 0, 16)); + + } + + else + { + rng.m_high = rng.m_low = static_cast (strtoul ( str.substr (tokenStartPos, tokenCommaPos - tokenStartPos).c_str (), 0, 16)); + } + + rg.m_ranges.push_back (rng); + + tokenStartPos = str.find_first_not_of (',', tokenCommaPos); + } + + m_rangeGroups.push_back (rg); + return true; + +} + +// ===================================================================== }, diff --git a/external/ours/library/unicode/src/shared/UnicodeBlocks.h b/external/ours/library/unicode/src/shared/UnicodeBlocks.h new file mode 100644 index 00000000..42ab4d9a --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeBlocks.h @@ -0,0 +1,396 @@ +// ====================================================================== +// +// UnicodeBlocks.h +// copyright (c) 2001 Sony Online Entertainment +// +// This contains Unicode 3.0 compliant Blocks. +// +// ====================================================================== + +#ifndef INCLUDED_UnicodeBlocks_H +#define INCLUDED_UnicodeBlocks_H + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4702) +#pragma warning (disable:4710) +#pragma warning (disable:4786) +#endif + +#include "Unicode.h" + +#include +#include +#include + +// ====================================================================== +namespace Unicode +{ + namespace Blocks + { + /** + * The enum values are precisely the beginning of the represented block range. + * These blocks are precisely the blocks defined by Unicode 3.0. + */ + enum Id + { + Basic_Latin = 0x0000, + Latin_1_Supplement = 0x0080, + Latin_Extended_A = 0x0100, + Latin_Extended_B = 0x0180, + IPA_Extensions = 0x0250, + Spacing_Modifier_Letters = 0x02B0, + Combining_Diacritical_Marks = 0x0300, + Greek = 0x0370, + Cyrillic = 0x0400, + Armenian = 0x0530, + Hebrew = 0x0590, + Arabic = 0x0600, + Syriac = 0x0700, + Thaana = 0x0780, + Devanagari = 0x0900, + Bengali = 0x0980, + Gurmukhi = 0x0A00, + Gujarati = 0x0A80, + Oriya = 0x0B00, + Tamil = 0x0B80, + Telugu = 0x0C00, + Kannada = 0x0C80, + Malayalam = 0x0D00, + Sinhala = 0x0D80, + Thai = 0x0E00, + Lao = 0x0E80, + Tibetan = 0x0F00, + Myanmar = 0x1000, + Georgian = 0x10A0, + Hangul_Jamo = 0x1100, + Ethiopic = 0x1200, + Cherokee = 0x13A0, + Unified_Canadian_Aboriginal_Syllabics = 0x1400, + Ogham = 0x1680, + Runic = 0x16A0, + Khmer = 0x1780, + Mongolian = 0x1800, + Latin_Extended_Additional = 0x1E00, + Greek_Extended = 0x1F00, + General_Punctuation = 0x2000, + Superscripts_and_Subscripts = 0x2070, + Currency_Symbols = 0x20A0, + Combining_Marks_for_Symbols = 0x20D0, + Letterlike_Symbols = 0x2100, + Number_Forms = 0x2150, + Arrows = 0x2190, + Mathematical_Operators = 0x2200, + Miscellaneous_Technical = 0x2300, + Control_Pictures = 0x2400, + Optical_Character_Recognition = 0x2440, + Enclosed_Alphanumerics = 0x2460, + Box_Drawing = 0x2500, + Block_Elements = 0x2580, + Geometric_Shapes = 0x25A0, + Miscellaneous_Symbols = 0x2600, + Dingbats = 0x2700, + Braille_Patterns = 0x2800, + CJK_Radicals_Supplement = 0x2E80, + Kangxi_Radicals = 0x2F00, + Ideographic_Description_Characters = 0x2FF0, + CJK_Symbols_and_Punctuation = 0x3000, + Hiragana = 0x3040, + Katakana = 0x30A0, + Bopomofo = 0x3100, + Hangul_Compatibility_Jamo = 0x3130, + Kanbun = 0x3190, + Bopomofo_Extended = 0x31A0, + Enclosed_CJK_Letters_and_Months = 0x3200, + CJK_Compatibility = 0x3300, + CJK_Unified_Ideographs_Extension_A = 0x3400, + CJK_Unified_Ideographs = 0x4E00, + Yi_Syllables = 0xA000, + Yi_Radicals = 0xA490, + Hangul_Syllables = 0xAC00, + High_Surrogates = 0xD800, + High_Private_Use_Surrogates = 0xDB80, + Low_Surrogates = 0xDC00, + Private_Use = 0xE000, + CJK_Compatibility_Ideographs = 0xF900, + Alphabetic_Presentation_Forms = 0xFB00, + Arabic_Presentation_Forms_A = 0xFB50, + Combining_Half_Marks = 0xFE20, + CJK_Compatibility_Forms = 0xFE30, + Small_Form_Variants = 0xFE50, + Arabic_Presentation_Forms_B = 0xFE70, + Specials_1 = 0xFEFF, + Halfwidth_and_Fullwidth_Forms = 0xFF00, + Specials_2 = 0xFFF0, + End_Block_Ids = 0xFFFE + + }; + + + //----------------------------------------------------------------- + + /** + * A simple pair representing an inclusive range of values + */ + + struct Range + { + unicode_char_t m_high; + unicode_char_t m_low; + }; + + //----------------------------------------------------------------- + + /** + * A group of ranges that can be additive or subtractive (inclusive or non) + */ + + struct RangeGroup + { + bool m_isInclusive; + typedef std::vector Ranges_t; + Ranges_t m_ranges; + }; + + //----------------------------------------------------------------- + + /** + * RawData is a simple aggregate data struct used to construct the more complex Data objects + */ + struct RawData + { + Id m_id; + const char * m_name; + unicode_char_t m_start; + unicode_char_t m_end; + }; + + /** + * Data is the representation of a single Unicode block. + * Including the Id, name, gross start-end range, and a set of RangeGroups (usually subtractive) + */ + struct Data + { + Id m_id; + Unicode::NarrowString m_name; + unicode_char_t m_start; + unicode_char_t m_end; + + typedef std::vector RangeGroups_t; + RangeGroups_t m_rangeGroups; + + explicit Data (const RawData & rhs); + Data (); + + const std::set & generateFilteredIdSet (std::set & ) const; + + bool addRangeGroup (bool inclusive, const Unicode::NarrowString & str); + + }; + + //----------------------------------------------------------------- + + /** + * Singleton class. + * + * Mapping creates a method of mapping block Ids _or_ names to Block Data objects. + * @todo: Data objects are duplicate for each map, need to optimize. + */ + + class Mapping + { + + public: + + explicit Mapping (const RawData dataArray[]); + static const Mapping & getDefaultMapping (); + static void explicitDestroy (); + + typedef std::map IdMap_t; + typedef std::map NameMap_t; + + const Data & findBlock (Id id) const; + Data & findBlock (Id id); + const Data & findBlock (const Unicode::NarrowString &) const; + const Data & findBlock (unicode_char_t code) const; + + const IdMap_t & getIdMap () const; + + bool addBlock (const Data & data); + + static const RawData ms_defaultBlockData []; + + private: + + Mapping (); + Mapping (const Mapping & rhs); + Mapping & operator= (const Mapping & rhs); + + static void initSingleton (); + + static Mapping * ms_singleton; + + IdMap_t m_idMap; + NameMap_t m_nameMap; + + Data m_errorData; + + }; + + //----------------------------------------------------------------- + + /** + * Convert a RawData into a Data object + */ + + inline Data::Data (const RawData & rhs) : + m_id (rhs.m_id), + m_name (rhs.m_name), + m_start (rhs.m_start), + m_end (rhs.m_end), + m_rangeGroups () + { + } + + //---------------------------------------------------------------------- + + /** + * Default constructor. + */ + + inline Data::Data () : + m_id (End_Block_Ids), + m_name (), + m_start (0), + m_end (0), + m_rangeGroups () + { + } + + //----------------------------------------------------------------- + + /** + * Static singleton accessor + */ + + inline const Mapping & Mapping::getDefaultMapping () + { + if (ms_singleton) + return *ms_singleton; + + initSingleton (); + return *ms_singleton; + } + + //----------------------------------------------------------------- + + /** + * Add a block Data object to the mapping. If the block already exists + * in the mapping, the new data overwrites the old. + */ + + inline bool Mapping::addBlock (const Data & data) + { + m_idMap [data.m_id] = data; + m_nameMap [data.m_name] = data; + return true; + } + + //----------------------------------------------------------------- + + /** + * Locate the block data by Id. + * @return a Data with End_Block_Ids Id if the block was not found + */ + + inline const Data & Mapping::findBlock (Id id) const + { + + const IdMap_t::const_iterator iter = m_idMap.find (id); + if (iter != m_idMap.end ()) + return (*iter).second; + else + return m_errorData; + } + + //----------------------------------------------------------------- + + /** + * Locate the block data by Id. + * @return a Data with End_Block_Ids Id if the block was not found + */ + + inline Data & Mapping::findBlock (Id id) + { + + const IdMap_t::iterator iter = m_idMap.find (id); + if (iter != m_idMap.end ()) + return (*iter).second; + else + return m_errorData; + } + + //----------------------------------------------------------------- + + /** + * Locate the block data by Name. + * @return a Data with End_Block_Ids Id if the block was not found + */ + + inline const Data & Mapping::findBlock (const Unicode::NarrowString & name) const + { + + const NameMap_t::const_iterator iter = m_nameMap.find (name); + if (iter != m_nameMap.end ()) + return (*iter).second; + else + return m_errorData; + } + + //----------------------------------------------------------------- + + /** + * Locate the block data by code point. + * @return a Data with End_Block_Ids Id if the block was not found. This only tests if the code point is in the gross range of the located block. + */ + + inline const Data & Mapping::findBlock (unicode_char_t code) const + { + + const IdMap_t::const_iterator iter = m_idMap.lower_bound (static_cast(code)); + if (iter != m_idMap.end () && code <= (*iter).second.m_end) + return (*iter).second; + else + return m_errorData; + } + + //----------------------------------------------------------------- + + /** + * Gets the Id->Data map. + */ + + inline const Mapping::IdMap_t & Mapping::getIdMap () const + { + return m_idMap; + } + + //----------------------------------------------------------------- + + /** + * Tests a character to see if belongs in an ideograph range + */ + + inline bool isIdeograph (unicode_char_t code) + { + return ((code >= static_cast(CJK_Unified_Ideographs_Extension_A) && code < static_cast(Yi_Syllables)) || + (code >= static_cast(Hangul_Syllables) && code < static_cast(CJK_Compatibility_Ideographs))); + } + + } //-- namespace Blocks + +} // -- namespace Unicode + +// ====================================================================== +#endif + diff --git a/external/ours/library/unicode/src/shared/UnicodeCharacterData.cpp b/external/ours/library/unicode/src/shared/UnicodeCharacterData.cpp new file mode 100644 index 00000000..e7cbf31c --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeCharacterData.cpp @@ -0,0 +1,61 @@ +// ====================================================================== +// +// UnicodeCharData.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// This contains Unicode 3.0 compliant Character data formats +// The data is loaded dynamically from the UnicodeData.txt definition file +// +// ====================================================================== + +#include "FirstUnicode.h" +#include "UnicodeCharacterData.h" + +// ====================================================================== + +using namespace Unicode; + +//----------------------------------------------------------------- + +#define CAT_NAME(s) { CharData::s, #s } + +const CharData::CategoryName CharData::ms_categoryNames [] = +{ + + CAT_NAME (Lu), // Letter, Uppercase + CAT_NAME (Ll), // Letter, Lowercase + CAT_NAME (Lt), // Letter, Titlecase + CAT_NAME (Lm), // Letter, Modifier + CAT_NAME (Lo), // Letter, Other + CAT_NAME (Mn), // Mark, Non-Spacing + CAT_NAME (Mc), // Mark, Spacing Combining + CAT_NAME (Me), // Mark, Enclosing + CAT_NAME (Nd), // Number, Decimal Digit + CAT_NAME (Nl), // Number, Letter + CAT_NAME (No), // Number, Other + CAT_NAME (Pc), // Punctuation, Connector + CAT_NAME (Pd), // Punctuation, Dash + CAT_NAME (Ps), // Punctuation, Open + CAT_NAME (Pe), // Punctuation, Close + CAT_NAME (Pi), // Punctuation, Initial quote (may behave like Ps or Pe depending on usage) + CAT_NAME (Pf), // Punctuation, Final quote (may behave like Ps or Pe depending on usage) + CAT_NAME (Po), // Punctuation, Other + CAT_NAME (Sm), // Symbol, Math + CAT_NAME (Sc), // Symbol, Currency + CAT_NAME (Sk), // Symbol, Modifier + CAT_NAME (So), // Symbol, Other + CAT_NAME (Zs), // Separator, Space + CAT_NAME (Zl), // Separator, Line + CAT_NAME (Zp), // Separator, Paragraph + CAT_NAME (Cc), // Other, Control + CAT_NAME (Cf), // Other, Format + CAT_NAME (Cs), // Other, Surrogate + CAT_NAME (Co), // Other, Private Use + CAT_NAME (Cn) // Other, Not Assigned (no characters in the file have this property) +}; + +#undef CAT_NAME + +//----------------------------------------------------------------- + +// ====================================================================== diff --git a/external/ours/library/unicode/src/shared/UnicodeCharacterData.h b/external/ours/library/unicode/src/shared/UnicodeCharacterData.h new file mode 100644 index 00000000..4c828ce3 --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeCharacterData.h @@ -0,0 +1,117 @@ +// ====================================================================== +// +// UnicodeCharData.h +// copyright (c) 2001 Sony Online Entertainment +// +// This contains Unicode 3.0 compliant Character data formats +// The data is loaded dynamically from the UnicodeData.txt definition file +// +// ====================================================================== + +#ifndef INCLUDED_UnicodeCharData_H +#define INCLUDED_UnicodeCharData_H + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +#pragma warning (disable:4786) +#endif + +// ====================================================================== + +namespace Unicode +{ + typedef unsigned short unicode_char_t; //lint !e761 redundant typedef + + struct CharData; + + //----------------------------------------------------------------- + + struct CharData + { + + enum Category + { + Lu, // Letter, Uppercase + Ll, // Letter, Lowercase + Lt, // Letter, Titlecase + Lm, // Letter, Modifier + Lo, // Letter, Other + Mn, // Mark, Non-Spacing + Mc, // Mark, Spacing Combining + Me, // Mark, Enclosing + Nd, // Number, Decimal Digit + Nl, // Number, Letter + No, // Number, Other + Pc, // Punctuation, Connector + Pd, // Punctuation, Dash + Ps, // Punctuation, Open + Pe, // Punctuation, Close + Pi, // Punctuation, Initial quote (may behave like Ps or Pe depending on usage) + Pf, // Punctuation, Final quote (may behave like Ps or Pe depending on usage) + Po, // Punctuation, Other + Sm, // Symbol, Math + Sc, // Symbol, Currency + Sk, // Symbol, Modifier + So, // Symbol, Other + Zs, // Separator, Space + Zl, // Separator, Line + Zp, // Separator, Paragraph + Cc, // Other, Control + Cf, // Other, Format + Cs, // Other, Surrogate + Co, // Other, Private Use + Cn // Other, Not Assigned (no characters in the file have this property) + }; + + + struct CategoryName + { + Category m_category; + const char * m_str; + }; + + static const CategoryName ms_categoryNames[]; + + unicode_char_t m_code; + Category m_category; + + /** + * Note: this Unicode system only supports upper and lower case. + * Titlecase is considered uppercase. The case of a character + * is determined by its category (Lu,Ll,or Lt) + */ + + unicode_char_t m_reverseCase; + + bool isLowerCase () const; + bool isUpperCase () const; + + unicode_char_t toUpper () const; + unicode_char_t toLower () const; + }; + + //----------------------------------------------------------------- + + /** + * Is this character lowercase. + */ + inline bool CharData::isLowerCase () const + { + return m_category == Ll; + } + + //----------------------------------------------------------------- + + /** + * Returns true for upper and title case + */ + inline bool CharData::isUpperCase () const + { + return m_category == Lu || m_category == Lt; + } +} + +// ====================================================================== + +#endif diff --git a/external/ours/library/unicode/src/shared/UnicodeCharacterDataMap.cpp b/external/ours/library/unicode/src/shared/UnicodeCharacterDataMap.cpp new file mode 100644 index 00000000..2be6eab9 --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeCharacterDataMap.cpp @@ -0,0 +1,347 @@ +// ====================================================================== +// +// UnicodeCharDataMap.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// This contains Unicode 3.0 compliant Character data formats +// The data is loaded dynamically from the UnicodeData.txt definition file +// +// ====================================================================== + +#include "FirstUnicode.h" +#include "UnicodeCharacterDataMap.h" +#include "UnicodeBlocks.h" +#include "UnicodeCharacterData.h" + +#include +#include +#include + +// ====================================================================== + +using namespace Unicode; +CharDataMap * CharDataMap::ms_singleton = 0; + +//----------------------------------------------------------------- + +CharDataMap::CharDataMap () : +m_blockIds (), +m_contiguousData (0), +m_map (), +m_dataIsValid (false) +{ + +} + +//----------------------------------------------------------------- + +CharDataMap::~CharDataMap () +{ + m_map.clear (); + m_blockIds.clear (); + + if (m_contiguousData) + { + delete[] m_contiguousData; + m_contiguousData = 0; + } +} + +//----------------------------------------------------------------- + +void CharDataMap::initSingleton () +{ + explicitDestroy (); + + // TODO: default blocks? + ms_singleton = new CharDataMap (); +} + +//----------------------------------------------------------------- + +void CharDataMap::explicitDestroy () +{ + if (ms_singleton) + delete ms_singleton; + + ms_singleton = 0; +} + +//----------------------------------------------------------------- + +namespace +{ + + typedef std::map CategoryMap_t; + + //----------------------------------------------------------------- + + + /** + * Load a CharData from a string buffer representing a file. + * The bufpos is set to the beginning of a line after this method returns. + */ + + + inline bool populateDataFromBuffer (size_t desiredCode, + CharData & charData, + const std::string & buffer, + size_t & bufpos, + const CategoryMap_t & cmap) + { + + size_t endpos = 0; + + bool found = false; + + do + { + size_t tokenNum = 0; + + while ( (endpos = buffer.find ( ';', bufpos)) != std::string::npos ) //lint !e737 // loss of sign in promotion - STL bug + { + // code value + if (tokenNum == 0) + { + const size_t code = static_cast (strtoul (buffer.substr (bufpos, endpos - bufpos).c_str (), 0, 16)); + + // we passed the desired line, so it must not exist + if (code > desiredCode) + return found; + // not there yet + else if (code < desiredCode) + break; + + charData.m_code = static_cast (code); + } + + // General Category + else if (tokenNum == 2) + { + // all category codes are currently 2 characters long + if (endpos - bufpos > 2) + break; + + const CategoryMap_t::const_iterator find_iter = cmap.find (buffer.substr (bufpos, 2)); + + if (find_iter == cmap.end ()) + { + break; + } + else + charData.m_category = (*find_iter).second; + } + + // Uppercase mapping + else if (charData.isLowerCase ()) + { + if (tokenNum == 12) + { + if (endpos - bufpos > 1) + charData.m_reverseCase = static_cast (strtoul (buffer.substr (bufpos, endpos - bufpos).c_str (), 0, 16)); + + found = true; + break; + } + } + // Lowercase mapping + else if (tokenNum == 13) + { + if (endpos - bufpos > 1) + charData.m_reverseCase = static_cast (strtoul (buffer.substr (bufpos, endpos - bufpos).c_str (), 0, 16)); + + found = true; + break; + } + + bufpos = endpos + 1; + ++tokenNum; + + } + } + while ( (bufpos = buffer.find ('\n', bufpos)) != std::string::npos && //lint !e737 // loss of sign in promotion - STL bug + (bufpos = buffer.find_first_not_of ('\n', bufpos + 1)) != std::string::npos); //lint !e737 // loss of sign in promotion - STL bug + + return found; + + } +} + +//----------------------------------------------------------------- + +/** +* Create a map of CharData's from a UnicodeData.txt standard Unicode 3.0 file, which has been packed into a buffer +*/ + +CharDataMap::ErrorCode CharDataMap::generateMapFromBuffer (const Unicode::Blocks::Mapping & blockMapping, const std::string & str_buf) +{ + m_dataIsValid = false; + + m_map.clear (); + + //-- create the category map for use by the inline population function + CategoryMap_t cmap; + + { + size_t i = 0; + for (; CharData::ms_categoryNames [i].m_category != CharData::Cn; ++i) + { + cmap [CharData::ms_categoryNames [i].m_str] = CharData::ms_categoryNames [i].m_category; + } + } + + typedef std::set IdSet_t; + + IdSet_t idSet; + + { + for (BlockIdSet_t::const_iterator iter = m_blockIds.begin (); iter != m_blockIds.end (); ++iter) + { + const Blocks::Data & blockData = blockMapping.findBlock (*iter); + + if (blockData.m_id != Blocks::End_Block_Ids) + { + blockData.generateFilteredIdSet (idSet); + } + } + } + + //-- read all the desired chars from file. + //-- the file must be sorted by code + + size_t bufpos = 0; + size_t validChars = 0; + + { + + for (IdSet_t::const_iterator iter = idSet.begin (); iter != idSet.end (); ++iter) + { + CharData * data = new CharData; + + assert (data != NULL); //lint !e1924 // c-style cast. MSVC bug + + data->m_reverseCase = 0; + + if (populateDataFromBuffer (*iter, *data, str_buf, bufpos, cmap) == false) + { + delete data; + continue; + } + + else + { + ++validChars; + m_map[data->m_code] = data; + } + } + } + + //-- + //-- pack all the char data into a contiguous block of memory + //-- + + if (m_contiguousData) + delete[] m_contiguousData; + + m_contiguousData = new CharData [validChars]; + + assert (m_contiguousData != NULL); //lint !e1924 // c-style cast. MSVC bug + + size_t dataIndex = 0; + + for (Map_t::iterator iter = m_map.begin (); iter != m_map.end (); ++iter, ++dataIndex) + { + m_contiguousData [dataIndex] = *(*iter).second; + delete (*iter).second; + (*iter).second = &m_contiguousData [dataIndex]; + } + + m_dataIsValid = true; + + return ERR_SUCCESS; +} + +//---------------------------------------------------------------------- + +/** +* Create a map of CharData's from a UnicodeData.txt standard Unicode 3.0 file, using cstdio +*/ + +CharDataMap::ErrorCode CharDataMap::generateMap (const Unicode::Blocks::Mapping & blockMapping, const char * filename) +{ + //-- load data from file as a big std::string + //-- read all the desired chars from file. + //-- the file must be sorted by code + + FILE * fl = fopen (filename, "rb"); + + if (fl == 0) + { + return ERR_NO_FILE; + } + + fseek (fl, 0, SEEK_END); + + size_t fileLen = static_cast(ftell (fl)); + + fseek (fl, 0, SEEK_SET); + + char * buffer = new char [fileLen + 1]; + buffer [fileLen] = 0; + + assert (buffer != NULL); //lint !e1924 // c-style cast. MSVC bug + + if (fread (buffer, fileLen, 1, fl) != 1) + { + delete[] buffer; + fclose (fl); + return ERR_READ_FAILED; + } + + if (fclose (fl)) + { + delete[] buffer; + return ERR_CLOSE_FAILED; + } + + std::string str_buf (buffer, fileLen); + + delete[] buffer; + + return generateMapFromBuffer (blockMapping, str_buf); +} + +//----------------------------------------------------------------- + +/** +* Add a block Id that this CharacterDataMap is concerned with. Does not change the data map. +*/ + +void CharDataMap::addBlock (Blocks::Id id) +{ + m_blockIds.insert (id); +} + +//----------------------------------------------------------------- + +/** +* Remove a block Id that this CharacterDataMap is concerned with. Does not change the data map. +*/ + +void CharDataMap::removeBlock (Blocks::Id id) +{ + m_blockIds.erase (id); +} + +//----------------------------------------------------------------- + +/* +* Add a block Id that this CharacterDataMap is concerned with. Does not change the data map. +*/ + +void CharDataMap::clearBlocks () +{ + m_blockIds.clear (); +} + +// ====================================================================== diff --git a/external/ours/library/unicode/src/shared/UnicodeCharacterDataMap.h b/external/ours/library/unicode/src/shared/UnicodeCharacterDataMap.h new file mode 100644 index 00000000..ba84cac2 --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeCharacterDataMap.h @@ -0,0 +1,148 @@ +// ====================================================================== +// +// UnicodeCharDataMap.h +// copyright (c) 2001 Sony Online Entertainment +// +// This contains Unicode 3.0 compliant Character data formats +// The data is loaded dynamically from the UnicodeData.txt definition file +// +// ====================================================================== + +#ifndef INCLUDED_UnicodeCharDataMap_H +#define INCLUDED_UnicodeCharDataMap_H + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +#pragma warning (disable:4786) +#endif + +#include +#include + +#include "UnicodeBlocks.h" + +// ====================================================================== + +namespace Unicode +{ + struct CharData; + + //----------------------------------------------------------------- + + /** + * Singleton class that allows mapping from code points to CharData objects. This will be + * used to change case of characters, character category testing, etc... + * A CharacterDataMap has a set of Unicode::Block::Id's with which it is solely concerned. + * In this way, you may instantiate many different CharDataMaps if needed. + */ + + class CharDataMap + { + public: + + enum ErrorCode + { + ERR_SUCCESS = 0, + ERR_NO_FILE, + ERR_READ_FAILED, + ERR_CLOSE_FAILED, + ERR_CHAR_NOT_FOUND, + ERR_FILE_TOO_SHORT + }; + + typedef std::map Map_t; + + static CharDataMap & getDefaultMap (); + static void explicitDestroy (); + + CharDataMap (); + ~CharDataMap (); + + void addBlock (Blocks::Id id); + void removeBlock (Blocks::Id id); + void clearBlocks (); + + ErrorCode generateMap (const Blocks::Mapping & blockMapping, const char * filename); + ErrorCode generateMapFromBuffer (const Blocks::Mapping & blockMapping, const std::string & str_buf); + + bool isValid () const; + + const CharData * findCharData (unicode_char_t code) const; + + const Map_t & getMap () const; + + private: + + static void initSingleton (); + + typedef std::set BlockIdSet_t; + + BlockIdSet_t m_blockIds; + CharData * m_contiguousData; + Map_t m_map; + bool m_dataIsValid; + + static CharDataMap * ms_singleton; + + }; + + //----------------------------------------------------------------- + + /** + * Singleton accessor. + */ + + inline CharDataMap & CharDataMap::getDefaultMap () + { + if (ms_singleton) + return *ms_singleton; + + initSingleton (); + return *ms_singleton; + } + + //----------------------------------------------------------------- + + /** + * Has the data been loaded successfully? + */ + + inline bool CharDataMap::isValid () const + { + return m_dataIsValid; + } + + //----------------------------------------------------------------- + + /** + * Find a CharData for the given code point. + * @return null if no CharData exists for this code point + */ + + inline const CharData * CharDataMap::findCharData (const unicode_char_t code) const + { + if (isValid ()) + { + const Map_t::const_iterator iter = m_map.find (code); + if (iter != m_map.end ()) + return (*iter).second; + } + + return 0; + } + + //----------------------------------------------------------------- + + /** + * Get the data mapping + */ + + inline const CharDataMap::Map_t & CharDataMap::getMap () const + { + return m_map; + } +} +// ====================================================================== + +#endif diff --git a/external/ours/library/unicode/src/shared/UnicodeUtils.cpp b/external/ours/library/unicode/src/shared/UnicodeUtils.cpp new file mode 100644 index 00000000..4417fc0c --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeUtils.cpp @@ -0,0 +1,610 @@ +// ====================================================================== +// +// Unicode.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstUnicode.h" +#include "UnicodeUtils.h" + +#include +#include +#include + + +// ====================================================================== + +namespace Unicode +{ + + //----------------------------------------------------------------- + + String utf8ToWide (const UTF8String & nstr) + { + String result; + utf8ToWide (nstr, result); + return result; + } + + String & utf8ToWide (const UTF8String & nstr, String & str) + { + str.clear(); + const UTF8String::const_iterator end = nstr.end(); + for (UTF8String::const_iterator from = nstr.begin(); from != end; ++from) + { + char first = *from; + const int len = UTF8_charSize(&first); + // if 1-byte, do 1-byte pad + if (len == 1) + { + str += static_cast(first); + } + // else if 3-byte conversion + else if (len == 3) + { + ++from; + if (from != end && + *from != 0) + { + const char second = *from++; + if (from != end && + *from != 0) + { + str += static_cast((static_cast( first & 0x0F ) << 12 ) + + (static_cast( second & 0x3F ) << 6 ) + + (( *from & 0x3F ))); + } + else + { + break; + } + } + else + { + break; + } + } + // else if 2-byte conversion + else + { + ++from; + // Make sure we didn't reach the end of a string in the middle of a character + if (from != end && + *from != 0) + { + str += static_cast((static_cast(first & 0x1F) << 6 ) + + (*from &0x3F)); + } + else + { + break; + } + } + } + + return str; + } + + + /** + * Convert a Unicode string to UTF8 format. + */ + UTF8String & wideToUTF8(const String & nstr, UTF8String & str) + { + str.clear(); + str.reserve(nstr.size() * 3 + 2); + for (String::const_iterator from = nstr.begin(); from != nstr.end(); ++from) + { + if (*from >= 0x0001 && *from <= 0x007f) + { + // 1-byte packing + str += static_cast(((*from) & 0x007f)); + } + else if (*from == 0x0000) + { + // null character + str += static_cast(0x0000); + } + else if (*from >= 0x0800) + { + // 3-byte packing + str += static_cast(0x00e0 | (((*from) >> 12) & 0x0f)); + str += static_cast(0x0080 | (((*from) >> 6) & 0x3f)); + str += static_cast(0x0080 | ((*from) & 0x3f)); + } + else + { + // 2-byte packing + str += static_cast(0x00c0 | (((*from) >> 6) & 0x1f)); + str += static_cast(0x0080 | ((*from) & 0x3f)); + } + } + return str; + } + + UTF8String wideToUTF8 (const String & nstr) + { + UTF8String result; + wideToUTF8 (nstr, result); + return result; + } + + /** + * Return by value a lowered version of nstr + */ + + NarrowString toLower (const NarrowString & nstr) + { + NarrowString retval (nstr); + std::transform (retval.begin (), retval.end (), retval.begin (), tolower); + return retval; + } + + //----------------------------------------------------------------- + + /** + * Return by value an uppered version of nstr + */ + + NarrowString toUpper (const NarrowString & nstr) + { + NarrowString retval (nstr); + std::transform (retval.begin (), retval.end (), retval.begin (), toupper); + return retval; + } + + //----------------------------------------------------------------- + + wchar_t toLowerWide (wchar_t inp) + { + if((inp >= wchar_t('A')) && (inp <= wchar_t('Z'))) + return static_cast((inp - wchar_t('A') + wchar_t('a'))); + return inp; + } + + //----------------------------------------------------------------- + + wchar_t toUpperWide (wchar_t inp) + { + if((inp >= wchar_t('a')) && (inp <= wchar_t('z'))) + return static_cast((inp - wchar_t('a') + wchar_t('A'))); + return inp; + } + + /** + * Return by value a lowered version of str. + * This currently only works properly with ASCII. + * @todo: implement this for all unicode blocks. + */ + + String toLower (const String & nstr) + { + String retval (nstr); + std::transform (retval.begin (), retval.end (), retval.begin (), toLowerWide); + return retval; + } + + //----------------------------------------------------------------- + + /** + * Return by value an uppered version of str. + * This currently only works properly with ASCII. + * @todo: implement this for all unicode blocks. + */ + + String toUpper (const String & nstr) + { + String retval (nstr); + std::transform (retval.begin (), retval.end (), retval.begin (), toUpperWide); + return retval; + } + + //----------------------------------------------------------------- + + /** + * Append src to dst, padding the field as needed, and truncating the field if desired. + */ + + String & appendStringField (String & dst, const String & src, size_t width, FieldAlignment fa, unicode_char_t pad, bool truncate) + { + + if (src.length () > width && truncate) + return dst.append (src.substr (0, width)); + + if (src.length () >= width) + return dst.append (src); + + const size_t diff = width - src.length (); + + if (fa == FA_RIGHT) + dst.append (diff, pad); + else if (fa == FA_CENTER) + dst.append (diff/2, pad); + + dst.append (src); + + if (fa == FA_LEFT) + dst.append (diff, pad); + else if (fa == FA_CENTER) + dst.append (diff - diff/2, pad); + + return dst; + } + + //----------------------------------------------------------------- + + template bool getFirstTokenTemplate (const T & str, size_t pos, size_t & endpos, T & token, const U * sepChars) + { + const size_t first_nonspace = str.find_first_not_of ( sepChars, pos ); + + if (first_nonspace != str.npos) + { + const size_t first_space = str.find_first_of (sepChars, first_nonspace ); + + if (first_space != str.npos) + token = str.substr (first_nonspace, first_space - first_nonspace); + else + token = str.substr (first_nonspace); + + endpos = first_space; + return true; + } + + return false; + } + + //----------------------------------------------------------------- + + /** + * Find the first token in str, starting at pos. The value of the token is placed in the token field. endpos is updated to point past the token. + */ + + bool getFirstToken (const Unicode::String & str, size_t pos, size_t & endpos, Unicode::String & token, const unicode_char_t * sepChars) + { + return getFirstTokenTemplate (str, pos, endpos, token, sepChars); + } + + //----------------------------------------------------------------- + + /** + * Find the first token in str, starting at pos. The value of the token is placed in the token field. endpos is updated to point past the token. + */ + + bool getFirstToken (const Unicode::NarrowString & str, size_t pos, size_t & endpos, Unicode::NarrowString & token, const char * sepChars) + { + return getFirstTokenTemplate (str, pos, endpos, token, sepChars); + } + + //----------------------------------------------------------------- + + /** + * Find the nth token in str, starting at pos. The value of the token is placed in the token field. endpos is updated to point past the token. + * pos is updated to point to the first character of token in str. + */ + + template bool getNthTokenTemplate (const T & str, const size_t n, size_t & pos, size_t & endpos, T & token, const U * sepChars) + { + size_t desired_first_nonspace = 0; + size_t space = 0; + + //check for trivial case + if(n == 0) + desired_first_nonspace = pos; + else + { + for (size_t i = 0; i < n; ++i) + { + //find the first whitespace character + space = str.find_first_of ( sepChars, pos ); + if (space == str.npos) + return false; + //now find the next non-whitespace character after it + desired_first_nonspace = str.find_first_not_of( sepChars, space ); + if (desired_first_nonspace == str.npos) + return false; + pos = desired_first_nonspace; + } + } + //find the end of the token + const size_t first_space = str.find_first_of (sepChars, desired_first_nonspace ); + //now get that token + if (first_space != str.npos) + token = str.substr (desired_first_nonspace, first_space - desired_first_nonspace); + else + token = str.substr (desired_first_nonspace); + + endpos = first_space; + + if (pos == endpos) + return false; + + return true; + } + + //---------------------------------------------------------------------- + + /** + * Find the nth token in str, starting at pos. The value of the token is placed in the token field. endpos is updated to point past the token. + * pos is updated to point to the first character of token in str. + */ + + bool getNthToken (const Unicode::String & str, const size_t n, size_t & pos, size_t & endpos, Unicode::String & token, const unicode_char_t * sepChars) + { + return getNthTokenTemplate (str, n, pos, endpos, token, sepChars); + } + + //----------------------------------------------------------------- + + /** + * Find the nth token in str, starting at pos. The value of the token is placed in the token field. endpos is updated to point past the token. + * pos is updated to point to the first character of token in str. + */ + + bool getNthToken (const Unicode::NarrowString & str, const size_t n, size_t & pos, size_t & endpos, Unicode::NarrowString & token, const char * sepChars) + { + return getNthTokenTemplate (str, n, pos, endpos, token, sepChars); + } + + //----------------------------------------------------------------- + + bool tokenize (const Unicode::String & theStr, UnicodeStringVector & result, Unicode::String const * const delimiters, Unicode::String const * const separators) + { + size_t tokenPos = 0; + size_t whichToken = 0; + size_t token_start = 0; + size_t token_end = 0; + + return tokenize(theStr, result, tokenPos, whichToken, token_start, token_end, delimiters, separators); + } + + //----------------------------------------------------------------- + + + bool tokenize (const Unicode::String & theStr, UnicodeStringVector & result, const size_t tokenPos, size_t & whichToken, size_t & token_start, size_t & token_end, Unicode::String const * const delimiters, Unicode::String const * const separators) + { + size_t end_pos = 0; + size_t start_pos = 0; + whichToken = 0; + + result.clear (); + + Unicode::String str = Unicode::getTrim (theStr); + + if(str.empty()) + return false; + + //-- skip the command character + if (str [0] == '/') + str.erase (0, 1); + + //-- Setup the separator marker. + size_t sep_pos = static_cast(str.npos); + + for (;;) + { + if (end_pos >= str.size ()) + break; + + if(separators) + { + sep_pos = str.find_first_of(*separators, end_pos); + } + + if(delimiters) + { + start_pos = str.find_first_not_of(*delimiters, end_pos); + } + else + { + start_pos = str.find_first_not_of (' ', end_pos); + } + + if (start_pos == str.npos) + break; + + bool separatorFound = false; + if(sep_pos != str.npos) + { + if(sep_pos <= start_pos || start_pos == str.npos) + { + start_pos = sep_pos; + end_pos = start_pos + 1; + + if (end_pos >= str.size()) + end_pos = str.size() - 1; + + separatorFound = true; + } + } + + + //---------------------------------------------------------------------- + + bool hasQuotes = false; + + if (!separatorFound) + { + if (str [start_pos] == '\"') + { + if (++start_pos >= str.size ()) + break; + end_pos = str.find_first_of ('\"', start_pos); + hasQuotes = true; + } + else + { + if(delimiters) + { + end_pos = str.find_first_of(*delimiters, start_pos); + } + else + { + end_pos = str.find_first_of (' ', start_pos); + } + + if(sep_pos != str.npos && sep_pos > 0) + { + if(sep_pos < end_pos) + { + end_pos = sep_pos; + } + } + } + } + + //---------------------------------------------------------------------- + + if (start_pos == end_pos) + { + if (hasQuotes) + { + // Empty string "", which is a valid token + + result.push_back (Unicode::String()); + ++end_pos; + continue; + } + + break; + } + + if (start_pos <= tokenPos) + { + whichToken = result.size (); + token_start = start_pos; + token_end = end_pos; + } + + if (end_pos == str.npos) + { + result.push_back (str.substr (start_pos)); + break; + } + else + { + result.push_back (str.substr (start_pos, end_pos - start_pos)); + if (hasQuotes) + ++end_pos; + } + + ++start_pos; + } + + return true; + } + + //----------------------------------------------------------------- + + float toFloat (const Unicode::String &str) + { + float result = 0.0f; + + if (!str.empty()) + { + result = static_cast(atof(Unicode::wideToNarrow(str).c_str())); + } + + return result; + } + + //----------------------------------------------------------------- + + int toInt (const Unicode::String &str) + { + int result = 0; + + if (!str.empty()) + { + result = atoi(Unicode::wideToNarrow(str).c_str()); + } + + return result; + } + + //----------------------------------------------------------------- + + bool toBool (const Unicode::String &str) + { + bool result = 0; + + if (!str.empty()) + { + result = (atoi(Unicode::wideToNarrow(str).c_str()) != 0); + } + + return result; + } + + //----------------------------------------------------------------- + + Unicode::String intToWide(int value) + { + Unicode::String str; + Unicode::intToWide(value, str); + return str; + } + + //----------------------------------------------------------------- + + void intToWide(int value, Unicode::String & str) + { + char buffer[16]; + sprintf(buffer, "%d", value); + str = Unicode::narrowToWide(buffer); + } + + //----------------------------------------------------------------- + + bool isUnicode (const Unicode::String & theStr) + { + for(Unicode::String::const_iterator i = theStr.begin(); i != theStr.end(); ++i) + { + if(*i > 255) + { + return true; + } + } + return false; + } + + //----------------------------------------------------------------- + + /** + * Return the size of the string if it were stored as UTF-8. Useful + * since we store all Unicode values as UTF-8 in the DB by handling + * the conversion ourselves instead of letting Oracle handle it. + */ + unsigned int utf8Size(const Unicode::String & str) + { + unsigned int count = 0; + + for (String::const_iterator from = str.begin(); from != str.end(); ++from) + { + Unicode::unicode_char_t value = (*from); + count += utf8CharSize(value); + } + + return count; + } + + //----------------------------------------------------------------- + + /** + * Set str to be a valid string after being truncated to size + * if str were stored as UTF-8. This means that we want to + * convert str to UTF-8, truncate to size, then throw out any + * extra bytes left on the end by a UTF-8 (multi-byte) value + * being truncated in the middle of its encoding. We take + * advantage of Unicode::utf8ToWide() guaranteeing that the + * extra bytes are thrown out. + */ + void truncateToUTF8Size(Unicode::String & str, unsigned int size) + { + if (size < utf8Size(str)) + str = Unicode::utf8ToWide(Unicode::wideToUTF8(str).substr(0, size)); + } + +} + +// ====================================================================== diff --git a/external/ours/library/unicode/src/shared/UnicodeUtils.h b/external/ours/library/unicode/src/shared/UnicodeUtils.h new file mode 100644 index 00000000..48f88155 --- /dev/null +++ b/external/ours/library/unicode/src/shared/UnicodeUtils.h @@ -0,0 +1,376 @@ +// ====================================================================== +// +// UnicodeUtils.h +// copyright (c) 2001 Sony Online Entertainment +// +// jwatson +// +// Basic Unicode string handling/manipulating functions +// ====================================================================== + +#ifndef INCLUDED_UnicodeUtils_H +#define INCLUDED_UnicodeUtils_H + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +#pragma warning (disable:4786) +#endif + +#include "Unicode.h" +#include "utf8.h" + +namespace std +{ + template class vector; +} + +template > struct unicode_stdvector +{ + typedef std::vector<_Tp, _Alloc> fwd; +}; + +//----------------------------------------------------------------- + +namespace Unicode +{ + typedef std::string UTF8String; + typedef unicode_stdvector::fwd UnicodeStringVector; + typedef unicode_stdvector::fwd UnicodeNarrowStringVector; + + String narrowToWide (const char * const nstr); + String narrowToWide (const NarrowString & nstr); + String & narrowToWide (const NarrowString & nstr, String & str); //lint !e1929 // function returning a reference + String & narrowToWide (const char * const nstr, String & str); //lint !e1929 // function returning a reference + + NarrowString wideToNarrow (const String & nstr); + NarrowString & wideToNarrow (const String & nstr, NarrowString & str); //lint !e1929 // function returning a reference + + String utf8ToWide (const UTF8String & nstr); + String & utf8ToWide (const UTF8String & nstr, String & str); //lint !e1929 // function returning a reference + UTF8String wideToUTF8 (const String & nstr); + UTF8String & wideToUTF8 (const String & nstr, UTF8String & str); //lint !e1929 // function returning a reference + + NarrowString toLower (const NarrowString & nstr); + NarrowString toUpper (const NarrowString & nstr); + String toLower (const String & nstr); + String toUpper (const String & nstr); + + float toFloat (const Unicode::String &str); + int toInt (const Unicode::String &str); + bool toBool (const Unicode::String &str); + + String intToWide(int value); + void intToWide(int value, String & str); + + const String getTrim (const Unicode::String & str, const unicode_char_t * white = whitespace); + String & trim (Unicode::String & str, const unicode_char_t * white = whitespace); + + bool getFirstToken (const Unicode::String & str, size_t pos, size_t & endpos, Unicode::String & token, const unicode_char_t * sepChars = whitespace); + bool getNthToken (const Unicode::String & str, const size_t n, size_t & pos, size_t & endpos, Unicode::String & token, const unicode_char_t * sepChars = whitespace); + size_t skipWhitespace (const Unicode::String & str, size_t pos, const unicode_char_t * white = whitespace); + + const NarrowString getTrim (const Unicode::NarrowString & str, const char * white = ascii_whitespace); + NarrowString & trim (Unicode::NarrowString & str, const char * white = ascii_whitespace); + + bool getFirstToken (const Unicode::NarrowString & str, size_t pos, size_t & endpos, Unicode::NarrowString & token, const char * sepChars = ascii_whitespace); + bool getNthToken (const Unicode::NarrowString & str, const size_t n, size_t & pos, size_t & endpos, Unicode::NarrowString & token, const char * sepChars = ascii_whitespace); + size_t skipWhitespace (const Unicode::NarrowString & str, size_t pos, const char * white = ascii_whitespace); + + bool tokenize (const Unicode::String & theStr, UnicodeStringVector & result, Unicode::String const * const delimiters = NULL, Unicode::String const * const separators = NULL); + bool tokenize (const Unicode::String & theStr, UnicodeStringVector & result, const size_t tokenPos, size_t & whichToken, size_t & token_start, size_t & token_end, Unicode::String const * const delimiters = NULL, Unicode::String const * const separators = NULL); + + bool isUnicode (const Unicode::String & theStr); + + unsigned int utf8Size(const Unicode::String & str); + unsigned short utf8CharSize(const Unicode::unicode_char_t value); + void truncateToUTF8Size(Unicode::String & str, unsigned int size); + + enum FieldAlignment + { + FA_LEFT, + FA_RIGHT, + FA_CENTER + }; + + String & appendStringField (String & dst, const String & src, size_t width, FieldAlignment fa = FA_LEFT, unicode_char_t pad = ' ', bool truncate = false); + String & appendStringField (String & dst, const NarrowString & src, size_t width, FieldAlignment fa = FA_LEFT, unicode_char_t pad = ' ', bool truncate = false); + + /** + * Compare substrings of str2 and str1, each starting with pos and containing n characters + */ + + template bool caseInsensitiveCompare (const String & str1, const T & str2, size_t pos, size_t n) + { + const size_t len1 = str1.size (); + const size_t len2 = str2.size (); + + if (pos >= len1 || pos >= len2 || pos + n > len1 || pos + n > len2) + return false; + + const size_t end = pos + n; + for (size_t i = pos; i < end; ++i) + { + if (tolower (str1 [i]) != tolower (str2 [i])) + return false; + } + + return true; + } + + /** + * Compare str1 and str2. + */ + + template bool caseInsensitiveCompare (const String & str1, const T & str2) + { + const size_t len1 = str1.size (); + + if (len1 != str2.size ()) + return false; + + return caseInsensitiveCompare (str1, str2, 0, len1); + } + + /** + * Optimized implementation of isWhitespace. Must be kept in line with Unicode::whitespace array + */ + + template bool isWhitespace (T c) + { + return c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == 0x3000; + } + + /* + * @todo: uncomment when caseInsensitiveCompare matures + * + template class CompareNoCasePredicate + { + public: + bool operator()( T & a, T & b ) const + { + return caseInsensitiveCompare (a, b) < 0; + }; + }; + + template class EqualsNoCasePredicate + { + public: + bool operator()( T & a, T & b ) const + { + return caseInsensitiveCompare (a, b) == 0; + }; + }; + + */ +} + +//----------------------------------------------------------------- +//-- implementation +//----------------------------------------------------------------- + +namespace Unicode +{ + //----------------------------------------------------------------- + + /** + * Utility to convert a string and obtain the result by value + */ + + inline String narrowToWide (const char * const nstr) + { + String result; + narrowToWide (nstr, result); + return result; + } + + //---------------------------------------------------------------------- + + /** + * Utility to convert a string and obtain the result by value + */ + + inline String narrowToWide (const NarrowString & nstr) + { + String result; + narrowToWide (nstr, result); + return result; + } + + //---------------------------------------------------------------------- + + inline String & narrowToWide (const char * const nstr, String & str) + { + std::basic_string ucstr (nstr, nstr + strlen (nstr)); + return str.assign (ucstr.begin (), ucstr.end ()); + } + + //----------------------------------------------------------------- + + /** + * Utility to convert a string and obtain the result by reference + */ + + inline String & narrowToWide (const NarrowString & nstr, String & str) + { //lint !e1929 // function returning a reference + std::basic_string ucstr (nstr.begin (), nstr.end ()); + return str.assign (ucstr.begin (), ucstr.end ()); + } + + //----------------------------------------------------------------- + + /** + * Utility to convert a string and obtain the result by value + * This should only be used when the Unicode string is known to contain only 8 bit assignable values + */ + + inline NarrowString wideToNarrow (const String & str) + { + return NarrowString (str.begin (), str.end ()); + } + + //----------------------------------------------------------------- + + /** + * Utility to convert a string and obtain the result by reference + * This should only be used when the Unicode string is known to contain only 8 bit assignable values + */ + inline NarrowString & wideToNarrow (const String & str, NarrowString & nstr) + { //lint !e1929 // function returning a reference + return nstr.assign (str.begin (), str.end ()); + } + + /** + * Get the trimmed version of str by value. + */ + + inline const String getTrim (const Unicode::String & str, const unicode_char_t * white) + { + size_t const first_nonspace = str.find_first_not_of(white); + + if (first_nonspace == str.npos) + { + // There is only whitespace + + return String(); + } + + size_t const last_nonspace = str.find_last_not_of(white); + + if (last_nonspace == str.npos) + { + // There is only leading whitespace + + return str.substr(first_nonspace, last_nonspace); + } + + // There is both leading and trailing whitespace + + return str.substr(first_nonspace, (last_nonspace - first_nonspace) + 1); + } + + //----------------------------------------------------------------- + + /** + * Trim the specified string and return a reference to it. + */ + + inline String & trim (Unicode::String & str, const unicode_char_t * white) + { + return (str = getTrim (str, white)); + } + + /** + * Get the trimmed version of str by value. + */ + + inline const NarrowString getTrim (const Unicode::NarrowString & str, const char * white) + { + size_t const first_nonspace = str.find_first_not_of(white); + + if (first_nonspace == str.npos) + { + // There is only whitespace + + return ""; + } + + size_t const last_nonspace = str.find_last_not_of(white); + + if (last_nonspace == str.npos) + { + // There is only leading whitespace + + return str.substr(first_nonspace, last_nonspace); + } + + // There is both leading and trailing whitespace + + return str.substr(first_nonspace, (last_nonspace - first_nonspace) + 1); + } + + //----------------------------------------------------------------- + + /** + * Trim the specified string and return a reference to it. + */ + + inline NarrowString & trim (Unicode::NarrowString & str, const char * white) + { + return (str = getTrim (str, white)); + } + + /** + * Return the first non-white position starting with pos. returns str.npos if there is no non-white characer after pos + */ + + inline size_t skipWhitespace (const Unicode::String & str, size_t pos, const unicode_char_t * white) + { + return str.find_first_not_of (white, pos); + } + + //----------------------------------------------------------------- + + /** + * Return the first non-white position starting with pos. returns str.npos if there is no non-white characer after pos + */ + + inline size_t skipWhitespace (const Unicode::NarrowString & str, size_t pos, const char * white) + { + return str.find_first_not_of (white, pos); + } + + //----------------------------------------------------------------- + + /** + * Append src to dst, padding the field as needed, and truncating the field if desired. + */ + + inline String & appendStringField (String & dst, const NarrowString & src, size_t width, FieldAlignment fa, unicode_char_t pad, bool truncate) + { + return appendStringField (dst, narrowToWide (src), width, fa, pad, truncate); + } + + //----------------------------------------------------------------- + + /** + * Return the size of the single character represented by the + * given two byte value if it were stored as UTF-8. + */ + inline unsigned short utf8CharSize(const Unicode::unicode_char_t value) + { + if (value < 0x80) + return 1; + else if (value < 0x800) + return 2; + else + return 3; + } +} + +// ====================================================================== + + +#endif + + diff --git a/external/ours/library/unicode/src/shared/utf8.cpp b/external/ours/library/unicode/src/shared/utf8.cpp new file mode 100644 index 00000000..016b0b78 --- /dev/null +++ b/external/ours/library/unicode/src/shared/utf8.cpp @@ -0,0 +1,248 @@ +#include "FirstUnicode.h" + +#include +#include "utf8.h" +#include "UnicodeUtils.h" + +namespace Unicode +{ + +/* + A little bit about UTF8... + +UTF8 stands for UCS Tranfer Format, while UCS stands for Universal Character Set. In +essence, UTF8 is a way to encode all the characters in the UCS. UTF8 attempts to +describe all the characters in the UCS using from 1 to 6 bytes per character. If the +first byte's high bit is not set, then the character is 1 byte long, and corresponds to +the US-ASCII character of the same number. In this way, all current US-ASCII strings are +already UTF8 compatible. If the the first byte's high bit is set, then the number of +high bits set until the first 0 describes the number of bytes in the character, e.g. a +first byte who's bits are 1110xxxx will be followed by 2 more "data bytes" that look like +10yyyyyy 10zzzzzz, so that the entire character's descriptive ID is x xxxxyyyy yyzzzzzz. + +With this method, all characters in the UCS-4 character space can be mapped to <=6 bytes of +data, and all characters int the UCS-2 character space can be mapped in <=3 bytes +maximum. Since only UCS-2 actually contains character assignments from known languages, +it is expected that a maximum of 3 bytes will be needed for any character encountered +during translation. + +Unfortunately, this method also means that there are sequences of bytes that do not form +valid UTF8 characters/strings. + +In this file, we've placed some string and character validation functions, as well as +replacements for functions where the standard function will not behave +properly on a UTF8 string. + +*/ + +// array of character byte-lengths. Index into the array with the first byte of the character & 0x3C then shift down twice. +// NOTE: this only works for multi-byte characters (up to 6). +static char UTF8_charSizeArray[16] = { 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6 }; + + +// UTF8_charSize returns the length of a UTF8 character in bytes (1-6). +// The array UTF8_size has 0's where an invalid header byte is detected. +// This is a "fast" function, so no error checking is done. +int UTF8_charSize( char * ptr ) +{ + // temp hold of "header" byte + unsigned char ct = *ptr; + // if high bit not set, it's just a normal character, length 1 + if (( ct & 0x80 ) == 0 ) + { + return 1; + } + return UTF8_charSizeArray[ ( ct & 0x3C ) >> 2 ]; +} + + + +// takes a pointer to a UTF8 character and fills the 2nd arg with the 16-bit UTF16 corresponding char. +// return value is the length of the UTF8 char converted. +int UTF8_convertCharToUTF16( char * ptr , UTF16 * ret ) +{ + int len = UTF8_charSize( ptr ); + if ( len == 1 ) + { + *ret = *ptr; + } + else if ( len == 2 ) + { + if (*(ptr+1) != 0) + *ret = (UTF16)(((UTF16)( *ptr & 0x1F ) << 6 ) + ( *(ptr+1) & 0x3F )); + else + { + *ret = 0; + len = 0; + } + } + else if ( len == 3 ) + { + if (*(ptr+1) != 0 && *(ptr+2) != 0) + *ret = (UTF16)(((UTF16)( *ptr & 0x0F ) << 12 ) + ((UTF16)( *(ptr+1) & 0x3F ) << 6 ) + (( *(ptr+2) & 0x3F ))); + else + { + *ret = 0; + len = 0; + } + } + return len; +} + + +// takes a pointer to a UTF8 String and fills the 2nd array with the corresponding UTF16 string, up to +// the limit number of characters. +// return value is the length of the UTF16 string converted. +int UTF8_convertToUTF16( char * ptr , UTF16 * ret, int limit ) +{ + int len = 0; + while (( *ptr != 0 ) && ( len < ( limit - 1 ) ))//we use limit -1 to leave room for the null terminator + { + int clen = UTF8_convertCharToUTF16( ptr, ret ); + if (clen == 0) + break; + ptr += clen; + ret++; + len++; + } + *ret = 0; + return len; +} + + +// takes a UTF16 character and fills the 2nd arg with the UTF8 corresponding char. +// return value is the size of the UTF8 char converted. +// return value is 0 if the size of the UTF8 char would be bigger than the limit allowed. In this +// case, the return string is not filled. +int UTF16_convertCharToUTF8( UTF16 char16, char * ret, int limit ) +{ + int len = 0; + if ( char16 <= 0x0080 ) + { + if ( limit >= 1 ) + { + *ret = (char)char16; + len = 1; + } + } + else if ( char16 <= 0x07FF ) + { + if ( limit >= 2 ) + { + *(ret) = (char)( 0xC0 + (( char16 & 0x07C0 ) >> 6 )); + *(ret+1) = (char)(0x80 + (char)( char16 & 0x003F )); + len = 2; + } + } + else + { + if ( limit >= 3 ) + { + *ret = (char)(0xE0 + (( char16 & 0xF000 ) >> 12 )); + *(ret+1) = (char)(0x80 + (( char16 & 0x0FC0 ) >> 6 )); + *(ret+2) = (char)(0x80 + ( char16 & 0x003F )); + len = 3; + } + } + return len; +} + +// takes a pointer to a UTF16 String and fills the 2nd array with the corresponding UTF8 string, up to +// the limit number of bytes. +// return value is the size of the UTF8 string converted. +int UTF16_convertToUTF8( UTF16 *ptr, char * ret, int limit ) +{ + int len = 0; + while (( *ptr != 0 ) && ( limit != 0 )) + { + int clen = UTF16_convertCharToUTF8( *ptr, ret, limit ); + if ( clen == 0 ) + { + break; + } + ret += clen; + ptr++; + limit -= clen; + } + *ret = 0; + return len; +} + + +void EnglishToFakeUtf8(char *fakeDestination, char *englishSource, int destinationSize) +{ + UTF16 utf16[8192]; + + UTF16 *d = utf16; + char *s = englishSource; + bool afterPercent = false; + while (*s != 0) + { + if (afterPercent) + { + *d = *s; + + if (*s == ' ') // assuming everthing between % and white-space is formatting and don't shift it, this may not be the case all the time, but the worst that will happen is we won't shift everything we possibly could have + afterPercent = false; + } + else + { + if ((*s >= 'a' && *s <= 'k') || (*s >= 'A' && *s <= 'Z')) + *d = (UTF16)(*s + 0xfee0); + else + *d = *s; + + if (*s == '%') // we can't be shifting formatting tokens or else sprintf won't work + afterPercent = true; + } + d++; + s++; + } + *d = 0; + + UTF16_convertToUTF8(utf16, fakeDestination, destinationSize); +} + +void FakeUtf8ToEnglish(char *englishDestination, char *fakeSource, int destinationSize) +{ + UTF16 utf16[8192]; + UTF8_convertToUTF16(fakeSource, utf16, sizeof(utf16)); + + UTF16 *s = utf16; + char *d = englishDestination; + char *endDest = d + destinationSize - 1; + while (*s != 0 && d < endDest) + { + if (*s >= 0xfee0) + *d = (char)(*s - 0xfee0); + else + *d = (char)*s; + d++; + s++; + } + *d = 0; +} + +Unicode::String UTF8ToUnicode(const char *source) +{ + Unicode::String s = narrowToWide(""); + int length = strlen(source) + 1; + UTF16 *buffer = new UTF16[length]; + if (buffer != NULL) + { + UTF8_convertToUTF16(const_cast(source) , buffer, length); + s =buffer; + delete [] buffer; + } + return s; +} + + +void UnicodeToUTF8(Unicode::String &source, char *dest, int limit) +{ + UTF16_convertToUTF8((UTF16 *)source.c_str(), dest, limit); +} + + +}; + diff --git a/external/ours/library/unicode/src/shared/utf8.h b/external/ours/library/unicode/src/shared/utf8.h new file mode 100644 index 00000000..64277ad2 --- /dev/null +++ b/external/ours/library/unicode/src/shared/utf8.h @@ -0,0 +1,29 @@ +#ifndef UTF8_H +#define UTF8_H + +#include +#include "Unicode.h" + +namespace Unicode +{ + +typedef char UTF8; +typedef unsigned short int UTF16; + +int UTF8_charSize( char * ); + +int UTF8_convertCharToUTF16( char * ptr , UTF16 * ret ); +int UTF8_convertToUTF16( char * ptr , UTF16 * ret, int limit ); +int UTF16_convertCharToUTF8( UTF16 ret, char * ptr , int limit ); +int UTF16_convertToUTF8( UTF16 * ret, char * ptr , int limit ); + +void EnglishToFakeUtf8(char *fakeDestination, char *englishSource, int destinationSize); +void FakeUtf8ToEnglish(char *englishDestination, char *fakeSource, int destinationSize); + +Unicode::String UTF8ToUnicode(const char *source); +void UnicodeToUTF8(Unicode::String &source, char *dest, int limit); + +}; + +#endif + diff --git a/external/ours/library/unicode/src/win32/FirstUnicode.cpp b/external/ours/library/unicode/src/win32/FirstUnicode.cpp new file mode 100644 index 00000000..176b4e33 --- /dev/null +++ b/external/ours/library/unicode/src/win32/FirstUnicode.cpp @@ -0,0 +1,8 @@ +// ====================================================================== +// +// FirstUnicode.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstUnicode.h"