From 20b521ebf9657d7520bc6090bfa1b29a7dc29116 Mon Sep 17 00:00:00 2001 From: Anonymous Date: Tue, 14 Jan 2014 03:49:25 -0700 Subject: [PATCH] Added localization library --- external/ours/library/CMakeLists.txt | 1 + .../ours/library/localization/CMakeLists.txt | 9 + .../localization/include/FirstLocalization.h | 1 + .../include/LocalizationManager.h | 1 + .../localization/include/LocalizedString.h | 1 + .../include/LocalizedStringTable.h | 1 + .../LocalizedStringTableReaderWriter.h | 1 + .../library/localization/include/StringId.h | 1 + .../library/localization/src/CMakeLists.txt | 35 ++ .../src/shared/FirstLocalization.h | 21 + .../src/shared/LocalizationManager.cpp | 592 ++++++++++++++++++ .../src/shared/LocalizationManager.h | 151 +++++ .../src/shared/LocalizedString.cpp | 332 ++++++++++ .../localization/src/shared/LocalizedString.h | 180 ++++++ .../src/shared/LocalizedStringTable.cpp | 502 +++++++++++++++ .../src/shared/LocalizedStringTable.h | 203 ++++++ .../LocalizedStringTableReaderWriter.cpp | 553 ++++++++++++++++ .../shared/LocalizedStringTableReaderWriter.h | 187 ++++++ .../localization/src/shared/StringId.cpp | 245 ++++++++ .../localization/src/shared/StringId.h | 153 +++++ .../src/win32/FirstLocalization.cpp | 8 + 21 files changed, 3178 insertions(+) create mode 100644 external/ours/library/localization/CMakeLists.txt create mode 100644 external/ours/library/localization/include/FirstLocalization.h create mode 100644 external/ours/library/localization/include/LocalizationManager.h create mode 100644 external/ours/library/localization/include/LocalizedString.h create mode 100644 external/ours/library/localization/include/LocalizedStringTable.h create mode 100644 external/ours/library/localization/include/LocalizedStringTableReaderWriter.h create mode 100644 external/ours/library/localization/include/StringId.h create mode 100644 external/ours/library/localization/src/CMakeLists.txt create mode 100644 external/ours/library/localization/src/shared/FirstLocalization.h create mode 100644 external/ours/library/localization/src/shared/LocalizationManager.cpp create mode 100644 external/ours/library/localization/src/shared/LocalizationManager.h create mode 100644 external/ours/library/localization/src/shared/LocalizedString.cpp create mode 100644 external/ours/library/localization/src/shared/LocalizedString.h create mode 100644 external/ours/library/localization/src/shared/LocalizedStringTable.cpp create mode 100644 external/ours/library/localization/src/shared/LocalizedStringTable.h create mode 100644 external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.cpp create mode 100644 external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.h create mode 100644 external/ours/library/localization/src/shared/StringId.cpp create mode 100644 external/ours/library/localization/src/shared/StringId.h create mode 100644 external/ours/library/localization/src/win32/FirstLocalization.cpp diff --git a/external/ours/library/CMakeLists.txt b/external/ours/library/CMakeLists.txt index 29b88ce8..9ee02bd0 100644 --- a/external/ours/library/CMakeLists.txt +++ b/external/ours/library/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(archive) add_subdirectory(fileInterface) +add_subdirectory(localization) add_subdirectory(unicode) diff --git a/external/ours/library/localization/CMakeLists.txt b/external/ours/library/localization/CMakeLists.txt new file mode 100644 index 00000000..2846e4dc --- /dev/null +++ b/external/ours/library/localization/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 2.8) + +project(localization) + +if(WIN32) + add_definitions(/D_CRT_SECURE_NO_WARNINGS) +endif() + +add_subdirectory(src) diff --git a/external/ours/library/localization/include/FirstLocalization.h b/external/ours/library/localization/include/FirstLocalization.h new file mode 100644 index 00000000..320898f8 --- /dev/null +++ b/external/ours/library/localization/include/FirstLocalization.h @@ -0,0 +1 @@ +#include "../src/shared/FirstLocalization.h" diff --git a/external/ours/library/localization/include/LocalizationManager.h b/external/ours/library/localization/include/LocalizationManager.h new file mode 100644 index 00000000..fc38f83a --- /dev/null +++ b/external/ours/library/localization/include/LocalizationManager.h @@ -0,0 +1 @@ +#include "../src/shared/LocalizationManager.h" diff --git a/external/ours/library/localization/include/LocalizedString.h b/external/ours/library/localization/include/LocalizedString.h new file mode 100644 index 00000000..280c0cd5 --- /dev/null +++ b/external/ours/library/localization/include/LocalizedString.h @@ -0,0 +1 @@ +#include "../src/shared/LocalizedString.h" diff --git a/external/ours/library/localization/include/LocalizedStringTable.h b/external/ours/library/localization/include/LocalizedStringTable.h new file mode 100644 index 00000000..7b2fe153 --- /dev/null +++ b/external/ours/library/localization/include/LocalizedStringTable.h @@ -0,0 +1 @@ +#include "../src/shared/LocalizedStringTable.h" diff --git a/external/ours/library/localization/include/LocalizedStringTableReaderWriter.h b/external/ours/library/localization/include/LocalizedStringTableReaderWriter.h new file mode 100644 index 00000000..498907b4 --- /dev/null +++ b/external/ours/library/localization/include/LocalizedStringTableReaderWriter.h @@ -0,0 +1 @@ +#include "../src/shared/LocalizedStringTableReaderWriter.h" diff --git a/external/ours/library/localization/include/StringId.h b/external/ours/library/localization/include/StringId.h new file mode 100644 index 00000000..ff99ac27 --- /dev/null +++ b/external/ours/library/localization/include/StringId.h @@ -0,0 +1 @@ +#include "../src/shared/StringId.h" diff --git a/external/ours/library/localization/src/CMakeLists.txt b/external/ours/library/localization/src/CMakeLists.txt new file mode 100644 index 00000000..d527e28c --- /dev/null +++ b/external/ours/library/localization/src/CMakeLists.txt @@ -0,0 +1,35 @@ + +set(SHARED_SOURCES + shared/FirstLocalization.h + shared/LocalizationManager.cpp + shared/LocalizationManager.h + shared/LocalizedString.cpp + shared/LocalizedString.h + shared/LocalizedStringTable.cpp + shared/LocalizedStringTable.h + shared/LocalizedStringTableReaderWriter.cpp + shared/LocalizedStringTableReaderWriter.h + shared/StringId.cpp + shared/StringId.h +) + +if(WIN32) + set(PLATFORM_SOURCES + win32/FirstLocalization.cpp + ) + + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/win32) +else() + set(PLATFORM_SOURCES "") +endif() + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/shared + ${SWG_EXTERNALS_SOURCE_DIR}/ours/library/fileInterface/include/public + ${SWG_EXTERNALS_SOURCE_DIR}/ours/library/unicode/include +) + +add_library(localization + ${SHARED_SOURCES} + ${PLATFORM_SOURCES} +) diff --git a/external/ours/library/localization/src/shared/FirstLocalization.h b/external/ours/library/localization/src/shared/FirstLocalization.h new file mode 100644 index 00000000..eb552316 --- /dev/null +++ b/external/ours/library/localization/src/shared/FirstLocalization.h @@ -0,0 +1,21 @@ +// ====================================================================== +// +// FirstLocalization.h +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#ifndef INCLUDED_FirstLocalization_H +#define INCLUDED_FirstLocalization_H + +// ====================================================================== + +#ifdef _WIN32 +#pragma warning (disable: 4702) +#endif + +typedef unsigned char byte; + +// ====================================================================== + +#endif diff --git a/external/ours/library/localization/src/shared/LocalizationManager.cpp b/external/ours/library/localization/src/shared/LocalizationManager.cpp new file mode 100644 index 00000000..f55d7f9c --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizationManager.cpp @@ -0,0 +1,592 @@ +// ====================================================================== +// +// LocalizationManager.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstLocalization.h" +#include "LocalizationManager.h" + +#include "LocalizedStringTable.h" +#include "StringId.h" +#include "UnicodeUtils.h" +#include "fileInterface/AbstractFile.h" +#include +#include +#include +#include +#include + +// ====================================================================== + +namespace LocalizationManagerNamespace +{ + const std::string s_pathString = "string/"; + const std::string s_suffix = ".stf"; + + const std::string s_englishLocale = "en"; + + std::string s_pathStringWithLocale; + std::string s_pathStringWithEnglishLocale; + + typedef std::vector PreloadedStringTables; + + PreloadedStringTables ms_preloadedStringTables; + + bool s_debugStrings = false; + bool s_displayStringIdInfo = false; + + + Unicode::String s_debugDisplayColor(Unicode::narrowToWide(" \\#00ff00")); + + //---------------------------------------------------------------------- + + void createBadStringRepresentation (const StringId & id, Unicode::String & value) + { + if (s_displayStringIdInfo) + { + value += s_debugDisplayColor; + } + + value += Unicode::narrowToWide(id.getText()); + + if (!value.empty ()) + { + size_t pos = 0; + + while (pos != std::string::npos) + { + const size_t underscore = value.find (Unicode::unicode_char_t ('_'), pos); + if (underscore != std::string::npos) + { + value [underscore] = ' '; + pos = underscore + 1; + } + else + break; + } + + value [0] = static_cast(toupper (value [0])); + } + } +} + +using namespace LocalizationManagerNamespace; + +//---------------------------------------------------------------------- + +bool LocalizationManager::ms_installed = 0; +LocalizationManager::LocalizationManagerHashMap * LocalizationManager::ms_singletonHashMap = NULL; +Unicode::NarrowString * LocalizationManager::ms_firstLocaleLoaded = NULL; + +//----------------------------------------------------------------- + +LocalizationManager::LocalizationManager (AbstractFileFactory * fileFactory, const Unicode::NarrowString & localeName, DebugBadStringsFunc debugBadStringsFunc, bool displayBadStringIds) : +m_fileFactory (fileFactory), +m_stringTableMap (), +m_englishStringTableMap (), +m_localeName (localeName), +m_debugBadStringsFunc (debugBadStringsFunc), +m_displayBadStringIds (displayBadStringIds) +{ + s_pathStringWithLocale = s_pathString + m_localeName; + s_pathStringWithLocale.push_back ('/'); + s_pathStringWithEnglishLocale = s_pathString + s_englishLocale; + s_pathStringWithEnglishLocale.push_back ('/'); + if(s_englishLocale.compare(m_localeName) == 0 || m_localeName.compare("none") == 0) + { + m_usingEnglishLocale = true; + } + else + { + m_usingEnglishLocale = false; + } + + assert (m_fileFactory != NULL);//lint !e1924 // c-style cast. MSVC bug +} + +//----------------------------------------------------------------- + +LocalizationManager::~LocalizationManager () +{ + m_debugBadStringsFunc = 0; + + for (StringTableMap_t::iterator iter = m_stringTableMap.begin (); iter != m_stringTableMap.end (); ++iter) + { + LocalizedStringTable * const table = (*iter).second.second; + + if (table) + { + //-- the LocalizationManager holds a reference to any table requested + assert (table->m_referenceCount == 1);//lint !e1924 // c-style cast. MSVC bug + + delete table; + (*iter).second.second = 0; + } + } + + m_stringTableMap.clear (); + + if(!m_usingEnglishLocale) + { + for (StringTableMap_t::iterator iter = m_englishStringTableMap.begin (); iter != m_englishStringTableMap.end (); ++iter) + { + LocalizedStringTable * const table = (*iter).second.second; + + if (table) + { + //-- the LocalizationManager holds a reference to any table requested + assert (table->m_referenceCount == 1);//lint !e1924 // c-style cast. MSVC bug + + delete table; + (*iter).second.second = 0; + } + } + + m_englishStringTableMap.clear (); + + } + delete m_fileFactory; + m_fileFactory = 0; +} + +//----------------------------------------------------------------- + +void LocalizationManager::install (AbstractFileFactory * fileFactory, Unicode::UnicodeNarrowStringVector & localeNameMap, bool debugStrings, DebugBadStringsFunc debugBadStringsFunc, bool displayBadStringIds) +{ + assert (!ms_installed);//lint !e1924 // c-style cast. MSVC bug + assert (!ms_singletonHashMap);//lint !e1924 // c-style cast. MSVC bug + assert (!ms_firstLocaleLoaded); + + ms_singletonHashMap = new LocalizationManagerHashMap(); + + Unicode::UnicodeNarrowStringVector::const_iterator end = localeNameMap.end(); + for ( Unicode::UnicodeNarrowStringVector::const_iterator iter = localeNameMap.begin(); iter != end; ++iter ) + { + Unicode::NarrowString localeName = (*iter); + LocalizationManager * localizationManager = new LocalizationManager (fileFactory, localeName, debugBadStringsFunc, displayBadStringIds); + (*ms_singletonHashMap)[localeName] = localizationManager; + } + + ms_firstLocaleLoaded = new Unicode::NarrowString((*(localeNameMap.begin()))); + + ms_installed = true; + + s_debugStrings = debugStrings; +} + +//----------------------------------------------------------------- + +void LocalizationManager::remove () +{ + assert (ms_installed);//lint !e1924 // c-style cast. MSVC bug + assert (ms_singletonHashMap != NULL);//lint !e1924 // c-style cast. MSVC bug + assert (ms_firstLocaleLoaded != NULL); + + LocalizationManagerHashMap::iterator end = ms_singletonHashMap->end(); + for (LocalizationManagerHashMap::iterator it = ms_singletonHashMap->begin(); it != end; ++it) + { + LocalizationManager * current = (*it).second; + (*it).second = NULL; + delete current; + } + + ms_singletonHashMap->clear(); + delete ms_singletonHashMap; + ms_singletonHashMap = NULL; + + delete ms_firstLocaleLoaded; + ms_firstLocaleLoaded = NULL; + + ms_installed = false; +} + +//----------------------------------------------------------------- + +/** +* Get the default locale (English). Undefined results if called without first calling install () +*/ +LocalizationManager & LocalizationManager::getManager () +{ + return *(*LocalizationManager::ms_singletonHashMap)[*ms_firstLocaleLoaded]; +} + +//----------------------------------------------------------------- + +/** +* Get the locale specified by locale ("en" == English, "ja" == Japanese). Undefined results if called without first calling install () +*/ +LocalizationManager & LocalizationManager::getManager (Unicode::NarrowString locale) +{ + LocalizationManagerHashMap::iterator f = LocalizationManager::ms_singletonHashMap->find(locale); + if(f != LocalizationManager::ms_singletonHashMap->end()) + return *((*f).second); + return getManager(); +} + +//----------------------------------------------------------------- + +void LocalizationManager::preload (const Unicode::NarrowString & name) +{ + LocalizedStringTable *table = fetchStringTable(name); + if (table) + ms_preloadedStringTables.push_back (table); +} + +//----------------------------------------------------------------- + +void LocalizationManager::releasePreloadedAssets () +{ + while (!ms_preloadedStringTables.empty ()) + { + releaseStringTable (ms_preloadedStringTables.back ()); + ms_preloadedStringTables.pop_back (); + } +} + +//----------------------------------------------------------------- + +LocalizedStringTable * LocalizationManager::fetchStringTable(const Unicode::NarrowString & name) +{ + return fetchStringTable(name, false); +} + +//----------------------------------------------------------------- + +LocalizedStringTable * LocalizationManager::fetchStringTable(const Unicode::NarrowString & name, bool forceUseEnglish) +{ + // DBE - this is to prevent using the m_englishStringTableMap English exception if we're already localizing to English. + // The rest of the code assumes that if we're localizing to English, m_englishStringTableMap will never be used. + if (m_usingEnglishLocale) + { + forceUseEnglish=false; + } + + // This bool is used to determine if we should try again with English. We shouldn't if we are running in English (m_usingEnglishLocale) + // or if we are already trying again with English (forceUseEnglish). + bool const useEnglish = forceUseEnglish || m_usingEnglishLocale; + + // Use the locale specified unless we are forcing the use of English + StringTableMap_t & stmap = (forceUseEnglish) ? m_englishStringTableMap : m_stringTableMap; + std::string const & pathPrefix = (forceUseEnglish) ? s_pathStringWithEnglishLocale : s_pathStringWithLocale; + + LocalizedStringTable * table = 0; + + StringTableMap_t::iterator const find_iter = stmap.find (name); + if(find_iter != stmap.end ()) + { + TimedStringTable & tst = (*find_iter).second; + //-- this can be null + table = tst.second; + tst.first = time(0); + } + else + { + static std::string filename; + filename.clear (); + filename += pathPrefix + name + s_suffix; + + table = LocalizedStringTable::load (*m_fileFactory, filename); + + if (table) + { + table->m_name = name; + //-- the LocalizationManager holds a reference to any table requested, until purged + fetchStringTable(table); + } + + stmap.insert (std::make_pair (name, TimedStringTable (time(0), table))); + } + + if (table) + fetchStringTable(table); + else if (!useEnglish) + table = fetchStringTable(name, true); + + return table; +} + +//---------------------------------------------------------------------- + +void LocalizationManager::garbageCollectUnused (int timeoutThresholdSecs) +{ + const time_t currentTime = time (0); //update last-used time + for (StringTableMap_t::iterator iter = m_stringTableMap.begin (); iter != m_stringTableMap.end ();) + { + TimedStringTable & tst = (*iter).second; + + LocalizedStringTable * const table = tst.second; + + if (table && table->m_referenceCount == 1) + { + const time_t lastTouched = tst.first; + const time_t diff = currentTime - lastTouched; + + //-- if the string table is older than the threshold, dump it + if (diff > timeoutThresholdSecs) + { + releaseStringTable(table); + m_stringTableMap.erase (iter++); + continue; + } + } + + ++iter; + } + if(!m_usingEnglishLocale) + { + for (StringTableMap_t::iterator iter = m_englishStringTableMap.begin (); iter != m_englishStringTableMap.end ();) + { + TimedStringTable & tst = (*iter).second; + + LocalizedStringTable * const table = tst.second; + + if (table && table->m_referenceCount == 1) + { + const time_t lastTouched = tst.first; + const time_t diff = currentTime - lastTouched; + + //-- if the string table is older than the threshold, dump it + if (diff > timeoutThresholdSecs) + { + releaseStringTable(table); + m_englishStringTableMap.erase (iter++); + continue; + } + } + + ++iter; + } + } +} + +//---------------------------------------------------------------------- + +void LocalizationManager::purgeUnusedStringTables () +{ + for (StringTableMap_t::iterator iter = m_stringTableMap.begin (); iter != m_stringTableMap.end ();) + { + LocalizedStringTable * const table = (*iter).second.second; + + if (table) + { + //-- the LocalizationManager holds a reference to any table requested + if (table->m_referenceCount == 1) + { + releaseStringTable(table); + iter = m_stringTableMap.begin(); + continue; + } + } + + ++iter; + } + if(!m_usingEnglishLocale) + { + for (StringTableMap_t::iterator iter = m_englishStringTableMap.begin (); iter != m_englishStringTableMap.end ();) + { + LocalizedStringTable * const table = (*iter).second.second; + + if (table) + { + //-- the LocalizationManager holds a reference to any table requested + if (table->m_referenceCount == 1) + { + releaseStringTable(table); + iter = m_englishStringTableMap.begin(); + continue; + } + } + + ++iter; + } + } +} + +//----------------------------------------------------------------- + +void LocalizationManager::fetchStringTable (const LocalizedStringTable * table) +{ + if (table == 0) + return; + + ++(table->m_referenceCount); +} + +//----------------------------------------------------------------- + +void LocalizationManager::releaseStringTable (const LocalizedStringTable * table) +{ + if (table == 0) + return; + + if (--(table->m_referenceCount) == 0) + { + const StringTableMap_t::iterator find_iter = m_stringTableMap.find (table->getName ()); + + //If we're using english it should be in m_stringTableMap + assert ((find_iter != m_stringTableMap.end () && table == (*find_iter).second.second) || !m_usingEnglishLocale);//lint !e1924 // c-style cast. MSVC bug + + // The re-check of m_referenceCount is because there can potentially be two tables + // of the same name when loading with a locale other than English. This can happen if an ID + // doesn't exist in the locale specific string file, whether it does exist in the English string + // file or not. + if(find_iter != m_stringTableMap.end() && + table == (*find_iter).second.second) + { + delete (*find_iter).second.second; + (*find_iter).second.second = 0; + + m_stringTableMap.erase (find_iter); + } + else if(!m_usingEnglishLocale) + { + const StringTableMap_t::iterator find_iter = m_englishStringTableMap.find (table->getName ()); + + assert (find_iter != m_englishStringTableMap.end ());//lint !e1924 // c-style cast. MSVC bug + + delete (*find_iter).second.second; + (*find_iter).second.second = 0; + + m_englishStringTableMap.erase (find_iter); + } + } + +} + +//---------------------------------------------------------------------- + +LocalizationManager::StringValueCode LocalizationManager::getLocalizedStringValue (const StringId & id, Unicode::String & value, bool useEnglish) +{ + value.clear (); + + + // This finds the English string table if it exists, so if the table isn't found then it won't be found. + LocalizedStringTable * const table = fetchStringTable (id.getTable (), useEnglish); + if (!table) + { + if (m_displayBadStringIds || s_displayStringIdInfo) + { + if (s_displayStringIdInfo) + { + value += s_debugDisplayColor; + } + value.push_back ('['); + value.append (Unicode::narrowToWide (id.getTable ())); + value.push_back (']'); + value.push_back (':'); + value.append (Unicode::narrowToWide (id.getText ())); + } + else + { + createBadStringRepresentation (id, value); + } + + if (s_debugStrings && m_debugBadStringsFunc) + { + m_debugBadStringsFunc (id, true); + } + + return SVC_bad_table; + } + + const LocalizedString * locstr = 0; + + if (id.getTextIndex () > 0) + locstr = table->getLocalizedString (id.getTextIndex ()); + else + { + locstr = table->getLocalizedString (id.getText ()); + if (locstr) + id.setTextIndex (locstr->getId ()); + } + + if (locstr) + { + // @todo: add any string substitution + value = locstr->getString (); + + if (s_displayStringIdInfo) + { + value += s_debugDisplayColor; + value.push_back ('['); + value.append (Unicode::narrowToWide (id.getTable ())); + value.push_back (']'); + value.push_back (':'); + value.append (Unicode::narrowToWide (id.getText ())); + } + + releaseStringTable (table); + return SVC_ok; + } + else + { + // The localized string table exists, but the id is not in it. See if the id exists in the English version of the file. + if(!useEnglish && !m_usingEnglishLocale) + { + LocalizationManager::StringValueCode stringValueCode = getLocalizedStringValue(id, value, true); + if(stringValueCode == LocalizationManager::SVC_ok) + { + if (m_displayBadStringIds || s_displayStringIdInfo) + { + if (s_displayStringIdInfo) + { + value += s_debugDisplayColor; + } + value.append (Unicode::narrowToWide (id.getTable ())); + value.push_back (':'); + value.push_back ('['); + value.append (Unicode::narrowToWide (id.getText ())); + value.push_back (']'); + } + + releaseStringTable (table); + return SVC_ok; + } + } + + if (m_displayBadStringIds || s_displayStringIdInfo) + { + if (s_displayStringIdInfo) + { + value += s_debugDisplayColor; + } + value.append (Unicode::narrowToWide (id.getTable ())); + value.push_back (':'); + value.push_back ('['); + value.append (Unicode::narrowToWide (id.getText ())); + value.push_back (']'); + } + else + { + createBadStringRepresentation (id, value); + } + + releaseStringTable (table); + + if (s_debugStrings && m_debugBadStringsFunc) + { + m_debugBadStringsFunc (id, false); + } + + return SVC_bad_name; + } +} + +//---------------------------------------------------------------------- + +void LocalizationManager::debugDisplayStrings(bool const debugEnabled) +{ + s_displayStringIdInfo = debugEnabled; +} + +//---------------------------------------------------------------------- + +void LocalizationManager::debugDisplayStringColor(Unicode::String const & colorString) +{ + s_debugDisplayColor = Unicode::narrowToWide(" ") + colorString; +} + + +// ====================================================================== diff --git a/external/ours/library/localization/src/shared/LocalizationManager.h b/external/ours/library/localization/src/shared/LocalizationManager.h new file mode 100644 index 00000000..70823546 --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizationManager.h @@ -0,0 +1,151 @@ +// ====================================================================== +// +// LocalizationManager.h +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#ifndef INCLUDED_LocalizationManager_H +#define INCLUDED_LocalizationManager_H + + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +// unref inline func removed +#pragma warning (disable:4514) +// symbol name too long +#pragma warning (disable:4786) +#endif + +#include +#include "Unicode.h" +#include "UnicodeUtils.h" + +class LocalizedStringTable; +class LocalizedString; +class AbstractFileFactory; +class StringId; + +// ====================================================================== + +/** +* A LocalizationManager represents a collection of LocalizedStringTables for a single locale. +* This can be used as a singleton, or with multiple instances in a larger collection. +*/ + +class LocalizationManager +{ +public: + + // return values from getLocalizedStringValue + enum StringValueCode + { + SVC_ok, + SVC_bad_table, + SVC_bad_name + }; + + typedef void (*DebugBadStringsFunc) (const StringId &, bool); + +public: + + LocalizationManager (AbstractFileFactory * fileFactory, const Unicode::NarrowString & localeName, DebugBadStringsFunc debugBadStringsFunc = 0, bool displayBadStringIds = true); + static LocalizationManager & getManager (); + static LocalizationManager & getManager (Unicode::NarrowString locale); + + ~LocalizationManager (); + + typedef std::pair TimedStringTable; + typedef std::hash_map StringTableMap_t; + typedef std::hash_map LocalizationManagerHashMap; + + static void install (AbstractFileFactory * fileFactory, Unicode::UnicodeNarrowStringVector & localeNames, bool debugStrings, DebugBadStringsFunc debugBadStringsFunc = 0, bool displayBadStringIds = true); + static void remove (); + + LocalizedStringTable * fetchStringTable (const Unicode::NarrowString & name); + void fetchStringTable (const LocalizedStringTable * table); + + void releaseStringTable (const LocalizedStringTable * table); + + const Unicode::NarrowString & getLocaleName (); + + void setLocaleName (const Unicode::NarrowString & name); + + const StringTableMap_t & getTableMap () const; + + StringValueCode getLocalizedStringValue (const StringId & id, Unicode::String & value, bool useEnglish = false); + + void purgeUnusedStringTables (); + + void garbageCollectUnused (int timeoutThresholdSecs = 20); + + static void debugDisplayStrings(bool const debugEnabled); + static void debugDisplayStringColor(Unicode::String const & colorString); + +private: + LocalizationManager (); + LocalizationManager (const LocalizationManager & rhs); + LocalizationManager & operator= (const LocalizationManager & rhs); + + void preload (const Unicode::NarrowString & name); + void releasePreloadedAssets (); + + LocalizedStringTable * fetchStringTable (const Unicode::NarrowString & name, bool forceUseEnglish); + + static bool ms_installed; + static LocalizationManagerHashMap * ms_singletonHashMap; + static Unicode::NarrowString * ms_firstLocaleLoaded; + + + AbstractFileFactory * m_fileFactory; + + StringTableMap_t m_stringTableMap; + StringTableMap_t m_englishStringTableMap; + Unicode::NarrowString m_localeName; + DebugBadStringsFunc m_debugBadStringsFunc; + bool m_displayBadStringIds; + + bool m_usingEnglishLocale; +}; + +//----------------------------------------------------------------- + +/** +* Get the short name of the locale. E.g. "english", "japanese", etc. +* The locale name affects the path the LocalizationManager searches in, +* and should represent the paths present in the AbstractFileFactory +*/ + +inline const Unicode::NarrowString & LocalizationManager::getLocaleName () +{ + return m_localeName; +} + +//----------------------------------------------------------------- + +/** +* Set the short name of the locale. E.g. "english", "japanese", etc. +* The locale name affects the path the LocalizationManager searches in, +* and should represent the paths present in the AbstractFileFactory +*/ + +inline void LocalizationManager::setLocaleName (const Unicode::NarrowString & name) +{ + m_localeName = name; +} + +//----------------------------------------------------------------- + +/** +* Get a const ref to the table name -> table mapping +*/ + +inline const LocalizationManager::StringTableMap_t & LocalizationManager::getTableMap () const +{ + return m_stringTableMap; +} + +// ====================================================================== + +#endif diff --git a/external/ours/library/localization/src/shared/LocalizedString.cpp b/external/ours/library/localization/src/shared/LocalizedString.cpp new file mode 100644 index 00000000..88817b83 --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizedString.cpp @@ -0,0 +1,332 @@ +// ====================================================================== +// +// LocalizedString.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstLocalization.h" +#include "LocalizedString.h" + +#include "fileInterface/AbstractFile.h" + +#include +#include +#include +#include + +// ====================================================================== + +namespace LocalizedStringNamespace +{ + unsigned long const crctable[256] = + { + 0x00000000, 0x04C11DB7, 0x09823B6E, 0x0D4326D9, 0x130476DC, 0x17C56B6B, 0x1A864DB2, 0x1E475005, + 0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, 0x2B4BCB61, 0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD, + 0x4C11DB70, 0x48D0C6C7, 0x4593E01E, 0x4152FDA9, 0x5F15ADAC, 0x5BD4B01B, 0x569796C2, 0x52568B75, + 0x6A1936C8, 0x6ED82B7F, 0x639B0DA6, 0x675A1011, 0x791D4014, 0x7DDC5DA3, 0x709F7B7A, 0x745E66CD, + 0x9823B6E0, 0x9CE2AB57, 0x91A18D8E, 0x95609039, 0x8B27C03C, 0x8FE6DD8B, 0x82A5FB52, 0x8664E6E5, + 0xBE2B5B58, 0xBAEA46EF, 0xB7A96036, 0xB3687D81, 0xAD2F2D84, 0xA9EE3033, 0xA4AD16EA, 0xA06C0B5D, + 0xD4326D90, 0xD0F37027, 0xDDB056FE, 0xD9714B49, 0xC7361B4C, 0xC3F706FB, 0xCEB42022, 0xCA753D95, + 0xF23A8028, 0xF6FB9D9F, 0xFBB8BB46, 0xFF79A6F1, 0xE13EF6F4, 0xE5FFEB43, 0xE8BCCD9A, 0xEC7DD02D, + 0x34867077, 0x30476DC0, 0x3D044B19, 0x39C556AE, 0x278206AB, 0x23431B1C, 0x2E003DC5, 0x2AC12072, + 0x128E9DCF, 0x164F8078, 0x1B0CA6A1, 0x1FCDBB16, 0x018AEB13, 0x054BF6A4, 0x0808D07D, 0x0CC9CDCA, + 0x7897AB07, 0x7C56B6B0, 0x71159069, 0x75D48DDE, 0x6B93DDDB, 0x6F52C06C, 0x6211E6B5, 0x66D0FB02, + 0x5E9F46BF, 0x5A5E5B08, 0x571D7DD1, 0x53DC6066, 0x4D9B3063, 0x495A2DD4, 0x44190B0D, 0x40D816BA, + 0xACA5C697, 0xA864DB20, 0xA527FDF9, 0xA1E6E04E, 0xBFA1B04B, 0xBB60ADFC, 0xB6238B25, 0xB2E29692, + 0x8AAD2B2F, 0x8E6C3698, 0x832F1041, 0x87EE0DF6, 0x99A95DF3, 0x9D684044, 0x902B669D, 0x94EA7B2A, + 0xE0B41DE7, 0xE4750050, 0xE9362689, 0xEDF73B3E, 0xF3B06B3B, 0xF771768C, 0xFA325055, 0xFEF34DE2, + 0xC6BCF05F, 0xC27DEDE8, 0xCF3ECB31, 0xCBFFD686, 0xD5B88683, 0xD1799B34, 0xDC3ABDED, 0xD8FBA05A, + 0x690CE0EE, 0x6DCDFD59, 0x608EDB80, 0x644FC637, 0x7A089632, 0x7EC98B85, 0x738AAD5C, 0x774BB0EB, + 0x4F040D56, 0x4BC510E1, 0x46863638, 0x42472B8F, 0x5C007B8A, 0x58C1663D, 0x558240E4, 0x51435D53, + 0x251D3B9E, 0x21DC2629, 0x2C9F00F0, 0x285E1D47, 0x36194D42, 0x32D850F5, 0x3F9B762C, 0x3B5A6B9B, + 0x0315D626, 0x07D4CB91, 0x0A97ED48, 0x0E56F0FF, 0x1011A0FA, 0x14D0BD4D, 0x19939B94, 0x1D528623, + 0xF12F560E, 0xF5EE4BB9, 0xF8AD6D60, 0xFC6C70D7, 0xE22B20D2, 0xE6EA3D65, 0xEBA91BBC, 0xEF68060B, + 0xD727BBB6, 0xD3E6A601, 0xDEA580D8, 0xDA649D6F, 0xC423CD6A, 0xC0E2D0DD, 0xCDA1F604, 0xC960EBB3, + 0xBD3E8D7E, 0xB9FF90C9, 0xB4BCB610, 0xB07DABA7, 0xAE3AFBA2, 0xAAFBE615, 0xA7B8C0CC, 0xA379DD7B, + 0x9B3660C6, 0x9FF77D71, 0x92B45BA8, 0x9675461F, 0x8832161A, 0x8CF30BAD, 0x81B02D74, 0x857130C3, + 0x5D8A9099, 0x594B8D2E, 0x5408ABF7, 0x50C9B640, 0x4E8EE645, 0x4A4FFBF2, 0x470CDD2B, 0x43CDC09C, + 0x7B827D21, 0x7F436096, 0x7200464F, 0x76C15BF8, 0x68860BFD, 0x6C47164A, 0x61043093, 0x65C52D24, + 0x119B4BE9, 0x155A565E, 0x18197087, 0x1CD86D30, 0x029F3D35, 0x065E2082, 0x0B1D065B, 0x0FDC1BEC, + 0x3793A651, 0x3352BBE6, 0x3E119D3F, 0x3AD08088, 0x2497D08D, 0x2056CD3A, 0x2D15EBE3, 0x29D4F654, + 0xC5A92679, 0xC1683BCE, 0xCC2B1D17, 0xC8EA00A0, 0xD6AD50A5, 0xD26C4D12, 0xDF2F6BCB, 0xDBEE767C, + 0xE3A1CBC1, 0xE760D676, 0xEA23F0AF, 0xEEE2ED18, 0xF0A5BD1D, 0xF464A0AA, 0xF9278673, 0xFDE69BC4, + 0x89B8FD09, 0x8D79E0BE, 0x803AC667, 0x84FBDBD0, 0x9ABC8BD5, 0x9E7D9662, 0x933EB0BB, 0x97FFAD0C, + 0xAFB010B1, 0xAB710D06, 0xA6322BDF, 0xA2F33668, 0xBCB4666D, 0xB8757BDA, 0xB5365D03, 0xB1F740B4 + }; + + unsigned long const s_initsCrc = 0xFFFFFFFF; + + LocalizedString::crc_type generateCrc(void const * const crcData, size_t stringSize) + { + if (stringSize) + { + unsigned long crc; + unsigned char const * data = reinterpret_cast(crcData); + + for (crc = s_initsCrc; stringSize; --stringSize, ++data) + crc = crctable[((crc>>24) ^ static_cast(*data)) & 0xFF] ^ (crc << 8); + + return (crc ^ s_initsCrc); + } + + return LocalizedString::nullCrc; + } +} + +using namespace LocalizedStringNamespace; + +// ====================================================================== + +LocalizedString::crc_type const LocalizedString::nullCrc = s_initsCrc; + +// ====================================================================== + +LocalizedString::LocalizedString (LocalizedString::id_type id, LocalizedString::crc_type sourceCrc, const Unicode::String & str) : +m_str (str), +m_id (id), +m_crc(nullCrc), +m_sourceCrc(sourceCrc), +m_numLines (0), +m_lineStarts (0) +{ + resetLineCounts (); + buildCrc(); +} + +//----------------------------------------------------------------- + +LocalizedString::LocalizedString (const LocalizedString & rhs) : +m_str(rhs.m_str), +m_id(rhs.m_id), +m_crc(rhs.m_crc), +m_sourceCrc(rhs.m_sourceCrc), +m_numLines (0), +m_lineStarts (0) +{ + resetLineCounts (); +} + +//----------------------------------------------------------------- + +LocalizedString::LocalizedString () : +m_str (), +m_id (0), +m_crc(nullCrc), +m_sourceCrc(nullCrc), +m_numLines (0), +m_lineStarts (0) +{ +} + +//----------------------------------------------------------------- + +LocalizedString::LocalizedString (LocalizedString::id_type id, const Unicode::String & str) : +m_str(str), +m_id(id), +m_crc(nullCrc), +m_sourceCrc(nullCrc), +m_numLines(0), +m_lineStarts(0) +{ + resetLineCounts(); + buildCrc(); +} + + +//----------------------------------------------------------------- + +LocalizedString::~LocalizedString () +{ + delete[] m_lineStarts; + m_lineStarts = 0; + m_numLines = 0; + m_crc = nullCrc; + m_sourceCrc = nullCrc; +} + +//----------------------------------------------------------------- + +LocalizedString & LocalizedString::operator= (const LocalizedString & rhs) +{ + if (this != &rhs) + { + m_str = rhs.m_str; + m_id = rhs.m_id; + m_crc = rhs.m_crc; + m_sourceCrc = rhs.m_sourceCrc; + + resetLineCounts (); + } + + return *this; +} + +//----------------------------------------------------------------- + +bool LocalizedString::operator==(const LocalizedString & rhs) +{ + return m_id == rhs.m_id && m_crc == rhs.m_crc && m_sourceCrc == rhs.m_sourceCrc && m_str == rhs.m_str; +} + +//----------------------------------------------------------------- + +/** +* Load a LocalizedString from the current position of the file. +*/ + +LocalizedString * LocalizedString::load_0000 (AbstractFile & fl) +{ + LocalizedString * loc_str = 0; + + id_type id; + unsigned long dummy_time = nullCrc; + id_type buflen; + + if (!fl.read (&id, sizeof (id_type))) + return 0; + + if (!fl.read(&dummy_time, sizeof(unsigned long))) + return 0; + + + if (!fl.read (&buflen, sizeof (id_type))) + return 0; + + // buflen does not include null terminator + Unicode::unicode_char_t * buf = new Unicode::unicode_char_t [buflen+1]; + + assert (buf != NULL); //lint !e1924 // c-style cast. MSVC bug + + buf [buflen] = 0; + + if (buflen) + { + const size_t readlen = buflen * sizeof (Unicode::unicode_char_t); + if (!fl.read (buf, readlen)) + { + delete[] buf;; + return 0; + } + } + + // TODO: swap all values if needed on big endian systems + // TODO: Localized strings are stored little endian! + + loc_str = new LocalizedString(id, buf); + assert (loc_str != NULL); //lint !e1924 // c-style cast. MSVC bug + + loc_str->resetLineCounts (); + + delete[] buf; + buf = 0; + + return loc_str; +} + +//----------------------------------------------------------------- + +/** +* Load a LocalizedString from the current position of the file. +*/ + +LocalizedString * LocalizedString::load_0001 (AbstractFile & fl) +{ + LocalizedString * loc_str = 0; + + id_type id; + crc_type crcSource = nullCrc; + id_type buflen; + + if (!fl.read (&id, sizeof (id_type))) + return 0; + + if (!fl.read(&crcSource, sizeof(crc_type))) + return 0; + + if (!fl.read (&buflen, sizeof (id_type))) + return 0; + + // buflen does not include null terminator + Unicode::unicode_char_t * buf = new Unicode::unicode_char_t [buflen+1]; + + assert (buf != NULL); //lint !e1924 // c-style cast. MSVC bug + + buf [buflen] = 0; + + if (buflen) + { + const size_t readlen = buflen * sizeof (Unicode::unicode_char_t); + if (!fl.read (buf, readlen)) + { + delete[] buf;; + return 0; + } + } + + // TODO: swap all values if needed on big endian systems + // TODO: Localized strings are stored little endian! + + loc_str = new LocalizedString(id, crcSource, buf); + assert (loc_str != NULL); //lint !e1924 // c-style cast. MSVC bug + + loc_str->resetLineCounts (); + + delete[] buf; + buf = 0; + + return loc_str; +} + +//----------------------------------------------------------------- + +void LocalizedString::resetLineCounts () +{ + size_t pos = 0; + const size_t len = m_str.length (); + + std::vector lineStarts; + + for (;;) + { + pos = m_str.find ('\n', pos+1); + + if (pos < len) + lineStarts.push_back (pos+1); + else + break; + } + + if (m_lineStarts) + { + delete[] m_lineStarts; + m_lineStarts = 0; + } + + const size_t num = lineStarts.size (); + m_numLines = num + 1; + + if (m_numLines > 1) + { + m_lineStarts = new size_t [m_numLines]; + m_lineStarts [0] = 0; + + for (size_t i = 0; i < num; ++i) + { + m_lineStarts [i+1] = lineStarts [i]; + } + } +} + +//----------------------------------------------------------------- + +void LocalizedString::buildCrc() +{ + Unicode::unicode_char_t const * unicodeString = m_str.c_str(); + size_t stringSize = m_str.length() * sizeof(Unicode::unicode_char_t); + + m_crc = generateCrc(unicodeString, stringSize); +} + + +// ====================================================================== diff --git a/external/ours/library/localization/src/shared/LocalizedString.h b/external/ours/library/localization/src/shared/LocalizedString.h new file mode 100644 index 00000000..bd3b8cb3 --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizedString.h @@ -0,0 +1,180 @@ +// ====================================================================== +// +// LocalizedString.h +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#ifndef INCLUDED_LocalizedString_H +#define INCLUDED_LocalizedString_H + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +// unref inline func removed +#pragma warning (disable:4514) +// symbol name too long +#pragma warning (disable:4786) +#endif + +#include "Unicode.h" + +// ====================================================================== + +/** +* LocalizedString is a class that wraps a Unicode string, a string id, +* and a timestamp indicating the last modification to this string on disk. +* The string id is an index into localization map, in this case the +* LocalizedStringTable. +*/ + +class LocalizedStringTableRW; +class AbstractFile; + +class LocalizedString +{ +public: + friend class LocalizedStringTableRW; + + // TODO: make these typedefs platform dependent + + typedef unsigned long id_type; + typedef unsigned long crc_type; + + LocalizedString(id_type id, crc_type sourceCrc, Unicode::String const & str); + LocalizedString(id_type id, Unicode::String const & str); // GENERATE Crc FROM UNICODE. + + LocalizedString (const LocalizedString & rhs); + LocalizedString & operator= (const LocalizedString & rhs); + LocalizedString (); + ~LocalizedString (); + bool operator== (const LocalizedString & rhs); + + + + id_type getId () const; + crc_type getCrc() const; + const Unicode::String & getString () const; + + size_t getNumLines () const; + const Unicode::String getStringLine (size_t num) const; + + static LocalizedString * load_0000 (AbstractFile & fl); + static LocalizedString * load_0001(AbstractFile & fl); + + + static crc_type const nullCrc; + void buildCrc(); + + // rls - The source crc is used with translated string pairs. The sourceCrc + // is the Crc of the string that was used to generate the translatd text. + void setSourceCrc(crc_type const crc); + crc_type getSourceCrc() const; + +protected: + + Unicode::String m_str; + id_type m_id; + + crc_type m_crc; + crc_type m_sourceCrc; + + //-- not stored in file + size_t m_numLines; + size_t * m_lineStarts; + +protected: + + void resetLineCounts (); + +private: + +}; + +//----------------------------------------------------------------- + +/** +* Get the unique identifier (withing a table) for this LocalizedString. +*/ +inline LocalizedString::id_type LocalizedString::getId () const +{ + return m_id; +} + +//----------------------------------------------------------------- + +/** +* Get the last-modified timestamp. +*/ + +inline LocalizedString::crc_type LocalizedString::getCrc() const +{ + return m_crc; +} + +//----------------------------------------------------------------- + +/** +* Get the actual string value. +*/ + +inline const Unicode::String & LocalizedString::getString () const +{ + return m_str; +} + +//----------------------------------------------------------------- + +/** +* Get the (cached) number of lines in the string. +*/ + +inline size_t LocalizedString::getNumLines () const +{ + return m_numLines; +} + +//----------------------------------------------------------------- + +/** +* Returns a single line of the string. If num is greater or equal than the number of +* available lines, the entire string is returned. +*/ +inline const Unicode::String LocalizedString::getStringLine (size_t num) const +{ + if (m_numLines < 2 || num >= m_numLines) + return m_str; + + if (num == m_numLines - 1) + return m_str.substr (m_lineStarts [num]); + + return m_str.substr (m_lineStarts [num], (m_lineStarts [num+1] - m_lineStarts [num] - 1)); +} + + +//----------------------------------------------------------------- + +/** +* Set the source crc. +*/ + +inline void LocalizedString::setSourceCrc(LocalizedString::crc_type const crc) +{ + m_sourceCrc = crc; +} + +//----------------------------------------------------------------- + +/** +* Get the source crc. +*/ + +inline LocalizedString::crc_type LocalizedString::getSourceCrc() const +{ + return m_sourceCrc; +} + + +// ====================================================================== + +#endif diff --git a/external/ours/library/localization/src/shared/LocalizedStringTable.cpp b/external/ours/library/localization/src/shared/LocalizedStringTable.cpp new file mode 100644 index 00000000..a14f922f --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizedStringTable.cpp @@ -0,0 +1,502 @@ +// ====================================================================== +// +// LocalizedStringTable.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstLocalization.h" +#include "LocalizedStringTable.h" + +#include "fileInterface/AbstractFile.h" +#include "LocalizedString.h" +#include "UnicodeUtils.h" + +#include +#include +#include + +// ====================================================================== + +namespace +{ + struct AbbrevFinder + { + const std::string & m_str; + + AbbrevFinder (const std::string & str) : m_str (str) {} + + bool operator () (const LocalizedStringTable::NameMap_t::value_type & t) const + { +#if defined(WIN32) + return _strnicmp (t.first.c_str (), m_str.c_str (), m_str.length ()) == 0; +#elif defined(linux) + return strncasecmp (t.first.c_str (), m_str.c_str (), m_str.length ()) == 0; +#endif + } + + AbbrevFinder (const AbbrevFinder & rhs) : m_str (rhs.m_str) {} + + private: + AbbrevFinder & operator=(const AbbrevFinder &); + }; + + //---------------------------------------------------------------------- + + const std::string & getValidString () + { + static std::string valid; + static bool init = false; + + if (!init) + { + valid.reserve (38); // 26 alphabet + 10 digits + '_' + '+' + int c = 0; + + for (c = 'a'; c <= 'z'; ++c) + valid.push_back (static_cast(c)); + + for (c = '0'; c <= '9'; ++c) + valid.push_back (static_cast(c)); + + valid.push_back ('_'); + valid.push_back ('+'); + init = true; + } + + return valid; + } + + //---------------------------------------------------------------------- + + // Set the current version of the LocalizedString + int const FILE_VERSION = 1; +} + +//---------------------------------------------------------------------- + +const LocalizedStringTable::magic_type LocalizedStringTable::ms_MAGIC = 0xabcd; + +//----------------------------------------------------------------- + +LocalizedStringTable::LocalizedStringTable (const std::string & filename) : +m_name (), +m_filename (filename), +m_map (), +m_nameMap (), +m_nextUniqueId (0), +m_version(FILE_VERSION), +m_referenceCount (0) +{ +} + +//----------------------------------------------------------------- + +LocalizedStringTable::LocalizedStringTable () : +m_name (), +m_filename (0), +m_map (), +m_nameMap (), +m_nextUniqueId (0), +m_version(FILE_VERSION), +m_referenceCount (0) +{ +} + +//----------------------------------------------------------------- + +LocalizedStringTable::LocalizedStringTable (const LocalizedStringTable & rhs) : +m_name (rhs.m_name), +m_filename (rhs.m_filename), +m_map (rhs.m_map), +m_nameMap (), +m_nextUniqueId (rhs.m_nextUniqueId), +m_version(FILE_VERSION), +m_referenceCount (0) +{ + // TODO: make reference count copy? - jww +} + +//----------------------------------------------------------------- + +LocalizedStringTable::~LocalizedStringTable () +{ + for (Map_t::iterator iter = m_map.begin (); iter != m_map.end (); ++iter) + { + delete (*iter).second; + (*iter).second = 0; + } + + m_map.clear (); +} +//----------------------------------------------------------------- + + +/** +* Load a version 0000 stringtable from a file stream +*/ + +bool LocalizedStringTable::load_0000 (AbstractFile & fl) +{ + if (!fl.read (&m_nextUniqueId, sizeof (LocalizedString::id_type))) + return false; + + LocalizedString::id_type num_entries; + + if (!fl.read (&num_entries, sizeof (LocalizedString::id_type))) + return false; + + // load the string table + + { + for (size_t i = 0; i < num_entries; ++i) + { + LocalizedString * loc_str = LocalizedString::load_0000 (fl); + + if (loc_str == 0) + return false; + + assert (m_map.find (loc_str->getId ()) == m_map.end ()); //lint !e1924 // c-style cast. MSVC bug + + std::pair retval = m_map.insert (Map_t::value_type (loc_str->getId (), loc_str)); + + // failed to insert string into map? + assert (retval.second == true); //lint !e1924 // c-style cast. MSVC bug + + if (retval.second == false) + return false; + + m_nextUniqueId = std::max (m_nextUniqueId, loc_str->getId () + 1); + } + } + + // load the name lookup table for (size_t i = 0; i < num_entries; ++i) + + { + for (size_t i = 0; i < num_entries; ++i) + { + LocalizedString::id_type id; + + if (!fl.read (&id, sizeof (LocalizedString::id_type))) + return false; + + LocalizedString::id_type buflen; + + if (!fl.read (&buflen, sizeof (LocalizedString::id_type))) + return false; + + // buflen does not include null terminator + char * buf = new char [buflen+1]; + + assert (buf != NULL); //lint !e1924 // c-style cast. MSVC bug + + buf [buflen] = 0; + + if (buflen && !fl.read (buf, buflen)) + { + delete[] buf; + buf = 0; + return false; + } + + std::pair retval = m_nameMap.insert (NameMap_t::value_type (buf, id)); + + delete[] buf; + buf = 0; + + // failed to insert pair into map? + assert (retval.second == true); //lint !e1924 // c-style cast. MSVC bug + + if (retval.second == false) + { + return false; + } + } + } + + setVersion(0); + + return true; +} + +// ---------------------------------------------------------------------- + +/** +* Load a version 0001 stringtable from a file stream +*/ + +bool LocalizedStringTable::load_0001(AbstractFile & fl) +{ + if (!fl.read (&m_nextUniqueId, sizeof (LocalizedString::id_type))) + return false; + + LocalizedString::id_type num_entries; + + if (!fl.read (&num_entries, sizeof (LocalizedString::id_type))) + return false; + + // load the string table + + { + for (size_t i = 0; i < num_entries; ++i) + { + LocalizedString * loc_str = LocalizedString::load_0001(fl); + + if (loc_str == 0) + return false; + + assert (m_map.find (loc_str->getId ()) == m_map.end ()); //lint !e1924 // c-style cast. MSVC bug + + std::pair retval = m_map.insert (Map_t::value_type (loc_str->getId (), loc_str)); + + // failed to insert string into map? + assert (retval.second == true); //lint !e1924 // c-style cast. MSVC bug + + if (retval.second == false) + return false; + + m_nextUniqueId = std::max (m_nextUniqueId, loc_str->getId () + 1); + } + } + + // load the name lookup table for (size_t i = 0; i < num_entries; ++i) + + { + for (size_t i = 0; i < num_entries; ++i) + { + LocalizedString::id_type id; + + if (!fl.read (&id, sizeof (LocalizedString::id_type))) + return false; + + LocalizedString::id_type buflen; + + if (!fl.read (&buflen, sizeof (LocalizedString::id_type))) + return false; + + // buflen does not include null terminator + char * buf = new char [buflen+1]; + + assert (buf != NULL); //lint !e1924 // c-style cast. MSVC bug + + buf [buflen] = 0; + + if (buflen && !fl.read (buf, buflen)) + { + delete[] buf; + buf = 0; + return false; + } + + std::pair retval = m_nameMap.insert (NameMap_t::value_type (buf, id)); + + delete[] buf; + buf = 0; + + // failed to insert pair into map? + assert (retval.second == true); //lint !e1924 // c-style cast. MSVC bug + + if (retval.second == false) + { + return false; + } + } + } + + setVersion(1); + + return true; +} + + +// ---------------------------------------------------------------------- + +/** +* Load a table from a file specified by filename. +*/ + +LocalizedStringTable *LocalizedStringTable::load (AbstractFileFactory & fileFactory, const std::string & filename) +{ + + char version = -1; + + AbstractFile * fl = openLoadFile (fileFactory, filename, version); + + if (fl == 0) + return 0; + + LocalizedStringTable * table = 0; + + switch (version) + { + case 0: + table = new LocalizedStringTable (filename); + assert (table != NULL); //lint !e1924 // c-style cast. MSVC bug + + if (table->load_0000 (*fl) == false) + { + delete table; + table = 0; + } + break; + + case 1: + table = new LocalizedStringTable (filename); + assert (table != NULL); //lint !e1924 // c-style cast. MSVC bug + + if (table->load_0001 (*fl) == false) + { + delete table; + table = 0; + } + break; + + default: + assert (true); //lint !e1924 // c-style cast. MSVC bug + break; + } + + delete fl; + return table; +} + +//----------------------------------------------------------------- + +/** +* +*/ + +AbstractFile * LocalizedStringTable::openLoadFile (AbstractFileFactory & fileFactory, const std::string & filename, char & version) +{ + static_cast(filename); + + AbstractFile * fl = fileFactory.createFile (filename.c_str(), "rb"); + + if (fl == 0) + return 0; + + magic_type magic; + + if (!fl->read (&magic, sizeof(magic_type))) + { + delete fl; + return 0; + } + + // TODO: swab magic if big endian + + // wrong magic file type found in file header + if (magic != LocalizedStringTable::ms_MAGIC) + { + delete fl; + return 0; + } + + char local_version; + + if (!fl->read (&local_version, sizeof(char))) + { + delete fl; + return 0; + } + + version = local_version; + + return fl; +} + +//----------------------------------------------------------------- + +/** +* The name must be cased the same as the key +*/ + +LocalizedString::id_type LocalizedStringTable::getIdByAbbrev (const std::string & name, std::string & fullName) const +{ + + NameMap_t::const_iterator it = std::find_if (m_nameMap.begin (), m_nameMap.end (), AbbrevFinder (name)); + if (it != m_nameMap.end ()) + { + fullName = (*it).first; + return (*it).second; + } + + return 0; +} + +//----------------------------------------------------------------- + +/** +* Find an id by case-insensitive abbreviation. Slow linear search w/ string manipulation overhead. +*/ + +LocalizedString::id_type LocalizedStringTable::getIdByAbbrevNocase (const std::string & name, std::string & fullName) const +{ + return getIdByAbbrev (Unicode::toLower (name), fullName); +} + +//----------------------------------------------------------------- + +/** +* Find an id by name. Slow linear search. +*/ + +const std::string * LocalizedStringTable::getStringNameByIdSlowly (size_t id) const +{ + for ( LocalizedStringTable::NameMap_t::const_iterator it = m_nameMap.begin (); it != m_nameMap.end (); ++it) + { + if ((*it).second == id) + return &(*it).first; + } + + return 0; +} + +//---------------------------------------------------------------------- + +bool LocalizedStringTable::validateStringName (const std::string & name) +{ + if (name.empty ()) + return false; + + const std::string & valid = getValidString (); + + //-- name must begin with a lowercase alpha + return isalpha (name [0]) && name.find_first_not_of (valid) == name.npos; +} + +//---------------------------------------------------------------------- + +void LocalizedStringTable::fixupStringName (std::string & name) +{ + if (name.empty ()) + return; + + for (size_t i = 0; i < name.size ();) + { + const char c = name [i]; + if (isalpha (c)) + name [i] = static_cast(tolower (c)); + else if (c == ' ') + name [i] = '_'; + else if (!isdigit (c) && c != '_') + { + name.erase (i, 1); + continue; + } + ++i; + } + + while (!name.empty () && !isalpha (name [0])) + name.erase (static_cast(0), 1); +} + + +//---------------------------------------------------------------------- + +char LocalizedStringTable::getCurrentVersion() +{ + return FILE_VERSION; +} + +// ====================================================================== diff --git a/external/ours/library/localization/src/shared/LocalizedStringTable.h b/external/ours/library/localization/src/shared/LocalizedStringTable.h new file mode 100644 index 00000000..cea6635e --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizedStringTable.h @@ -0,0 +1,203 @@ +// ====================================================================== +// +// LocalizedStringTable.h +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#ifndef INCLUDED_LocalizedStringTable_H +#define INCLUDED_LocalizedStringTable_H + + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +// unref inline func removed +#pragma warning (disable:4514) +// symbol name too long +#pragma warning (disable:4786) +#endif + +#include "Unicode.h" +#include "LocalizedString.h" + +#include + +class LocalizedString; +class AbstractFile; +class AbstractFileFactory; + +// ====================================================================== + +/** +* A LocalizedStringTable maps ids and names to LocalizedString. +* The fastest lookup is done via find-by-id. +*/ + +class LocalizedStringTable +{ +public: + + class StringSearchPredicate; + friend class LocalizationManager; + + + typedef std::map Map_t; + typedef std::map NameMap_t; + + // TODO: make this typedef platform dependent + typedef long magic_type; + static const magic_type ms_MAGIC; + + explicit LocalizedStringTable (const std::string & filename); + virtual ~LocalizedStringTable (); + + const std::string & getName () const; + const std::string & getFileName () const; + const Map_t & getMap () const; + const NameMap_t & getNameMap () const; + + static LocalizedStringTable * load (AbstractFileFactory & fileFactory, const std::string & filename); + + const LocalizedString * getLocalizedString (LocalizedString::id_type id) const; + const LocalizedString * getLocalizedString (const std::string & name) const; + LocalizedString::id_type getIdByName (const std::string & name) const; + LocalizedString::id_type getIdByAbbrev (const std::string & name, std::string & fullName) const; + LocalizedString::id_type getIdByAbbrevNocase (const std::string & name, std::string & fullName) const; + + const std::string * getStringNameByIdSlowly (size_t id) const; + + static bool validateStringName (const std::string & name); + static void fixupStringName (std::string & name); + + char getVersion() const; + static char getCurrentVersion(); + +protected: + + LocalizedStringTable (const LocalizedStringTable & rhs); + + bool load_0000(AbstractFile & fl); + bool load_0001(AbstractFile & fl); + + static AbstractFile * openLoadFile (AbstractFileFactory & fileFactory, const std::string & filename, char & version); + + void setVersion(char version); + + std::string m_name; + std::string m_filename; + Map_t m_map; + NameMap_t m_nameMap; + LocalizedString::id_type m_nextUniqueId; + char m_version; + + /** + * used by LocalizationManager + */ + mutable size_t m_referenceCount; + + +private: + LocalizedStringTable (); + LocalizedStringTable & operator= (const LocalizedStringTable & rhs); +}; + +//----------------------------------------------------------------- + +/** +* Get the short name of this table. +*/ + +inline const std::string & LocalizedStringTable::getName () const +{ + return m_name; +} + +//----------------------------------------------------------------- + +/** +* Get the absolute filename of this table. +*/ + +inline const std::string & LocalizedStringTable::getFileName () const +{ + return m_filename; +} + +//----------------------------------------------------------------- + +/** +* Get a const reference to the id -> LocalizedString map +*/ + +inline const LocalizedStringTable::Map_t & LocalizedStringTable::getMap () const +{ + return m_map; +} + +//----------------------------------------------------------------- + +/** +* Get a const reference to the name -> id map +*/ + +inline const LocalizedStringTable::NameMap_t & LocalizedStringTable::getNameMap () const +{ + return m_nameMap; +} + +//----------------------------------------------------------------- + +/** +* Find a LocalizedString by id. Fast map lookup +*/ + +inline const LocalizedString * LocalizedStringTable::getLocalizedString (LocalizedString::id_type id) const +{ + Map_t::const_iterator iter = m_map.find (id); + + return (iter != m_map.end ()) ? (*iter).second : 0; +} + +//----------------------------------------------------------------- + +/** +* Find an id by name. Fast map lookup. +* index zero is special and indicates a failed lookup. +*/ + +inline LocalizedString::id_type LocalizedStringTable::getIdByName (const std::string & name) const +{ + NameMap_t::const_iterator iter = m_nameMap.find (name); + + return (iter != m_nameMap.end ()) ? (*iter).second : 0; +} + +//----------------------------------------------------------------- + +/** +* Find a LocalizedString by id. Done via 2 fast map lookups. +*/ + +inline const LocalizedString * LocalizedStringTable::getLocalizedString (const std::string & name) const +{ + return getLocalizedString (getIdByName (name)); +} + +//----------------------------------------------------------------- + +inline char LocalizedStringTable::getVersion() const +{ + return m_version; +} + +//----------------------------------------------------------------- + +inline void LocalizedStringTable::setVersion(char const version) +{ + m_version = version; +} + +// ====================================================================== + +#endif diff --git a/external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.cpp b/external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.cpp new file mode 100644 index 00000000..c10c7fb0 --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.cpp @@ -0,0 +1,553 @@ +// ====================================================================== +// +// LocalizedStringTableRW.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstLocalization.h" +#include "LocalizedStringTableReaderWriter.h" + +#include "LocalizedString.h" +#include "fileInterface/AbstractFile.h" +#include "UnicodeUtils.h" +#include + +#include +#include + +namespace LocalizedStringTableReaderWriterNamespace +{ + void unfubarMicrosoftInvalidTextCharacters (Unicode::String & str) + { + typedef std::pair FubarCharacterInfo; + typedef std::pair FubarStringInfo; + + static const FubarCharacterInfo FubarCharactersCourtesyMicrosoft [] = + { + FubarCharacterInfo (139, '<'), + FubarCharacterInfo (155, '>'), + + FubarCharacterInfo (8216, '\''), + FubarCharacterInfo (8217, '\''), + FubarCharacterInfo (8221, '"'), + FubarCharacterInfo (8220, '"'), + FubarCharacterInfo (149, '-'), + FubarCharacterInfo (150, '-'), + FubarCharacterInfo (151, '-'), + FubarCharacterInfo (152, '~') + }; + + static const int FubarCharactersCourtesyMicrosoftLength = sizeof (FubarCharactersCourtesyMicrosoft) / sizeof (FubarCharactersCourtesyMicrosoft[0]); + + static const FubarStringInfo FubarStringsCourtesyMicrosoft [] = + { + FubarStringInfo (133, Unicode::narrowToWide ("...")), + //-- carriage return + FubarStringInfo (13, Unicode::narrowToWide ("")) + }; + + static const int FubarStringsCourtesyMicrosoftLength = sizeof (FubarStringsCourtesyMicrosoft) / sizeof (FubarStringsCourtesyMicrosoft[0]); + + { + for (int i = 0; i < FubarCharactersCourtesyMicrosoftLength; ++i) + { + const Unicode::unicode_char_t correctCharacter = FubarCharactersCourtesyMicrosoft [i].second; + const Unicode::unicode_char_t fubarCharacter = FubarCharactersCourtesyMicrosoft [i].first; + + size_t pos = 0; + while ((pos = str.find (fubarCharacter, pos)) != Unicode::String::npos) + { + str [pos] = correctCharacter; + ++pos; + } + } + } + + { + for (int i = 0; i < FubarStringsCourtesyMicrosoftLength; ++i) + { + const Unicode::unicode_char_t fubarCharacter = FubarStringsCourtesyMicrosoft [i].first; + const Unicode::String & correctString = FubarStringsCourtesyMicrosoft [i].second; + + size_t pos = 0; + + while ((pos = str.find (fubarCharacter, pos)) != Unicode::String::npos) + { + str.replace (pos, 1, correctString); + pos += correctString.size (); + } + } + } + } +} + +using namespace LocalizedStringTableReaderWriterNamespace; + +//----------------------------------------------------------------- + +LocalizedStringTableRW::LocalizedStringTableRW (const Unicode::NarrowString & filename) : +LocalizedStringTable (filename), +m_idNameMap () +{ + +} + +//----------------------------------------------------------------- + +LocalizedStringTableRW::LocalizedStringTableRW (const LocalizedStringTable & rhs) : +LocalizedStringTable (rhs), +m_idNameMap () +{ + +} + +//----------------------------------------------------------------- + +bool LocalizedStringTableRW::str_write(AbstractFile & fl, LocalizedString & locstr) +{ + // TODO: swab all values on big endian systems + + LocalizedString::id_type id = locstr.m_id; + LocalizedString::crc_type crcSource = locstr.m_sourceCrc; + LocalizedString::id_type buflen = locstr.m_str.length (); + + if (!fl.write (sizeof (LocalizedString::id_type), &id)) + return false; + + if (!fl.write (sizeof (LocalizedString::crc_type), &crcSource)) + return false; + + if (!fl.write (sizeof (LocalizedString::id_type), &buflen)) + return false; + + // TODO: swab this buffer + Unicode::unicode_char_t * buf = new Unicode::unicode_char_t [buflen+1]; + + assert (buf != NULL); //lint !e1924 // c-style cast. MSVC bug + + buf [buflen] = 0; + memcpy (buf, locstr.m_str.c_str (), sizeof (Unicode::unicode_char_t) * buflen); + + if (buflen && !fl.write (buflen * sizeof (Unicode::unicode_char_t), buf)) + { + delete[] buf; + return false; + } + + delete[] buf; + + return true; +} + +//----------------------------------------------------------------- + +bool LocalizedStringTableRW::write(AbstractFile & fl) const +{ + // TODO: swab these values on big endian systems + LocalizedString::id_type next_unique = m_nextUniqueId; + LocalizedString::id_type num_entries = m_map.size (); + + if (!fl.write (sizeof (LocalizedString::id_type), &next_unique)) + return false; + + if (!fl.write (sizeof (LocalizedString::id_type), &num_entries)) + return false; + + // write the string table + + { + for (Map_t::const_iterator iter = m_map.begin (); iter != m_map.end (); ++iter) + { + if (str_write(fl, *((*iter).second)) == false) + return false; + } + } + + // write the string/id map + { + for (NameMap_t::const_iterator iter = m_nameMap.begin (); iter != m_nameMap.end (); ++iter) + { + + // TODO: swab all these values on big endian systems + LocalizedString::id_type id = (*iter).second; + LocalizedString::id_type buflen = (*iter).first.length (); + + if (!fl.write (sizeof (LocalizedString::id_type), &id)) + return false; + + if (!fl.write (sizeof (LocalizedString::id_type), &buflen)) + return false; + + char * buf = new char [buflen + 1]; + + assert (buf != NULL); //lint !e1924 // c-style cast. MSVC bug + + buf [buflen] = 0; + + memcpy (buf, (*iter).first.c_str (), buflen); + + if (buflen && !fl.write (sizeof (char) * buflen, buf)) + { + delete[] buf; + buf = 0; + return false; + } + + delete[] buf; + buf = 0; + } + } + + return true; +} + + +// ---------------------------------------------------------------------- +/** + */ + +LocalizedStringTableRW *LocalizedStringTableRW::loadRW (AbstractFileFactory & fileFactory, const Unicode::NarrowString & filename) +{ + char version = -1; + + AbstractFile * const fl = openLoadFile (fileFactory, filename, version); + + if (fl == 0) + return 0; + + LocalizedStringTableRW * table = 0; + + if (!fl->isOpen ()) + { + + } + else + { + switch (version) + { + case 0: + table = new LocalizedStringTableRW (filename); + assert (table != NULL); //lint !e1924 // c-style cast. MSVC bug + + if (table->load_0000 (*fl) == false) + { + delete table; + table = 0; + } + break; + + case 1: + table = new LocalizedStringTableRW (filename); + assert (table != NULL); //lint !e1924 // c-style cast. MSVC bug + + if (table->load_0001 (*fl) == false) + { + delete table; + table = 0; + } + break; + default: + assert (true); //lint !e1924 // c-style cast. MSVC bug + break; + } + } + + delete fl; + + //Go through all the strings and un-microsoft-fubar them + for(Map_t::iterator mapIter = table->m_map.begin(); mapIter != table->m_map.end(); ++mapIter) + { + LocalizedString *str = (*mapIter).second; + Unicode::String & strRef = str->m_str; + LocalizedStringTableReaderWriterNamespace::unfubarMicrosoftInvalidTextCharacters(strRef); + } + + // populate idNameMap + if (table) + { + table->m_idNameMap.clear (); + + for (NameMap_t::const_iterator iter = table->m_nameMap.begin (); iter != table->m_nameMap.end (); ++iter) + { + table->m_idNameMap.insert (IdNameMap_t::value_type ((*iter).second, (*iter).first)); + } + } + + return table; +} + +// ---------------------------------------------------------------------- +/** + */ + +bool LocalizedStringTableRW::writeRW (AbstractFileFactory & fileFactory, const Unicode::NarrowString & filename) const +{ + //Go through all the strings and un-microsoft-fubar them + for(Map_t::const_iterator mapIter = m_map.begin(); mapIter != m_map.end(); ++mapIter) + { + LocalizedString *str = (*mapIter).second; + Unicode::String & strRef = str->m_str; + LocalizedStringTableReaderWriterNamespace::unfubarMicrosoftInvalidTextCharacters(strRef); + } + + AbstractFile * const fl = fileFactory.createFile (filename.c_str (), "wb"); + + if (fl == 0) + return false; + + bool retval = false; + + if (!fl->isOpen ()) + { + retval = false; + } + else + { + // TODO: swab magic if big endian + + magic_type local_magic = LocalizedStringTable::ms_MAGIC; + + if (!fl->write (sizeof (magic_type), &local_magic)) + { + delete fl; + return false; + } + + char currentVersion = LocalizedStringTable::getCurrentVersion(); + if (!fl->write(sizeof (char), ¤tVersion)) + { + delete fl; + return false; + } + + retval = write(*fl); + } + + delete fl; + return retval; +} + +//----------------------------------------------------------------- + +LocalizedString * LocalizedStringTableRW::addString (const Unicode::String & str, Unicode::NarrowString & theNameResult) +{ + LocalizedString * const locstr = (m_map [m_nextUniqueId] = new LocalizedString (m_nextUniqueId, int(time(0)), str)); + + assert (locstr != NULL); //lint !e1924 // c-style cast. MSVC bug + + char buf[64]; + sprintf (buf, "%03ld_default", m_nextUniqueId); + + Unicode::NarrowString name(buf); + + const std::pair retval = m_nameMap.insert (NameMap_t::value_type (name, m_nextUniqueId)); + + assert (retval.second == true); //lint !e1924 // c-style cast. MSVC bug + + m_idNameMap [locstr->getId ()] = name; + + theNameResult = name; + + ++m_nextUniqueId; + + return retval.second == true ? locstr : 0; +} + +//----------------------------------------------------------------- + +LocalizedString * LocalizedStringTableRW::addString (LocalizedString * locstr, const Unicode::NarrowString & name, std::string & resultStr) +{ + char buf [1024]; + const size_t id = locstr->getId (); + + { + Map_t::const_iterator it = m_map.find (id); + if (it != m_map.end ()) + { + const std::string * old_name = getNameById (id); + const LocalizedString * old_locstr = (*it).second; + const std::string old_str = old_locstr ? Unicode::wideToNarrow (old_locstr->getString ()) : std::string (); + + sprintf (buf, + "LocalizedStringTableRW::addString failed inserting duplicate id [%d]\n" + "Existing string name=[%s], str=[%s]\n", + id, + old_name ? old_name->c_str () : "", + old_str.c_str ()); + resultStr += buf; + return 0; + } + } + + { + NameMap_t::const_iterator it = m_nameMap.find (name); + if (it != m_nameMap.end ()) + { + const size_t old_id = (*it).second; + const std::string * old_name = getNameById (old_id); + const LocalizedString * old_locstr = getLocalizedString (old_id); + const std::string old_str = old_locstr ? Unicode::wideToNarrow (old_locstr->getString ()) : std::string (); + + sprintf (buf, + "LocalizedStringTableRW::addString failed inserting duplicate name [%s]\n" + "Existing string id=[%d] name=[%s], str=[%s]\n", + name.c_str (), + id, + old_name ? old_name->c_str () : "", + old_str.c_str ()); + resultStr += buf; + return 0; + } + } + + { + IdNameMap_t::const_iterator it = m_idNameMap.find (id); + if (it != m_idNameMap.end ()) + { + const std::string & old_name = (*it).second; + const LocalizedString * old_locstr = getLocalizedString (id); + const std::string old_str = old_locstr ? Unicode::wideToNarrow (old_locstr->getString ()) : std::string (); + + sprintf (buf, + "LocalizedStringTableRW::addString failed inserting duplicate name [%s]\n" + "Existing string name=[%s], str=[%s]\n", + name.c_str (), + old_name.c_str (), + old_str.c_str ()); + resultStr += buf; + return 0; + } + } + + m_map [id] = locstr; + m_nameMap [name] = locstr->getId (); + m_idNameMap [id] = name; + + m_nextUniqueId = std::max (m_nextUniqueId, locstr->getId () + 1); + + return locstr; +} + +//---------------------------------------------------------------------- + +LocalizedString * LocalizedStringTableRW::addString (LocalizedString * locstr, const Unicode::String & name, std::string & resultStr) +{ + return addString (locstr, Unicode::wideToNarrow (name), resultStr); +} + +//----------------------------------------------------------------- + +bool LocalizedStringTableRW::removeStringByName (const Unicode::NarrowString & name) +{ + + NameMap_t::iterator find_iter_id = m_nameMap.find (name); + + if (find_iter_id == m_nameMap.end ()) + return false; + + const size_t id = (*find_iter_id).second; + + Map_t::iterator find_iter_string = m_map.find (id); + + assert (find_iter_string != m_map.end ()); //lint !e1924 // c-style cast. MSVC bug + + IdNameMap_t::iterator find_iter_id_map = m_idNameMap.find (id); + + assert (find_iter_id_map != m_idNameMap.end ()); //lint !e1924 // c-style cast. MSVC bug + + m_nameMap.erase (find_iter_id); + m_map.erase (find_iter_string); + m_idNameMap.erase (find_iter_id_map); + + return true; +} + +//----------------------------------------------------------------- + +bool LocalizedStringTableRW::rename (const Unicode::NarrowString & name, const Unicode::NarrowString & newName) +{ + // + if (name == newName) + return false; + + // names that start with numerals are special + if (isdigit (newName [0])) + return false; + + NameMap_t::iterator find_iter_id = m_nameMap.find (newName); + + // newName already exists + if (find_iter_id != m_nameMap.end ()) + { + + LocalizedString::id_type id = (*find_iter_id).second; + + LocalizedString * locstr = getLocalizedString (id); + + static_cast(locstr); + + return false; + + } + + find_iter_id = m_nameMap.find (name); + + // old name does not exist + if (find_iter_id == m_nameMap.end ()) + return false; + + size_t id = (*find_iter_id).second; + + m_nameMap.erase (find_iter_id); + + const std::pair retval = m_nameMap.insert (NameMap_t::value_type (newName, id)); + + assert (retval.second == true); //lint !e1924 // c-style cast. MSVC bug + + m_idNameMap [id] = newName; + + return retval.second == true; +} + +//----------------------------------------------------------------- + +void LocalizedStringTableRW::setName (const Unicode::NarrowString & name) +{ + m_name = name; +} + +//----------------------------------------------------------------- + +void LocalizedStringTableRW::prepareTable (const LocalizedStringTableRW & rhs) +{ + m_idNameMap.clear (); + m_nameMap.clear (); + m_map.clear (); + + m_idNameMap = rhs.m_idNameMap; + m_nameMap = rhs.m_nameMap; + m_map.clear (); + m_nextUniqueId = rhs.m_nextUniqueId; + m_name = rhs.m_name; + + for (Map_t::const_iterator iter = rhs.m_map.begin (); iter != rhs.m_map.end (); ++iter) + { + const LocalizedString * rhs_locstr = (*iter).second; + + LocalizedString * locstr = new LocalizedString (rhs_locstr->m_id, 0, rhs_locstr->m_str); + + assert (locstr != NULL); //lint !e1924 // c-style cast. MSVC bug + m_map.insert (Map_t::value_type (locstr->getId (), locstr )); + } +} + +//----------------------------------------------------------------- + +LocalizedString * LocalizedStringTableRW::getLocalizedStringByName (const Unicode::String & name) +{ + return getLocalizedStringByName(Unicode::wideToNarrow (name)); +} + + +// ====================================================================== diff --git a/external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.h b/external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.h new file mode 100644 index 00000000..699c5c75 --- /dev/null +++ b/external/ours/library/localization/src/shared/LocalizedStringTableReaderWriter.h @@ -0,0 +1,187 @@ +// ====================================================================== +// +// LocalizedStringTableRW.h +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#ifndef INCLUDED_LocalizedStringTableRW_H +#define INCLUDED_LocalizedStringTableRW_H + +#if WIN32 +// stl warning func not inlined +#pragma warning (disable:4710) +// unref inline func removed +#pragma warning (disable:4514) +// symbol name too long +#pragma warning (disable:4786) +#endif + +#include "LocalizedStringTable.h" +#include "LocalizedString.h" + +#include +#include +#include +#include + +class AbstractFile; + +//----------------------------------------------------------------- + +class LocalizedStringTableRW : public LocalizedStringTable +{ +public: + + class StringRW; + + typedef std::map IdNameMap_t; + + explicit LocalizedStringTableRW (const Unicode::NarrowString & name); + explicit LocalizedStringTableRW (const LocalizedStringTable & rhs); + + LocalizedString::id_type getNextUniqueId () const; + void setNextUniqueId (LocalizedString::id_type id); + + void setName (const Unicode::NarrowString & name); + Map_t & getMap (); //lint !e1511 // member hides non-virtual member + + const Unicode::NarrowString * getNameById (LocalizedString::id_type id) const; + + static LocalizedStringTableRW * loadRW (AbstractFileFactory & fileFactory, const Unicode::NarrowString & filename); + + bool write (AbstractFile & fl) const; + bool writeRW (AbstractFileFactory & fileFactory, const Unicode::NarrowString & filename) const; + + LocalizedString * addString (const Unicode::String & str, Unicode::NarrowString & theNameResult); + LocalizedString * addString (LocalizedString * locstr, const Unicode::NarrowString & name, std::string & resultStr); + LocalizedString * addString (LocalizedString * locstr, const Unicode::String & name, std::string & resultStr); + + bool removeStringByName (const Unicode::NarrowString & name); + LocalizedString * getLocalizedStringByName (const Unicode::NarrowString & name); + LocalizedString * getLocalizedStringByName (const Unicode::String & name); + LocalizedString * getLocalizedString (LocalizedString::id_type id); //lint !e1511 // member hides non-virtual member + + bool rename (const Unicode::NarrowString & name, const Unicode::NarrowString & newName); + + static bool str_write (AbstractFile & fl, LocalizedString & locstr); + + static const Unicode::String & str_getString (LocalizedString & locstr); + static void str_setString (LocalizedString & locstr, const Unicode::String & str); + static LocalizedString::id_type & str_getId (LocalizedString & locstr); + static LocalizedString::crc_type & str_getCrc(LocalizedString & locstr); + static LocalizedString::crc_type & str_getSourceCrc(LocalizedString & locstr); + + void prepareTable (const LocalizedStringTableRW & rhs); + + IdNameMap_t & getIdNameMap (); + NameMap_t & getNameMap (); //lint !e1511 // member hides non-virtual member + +private: + LocalizedStringTableRW (); + LocalizedStringTableRW (const LocalizedStringTableRW & rhs); + LocalizedStringTableRW & operator= (const LocalizedStringTableRW & rhs); + + IdNameMap_t m_idNameMap; +}; + +//----------------------------------------------------------------- + +inline LocalizedString::id_type LocalizedStringTableRW::getNextUniqueId () const +{ + return m_nextUniqueId; +} + +//----------------------------------------------------------------- + +inline void LocalizedStringTableRW::setNextUniqueId (const LocalizedString::id_type id) +{ + m_nextUniqueId = id; +} + +//----------------------------------------------------------------- + +inline LocalizedStringTableRW::Map_t & LocalizedStringTableRW::getMap () +{ + return m_map; //lint !e1536 // exposing low access member +} + +//----------------------------------------------------------------- + +inline const Unicode::String & LocalizedStringTableRW::str_getString (LocalizedString & locstr) +{ + return locstr.m_str; +} + +//----------------------------------------------------------------- + +inline void LocalizedStringTableRW::str_setString (LocalizedString & locstr, const Unicode::String & str) +{ + locstr.m_str = str; + locstr.buildCrc(); + locstr.resetLineCounts (); +} + +//----------------------------------------------------------------- + +inline LocalizedString::id_type & LocalizedStringTableRW::str_getId (LocalizedString & locstr) +{ + return locstr.m_id; +} + +//----------------------------------------------------------------- + +inline LocalizedString::crc_type & LocalizedStringTableRW::str_getCrc (LocalizedString & locstr) +{ + return locstr.m_crc; +} + +//----------------------------------------------------------------- + +inline LocalizedString::crc_type & LocalizedStringTableRW::str_getSourceCrc(LocalizedString & locstr) +{ + return locstr.m_sourceCrc; +} + +//----------------------------------------------------------------- + +inline LocalizedString * LocalizedStringTableRW::getLocalizedStringByName (const Unicode::NarrowString & name) +{ + return getLocalizedString (getIdByName (name)); +} + +//----------------------------------------------------------------- + +inline LocalizedString * LocalizedStringTableRW::getLocalizedString (LocalizedString::id_type id) +{ + Map_t::const_iterator iter = m_map.find (id); + + return (iter != m_map.end ()) ? (*iter).second : 0; +} + +//----------------------------------------------------------------- + +inline const Unicode::NarrowString * LocalizedStringTableRW::getNameById (LocalizedString::id_type id) const +{ + IdNameMap_t::const_iterator iter = m_idNameMap.find (id); + + return (iter != m_idNameMap.end ()) ? &(*iter).second : 0; +} + +//----------------------------------------------------------------- + +inline LocalizedStringTableRW::IdNameMap_t & LocalizedStringTableRW::getIdNameMap () +{ + return m_idNameMap; //lint !e1536 // exposing low access member +} + +//----------------------------------------------------------------- + +inline LocalizedStringTable::NameMap_t & LocalizedStringTableRW::getNameMap () +{ + return m_nameMap; //lint !e1536 // exposing low access member +} + +// ====================================================================== + +#endif diff --git a/external/ours/library/localization/src/shared/StringId.cpp b/external/ours/library/localization/src/shared/StringId.cpp new file mode 100644 index 00000000..9e6345d5 --- /dev/null +++ b/external/ours/library/localization/src/shared/StringId.cpp @@ -0,0 +1,245 @@ +//======================================================================== +// +// StringId.cpp - Used to access an entry in a string table. +// +// copyright 2001 Sony Online Entertainment +// +//======================================================================== + +#include "FirstLocalization.h" +#include "StringId.h" + +#include "LocalizationManager.h" +#include "UnicodeUtils.h" +#include +#include + +//---------------------------------------------------------------------- + +const StringId StringId::cms_invalid; + +//---------------------------------------------------------------------- + +StringId::StringId () : +m_table (), +m_text (), +m_textIndex (0) +{ +} + +//---------------------------------------------------------------------- + +StringId::StringId (const std::string & table, const std::string & text) : +m_table (table), +m_text (text), +m_textIndex (0) +{ +} + +//---------------------------------------------------------------------- + +StringId::StringId (const std::string & table, unsigned long textIndex) : +m_table (table), +m_text (), +m_textIndex (textIndex) +{ +} + +//---------------------------------------------------------------------- + +StringId::StringId (const std::string & str) : +m_table (), +m_text (), +m_textIndex (0) +{ + const size_t colonpos = str.find (':'); + + if (colonpos != std::string::npos) + { + m_table = str.substr (0, colonpos); + if (!m_table.empty () && m_table [0] == '@') + m_table.erase (m_table.begin ()); + + std::string second = str.substr (colonpos + 1); + + if (!second.empty ()) + { + if (isdigit (second [0])) + m_textIndex = atoi (second.c_str ()); + else + m_text = second; + } + } + else + m_text = str; + +} + +//---------------------------------------------------------------------- + +/** + * Debug output, for console e.g. + */ +std::string StringId::getDebugString() const +{ + return (m_table + ":" + m_text); +} + +// ---------------------------------------------------------------------- + +std::string StringId::getCanonicalRepresentation () const +{ + return (m_table + ":" + m_text); +} + +//---------------------------------------------------------------------- + +/** +* @return true if this stringId is definitely invalid +*/ +bool StringId::isInvalid () const +{ + return m_table.empty () || (m_text.empty () && m_textIndex == 0); +} + +//---------------------------------------------------------------------- + +bool StringId::isValid () const +{ + return !m_table.empty () && (!m_text.empty () || m_textIndex != 0); +} + +//---------------------------------------------------------------------- + +Unicode::String StringId::decodeString (const Unicode::String & str) +{ + if (str.empty ()) + return str; + + static Unicode::String result; + result.clear (); + + size_t pos = 0; + + //-- an encoded string can be a series of tokens, seperated by nulls + //-- each token can be an encoded stringid ("@table:name") or a literal string + + while (pos != std::string::npos) + { + size_t nullpos = str.find (Unicode::unicode_char_t (0), pos); + + if (pos == 0 && nullpos == std::string::npos) + { + if (str [0] == '@') + { + const StringId sid (Unicode::wideToNarrow (str)); + return sid.localize (); + } + else + return str; + } + + const Unicode::String & token = nullpos == std::string::npos ? (pos == 0 ? str : str.substr (pos)) : str.substr (pos, nullpos - pos); + + if (token.empty ()) + break; + + if (token [0] == '@') + { + const StringId sid (Unicode::wideToNarrow (token)); + result += sid.localize (); + } + else + result += token; + + if (nullpos == std::string::npos) + break; + + pos = nullpos + 1; + } + + return result; +} + +//---------------------------------------------------------------------- + +StringId StringId::decodeStringId (const Unicode::String & str) +{ + return StringId (Unicode::wideToNarrow (str.c_str () + 1)); +} + +//---------------------------------------------------------------------- + +void StringId::clear () +{ + m_table.clear (); + m_text.clear (); + m_textIndex = 0; +} + +//---------------------------------------------------------------------- + +bool StringId::localize (Unicode::String & result, bool forceEnglish) const +{ + return LocalizationManager::getManager ().getLocalizedStringValue (*this, result, forceEnglish) == LocalizationManager::SVC_ok; +} + +//---------------------------------------------------------------------- + +Unicode::String StringId::localize (bool forceEnglish) const +{ + static Unicode::String str; + str.clear (); + localize (str, forceEnglish); + return str; +} + +//---------------------------------------------------------------------- + +bool StringId::localize (Unicode::String & result, const Unicode::NarrowString & locale, bool forceEnglish) const +{ + return LocalizationManager::getManager ( locale ).getLocalizedStringValue (*this, result, forceEnglish) == LocalizationManager::SVC_ok; +} + +//---------------------------------------------------------------------- + +Unicode::String StringId::localize (const Unicode::NarrowString & locale, bool forceEnglish) const +{ + static Unicode::String str; + str.clear (); + localize (str, locale, forceEnglish); + return str; +} + +//---------------------------------------------------------------------- + +/** +* Setting the text invalidates the cached text index +*/ + +void StringId::setTable (const std::string & table) +{ + m_table = table; + m_textIndex = 0; +} + +//---------------------------------------------------------------------- + +/** +* Setting the text invalidates the cached text index +*/ + +void StringId::setText (const std::string & text) +{ + m_text = text; + m_textIndex = 0; +} + +//---------------------------------------------------------------------- + +void StringId::setTextIndex (unsigned long textIndex) const +{ + m_textIndex = textIndex; +} + +//---------------------------------------------------------------------- diff --git a/external/ours/library/localization/src/shared/StringId.h b/external/ours/library/localization/src/shared/StringId.h new file mode 100644 index 00000000..4b4572df --- /dev/null +++ b/external/ours/library/localization/src/shared/StringId.h @@ -0,0 +1,153 @@ +//======================================================================== +// +// StringId.h - Used to access an entry in a string table. +// +// copyright 2001 Sony Online Entertainment +// +//======================================================================== + +#ifndef _INCLUDED_StringId_H +#define _INCLUDED_StringId_H + +#include +#include "Unicode.h" + +// ====================================================================== + +namespace DB +{ + class BufferString; +} + +//======================================================================== +// class StringId + +class StringId +{ + friend int operator - (const StringId & lhs, const StringId & rhs); + friend bool operator != (const StringId & lhs, const StringId & rhs); + friend bool operator == (const StringId & lhs, const StringId & rhs); + friend bool operator < (const StringId & lhs, const StringId & rhs); + +public: + + static const StringId cms_invalid; + + //-- LocalizationUnicodeString is simply a typedef that is equivalent to Unicode::String + //-- it is defined here to prevent #including Unicode.h + typedef std::basic_string LocUnicodeString; + + StringId (); + StringId (const std::string & table, const std::string & text); + StringId (const std::string & table, unsigned long textIndex); + explicit StringId (const std::string & canonicalRepresentation); + + const std::string & getTable () const; + const std::string & getText () const; + unsigned long getTextIndex () const; + + bool isInvalid () const; + bool isValid () const; + void clear (); + + std::string getDebugString () const; + + std::string getCanonicalRepresentation () const; + + void setTable (const std::string & table); + void setText (const std::string & text); + + //-- text index is a mutable property of the StringId + //-- it is set the first time a StringId is localized, to speed up future lookups + + void setTextIndex (unsigned long textIndex) const; + + bool localize (LocUnicodeString & result, bool forceEnglish = false) const; + LocUnicodeString localize (bool forceEnglish = false) const; + bool localize (LocUnicodeString & result, const Unicode::NarrowString & locale, bool forceEnglish = false) const; + LocUnicodeString localize (const Unicode::NarrowString & locale, bool forceEnglish = false) const; + + static LocUnicodeString decodeString (const LocUnicodeString & str); + static StringId decodeStringId (const LocUnicodeString & str); + +public: + // These are defined in MiscPack.cpp, to avoid making this library require linking with the database library + void unpackFromDatabase (const DB::BufferString &table, const DB::BufferString &text); + void packToDatabase (DB::BufferString &table, DB::BufferString &text) const; + +private: + + std::string m_table; + std::string m_text; + mutable unsigned long m_textIndex; //if this number is nonzero, assume it is valid +}; + +//---------------------------------------------------------------------- + +inline const std::string & StringId::getTable() const +{ + return m_table; +} + +//---------------------------------------------------------------------- + +inline const std::string & StringId::getText() const +{ + return m_text; +} + +//---------------------------------------------------------------------- + +inline unsigned long StringId::getTextIndex() const +{ + return m_textIndex; +} + +//======================================================================== +// friend functions + +inline int operator -(const StringId & lhs, const StringId & rhs) +{ + return (lhs != rhs) ? 1 : 0; +} + +//---------------------------------------------------------------------- + +inline bool operator !=(const StringId & lhs, const StringId & rhs) +{ + return !(lhs == rhs); +} + +//---------------------------------------------------------------------- + +inline bool operator ==(const StringId & lhs, const StringId & rhs) +{ + if (&lhs == &rhs) + return true; + if (lhs.m_table != rhs.m_table) + return false; + if (lhs.m_text != rhs.m_text) + return false; + if (lhs.m_text.empty() && lhs.m_textIndex != rhs.m_textIndex) + return false; + return true; +} + +//---------------------------------------------------------------------- + +inline bool operator <(const StringId & lhs, const StringId & rhs) +{ + if (&lhs == &rhs) + return false; + if (lhs.m_table != rhs.m_table) + return lhs.m_table < rhs.m_table; + if (lhs.m_text != rhs.m_text) + return lhs.m_text < rhs.m_text; + if (lhs.m_text.empty() && lhs.m_textIndex != rhs.m_textIndex) + return lhs.m_textIndex < rhs.m_textIndex; + return false; +} + +//---------------------------------------------------------------------- + +#endif // _INCLUDED_StringId_H diff --git a/external/ours/library/localization/src/win32/FirstLocalization.cpp b/external/ours/library/localization/src/win32/FirstLocalization.cpp new file mode 100644 index 00000000..c21d66d0 --- /dev/null +++ b/external/ours/library/localization/src/win32/FirstLocalization.cpp @@ -0,0 +1,8 @@ +// ====================================================================== +// +// FirstLocalization.cpp +// copyright (c) 2001 Sony Online Entertainment +// +// ====================================================================== + +#include "FirstLocalization.h"