moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 05:55:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

avoid using thread local on Windows due to https://developercommunity.visualstudio.com/t/thread-local-variables-fail-to-be-initialized-when/124121

Browse Source
This commit is contained in:
Hieu Hoang 2024-03-14 13:14:06 -07:00
parent 0fc225c004
commit b1b7237305
5 changed files with 191 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
contrib/other-builds/moses2/moses2.sln
View File

@ -9,30 +9,30 @@ Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Debug-DLL|Win32 = Debug-DLL|Win32
Debug-DLL|x64 = Debug-DLL|x64
Debug-exe|Win32 = Debug-exe|Win32
Debug-exe|x64 = Debug-exe|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
Release-DLL|Win32 = Release-DLL|Win32
Release-DLL|x64 = Release-DLL|x64
Release-exe|Win32 = Release-exe|Win32
Release-exe|x64 = Release-exe|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|Win32.ActiveCfg = Debug|Win32
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|Win32.Build.0 = Debug|Win32
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|x64.ActiveCfg = Debug|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|x64.Build.0 = Debug|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-DLL|Win32.ActiveCfg = Debug-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-DLL|Win32.Build.0 = Debug-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-DLL|x64.ActiveCfg = Debug-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-DLL|x64.Build.0 = Debug-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|Win32.ActiveCfg = Debug-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|Win32.Build.0 = Debug-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|x64.ActiveCfg = Debug-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|x64.Build.0 = Debug-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|Win32.ActiveCfg = Release|Win32
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|Win32.Build.0 = Release|Win32
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|x64.ActiveCfg = Release|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|x64.Build.0 = Release|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-DLL|Win32.ActiveCfg = Release-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-DLL|Win32.Build.0 = Release-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-DLL|x64.ActiveCfg = Release-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-DLL|x64.Build.0 = Release-DLL|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|Win32.ActiveCfg = Release-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|Win32.Build.0 = Release-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|x64.ActiveCfg = Release-exe|x64
{B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|x64.Build.0 = Release-exe|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

99
contrib/other-builds/moses2/moses2.vcxproj
View File

@ -1,16 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug-DLL|x64">
<Configuration>Debug-DLL</Configuration>
<ProjectConfiguration Include="Debug-exe|x64">
<Configuration>Debug-exe</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release-DLL|x64">
<Configuration>Release-DLL</Configuration>
<ProjectConfiguration Include="Release-exe|x64">
<Configuration>Release-exe</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
@ -29,6 +29,13 @@
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport>
<UseOfMfc>false</UseOfMfc>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-exe|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
@ -43,6 +50,12 @@
<UseOfMfc>false</UseOfMfc>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-exe|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
@ -58,24 +71,36 @@
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-exe|Win32'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-DLL|Win32'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-exe|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-DLL|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-exe|Win32'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-DLL|Win32'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-exe|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-DLL|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
@ -84,6 +109,10 @@
<LinkIncremental>true</LinkIncremental>
<TargetName>$(ProjectName)decoder</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-exe|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>$(ProjectName)decoder</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-DLL|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>$(ProjectName)decoder</TargetName>
@ -92,6 +121,10 @@
<LinkIncremental>false</LinkIncremental>
<TargetName>$(ProjectName)decoder</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-exe|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>$(ProjectName)decoder</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-DLL|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>$(ProjectName)decoder</TargetName>
@ -126,6 +159,36 @@
<SuppressStartupBanner>false</SuppressStartupBanner>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-exe|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
<SuppressStartupBanner>false</SuppressStartupBanner>
<ShowIncludes>false</ShowIncludes>
<LanguageStandard>Default</LanguageStandard>
<LanguageStandard_C>
</LanguageStandard_C>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<CompileAsManaged>false</CompileAsManaged>
<AdditionalOptions>
</AdditionalOptions>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64</AdditionalLibraryDirectories>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<ShowProgress>LinkVerbose</ShowProgress>
<SuppressStartupBanner>false</SuppressStartupBanner>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-DLL|x64'">
<ClCompile>
<PrecompiledHeader>
@ -166,7 +229,33 @@
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
<ShowIncludes>false</ShowIncludes>
<LanguageStandard>Default</LanguageStandard>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64</AdditionalLibraryDirectories>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release-exe|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
<ShowIncludes>false</ShowIncludes>
<LanguageStandard>Default</LanguageStandard>

1
moses2/MemPool.cpp
View File

@ -55,6 +55,7 @@ uint8_t* MemPool::Allocate(std::size_t size) {
uint8_t* ret = current_;
current_ += size;
assert(m_currPage < m_pages.size());
Page& page = *m_pages[m_currPage];
if (current_ <= page.end) {
// return what we got

110
moses2/System.cpp
View File

@ -19,10 +19,11 @@ using namespace std;
namespace Moses2
{
thread_local MemPool System::m_managerPool;
thread_local MemPool System::m_systemPool;
thread_local Recycler<HypothesisBase*> System::m_hypoRecycler;
#ifndef WIN32
thread_local MemPool System::m_managerPool;
thread_local MemPool System::m_systemPool;
thread_local Recycler<HypothesisBase*> System::m_hypoRecycler;
#endif // WIN32
System::System(const Parameter &paramsArg) :
params(paramsArg), featureFunctions(*this)
@ -165,38 +166,6 @@ void System::LoadDecodeGraphBackoff()
}
}
MemPool &System::GetSystemPool() const
{
return m_systemPool;
}
MemPool &System::GetManagerPool() const
{
return m_managerPool;
}
FactorCollection &System::GetVocab() const
{
return m_vocab;
}
Recycler<HypothesisBase*> &System::GetHypoRecycler() const
{
return m_hypoRecycler;
}
Batch &System::GetBatch(MemPool &pool) const
{
Batch *obj;
obj = m_batch.get();
if (obj == NULL) {
obj = new Batch(pool);
m_batch.reset(obj);
}
assert(obj);
return *obj;
}
void System::IsPb()
{
switch (options.search.algo) {
@ -219,6 +188,75 @@ void System::IsPb()
}
}
FactorCollection& System::GetVocab() const
{
return m_vocab;
}
//////////////////////////////////////////////////////
// thread local stuff
Batch& System::GetBatch(MemPool& pool) const
{
Batch* obj;
obj = m_batch.get();
if (obj == NULL) {
obj = new Batch(pool);
m_batch.reset(obj);
}
assert(obj);
return *obj;
}
#ifdef WIN32
template<class C>
C& GetThreadSpecificObj(boost::thread_specific_ptr<C> &threadSpecificPtr)
{
C* obj;
obj = threadSpecificPtr.get();
if (obj == NULL) {
obj = new C();
threadSpecificPtr.reset(obj);
}
assert(obj);
return *obj;
}
MemPool& System::GetManagerPool() const
{
MemPool &obj = GetThreadSpecificObj<MemPool>(m_managerPool);
return obj;
}
MemPool& System::GetSystemPool() const
{
MemPool& obj = GetThreadSpecificObj<MemPool>(m_systemPool);
return obj;
}
Recycler<HypothesisBase*>& System::GetHypoRecycler() const
{
Recycler<HypothesisBase*>& obj = GetThreadSpecificObj<Recycler<HypothesisBase*> >(m_hypoRecycler);
return obj;
}
#else
MemPool& System::GetManagerPool() const
{
return m_managerPool;
}
MemPool& System::GetSystemPool() const
{
return m_systemPool;
}
Recycler<HypothesisBase*>& System::GetHypoRecycler() const
{
return m_hypoRecycler;
}
#endif
}

15
moses2/System.h Normal file → Executable file
View File

@ -65,15 +65,20 @@ public:
protected:
mutable FactorCollection m_vocab;
//mutable boost::thread_specific_ptr<MemPool> m_managerPool;
//mutable boost::thread_specific_ptr<MemPool> m_systemPool;
mutable boost::thread_specific_ptr<Batch> m_batch;
#ifdef WIN32
mutable boost::thread_specific_ptr<MemPool> m_managerPool;
mutable boost::thread_specific_ptr<MemPool> m_systemPool;
mutable boost::thread_specific_ptr <Recycler<HypothesisBase*> > m_hypoRecycler;
#else
thread_local static MemPool m_managerPool;
thread_local static MemPool m_systemPool;
thread_local static Recycler<HypothesisBase*> m_hypoRecycler;
//thread_local static MemPool d;
mutable boost::thread_specific_ptr<Batch> m_batch;
#endif
void LoadWeights();
void LoadMappings();