From 20a1a5a6dc5d3d9edd92efb536a71838558e28fe Mon Sep 17 00:00:00 2001 From: randoman <738b86bb93c44695854182cc459afcbb@lonestar.no> Date: Tue, 3 Sep 2019 22:22:50 +0200 Subject: [PATCH] Version 4.2.0 * CHANGE - Changed how splitting regexes are configured to supports scopes + more. --- CHANGELOG.md | 5 +- README.md | 21 ++- src/XUnity.AutoTranslator.Patcher/Patcher.cs | 2 +- ...nity.AutoTranslator.Plugin.BepIn-5x.csproj | 2 +- .../XUnity.AutoTranslator.Plugin.BepIn.csproj | 2 +- .../AutoTranslationPlugin.cs | 16 +- .../Configuration/Settings.cs | 8 +- .../Constants/PluginData.cs | 2 +- .../Parsing/GameLogTextParser.cs | 2 +- .../Parsing/ParserResult.cs | 38 ++-- .../Parsing/RegexSplittingTextParser.cs | 38 ++-- .../Parsing/RegexTranslationSplitter.cs | 61 ++++++- .../Parsing/RichTextParser.cs | 2 +- .../Parsing/UnityTextParsers.cs | 4 +- .../RegexTranslation.cs | 10 +- .../TextTranslationCache.cs | 167 ++++++++++++++---- .../XUnity.AutoTranslator.Plugin.Core.csproj | 2 +- .../XUnity.AutoTranslator.Plugin.IPA.csproj | 2 +- ...AutoTranslator.Plugin.UnityInjector.csproj | 2 +- .../XUnity.AutoTranslator.Setup.csproj | 2 +- 20 files changed, 292 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a3bbbf..ee9a7cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ -### 4.1.0 +### 4.2.0 + * CHANGE - Changed how splitting regexes are configured to supports scopes + more. + +### 4.1.0 * FEATURE - Added new concept that allows splitting texts to be translated with a regex before trying to translate them * MISC - Improved NGUI text resizing behaviour diff --git a/README.md b/README.md index fb046e2f..1a4040d6 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,6 @@ FromLanguage=ja ;The original language of the game Directory=Translation ;Directory to search for cached translation files. Can use placeholder: {GameExeName} OutputFile=Translation\_AutoGeneratedTranslations.{lang}.txt ;File to insert generated translations into. Can use placeholders: {GameExeName}, {lang} SubstitutionFile=Translation\_Substitutions.{lang}.txt ;File that contains substitution applied before translations. Can use placeholders: {GameExeName}, {lang} -PatternFile=Translation\_SplitterPatterns.{lang}.txt ;File that contains regexes that are applied to translatable texts before performing a translation to split them up into individual parts [TextFrameworks] EnableUGUI=True ;Enable or disable UGUI translation @@ -486,7 +485,21 @@ When creating manual translations, use this file as sparingly as you would use r *NOTE: If the text to be translated includes rich text, it cannot currently be parameterized.* -### Regex Splitting +### Regex Usage +Text translation files support regexes as well. Always remember to use regexes as sparing as possible and scope them as much as possible to avoid performance issues. + +Regexes can be applied to translations in two different ways. The following two sections describes these two ways: + +#### Standard Regex Translation +Standard regex translation are simply regexes that applied directly onto a translatable text, if no direct lookup can be found. + +``` +r:"^シンプルリング ([0-9]+)$"=Simple Ring $1 +``` + +These are identified by the untranslated text starting with 'r:'. + +#### Splitter Regex Sometimes games likes to combine texts before displaying them on screen. This means that it can sometimes be hard to know what text to add to the translation file because it appears in a number of different ways. This section explores a solution to this by applying a regex to split the text to be translated into individual pieces before trying to make lookups for the specified texts. @@ -498,11 +511,13 @@ However, if we split the translation before trying to make lookups it will allow Simply place the following regex in the `PatternFile` file: ``` -([0-9]{2}) ([\S\s]+)=$1 $2 +sr:"^([0-9]{2}) ([\S\s]+)$"=$1 $2 ``` This will split up the text to be translated into two parts, translate them individually and put them back together. +These are identified by the untranslated text starting with 'sr:'. + ### Translation Scoping The following two options are available when it comes to scoping translations to only part of the game: diff --git a/src/XUnity.AutoTranslator.Patcher/Patcher.cs b/src/XUnity.AutoTranslator.Patcher/Patcher.cs index 0ee19ea5..c8de9518 100644 --- a/src/XUnity.AutoTranslator.Patcher/Patcher.cs +++ b/src/XUnity.AutoTranslator.Patcher/Patcher.cs @@ -29,7 +29,7 @@ public override string Version { get { - return "4.1.0"; + return "4.2.0"; } } diff --git a/src/XUnity.AutoTranslator.Plugin.BepIn-5x/XUnity.AutoTranslator.Plugin.BepIn-5x.csproj b/src/XUnity.AutoTranslator.Plugin.BepIn-5x/XUnity.AutoTranslator.Plugin.BepIn-5x.csproj index 557a32b2..a5a6ddda 100644 --- a/src/XUnity.AutoTranslator.Plugin.BepIn-5x/XUnity.AutoTranslator.Plugin.BepIn-5x.csproj +++ b/src/XUnity.AutoTranslator.Plugin.BepIn-5x/XUnity.AutoTranslator.Plugin.BepIn-5x.csproj @@ -2,7 +2,7 @@ net35 - 4.1.0 + 4.2.0 diff --git a/src/XUnity.AutoTranslator.Plugin.BepIn/XUnity.AutoTranslator.Plugin.BepIn.csproj b/src/XUnity.AutoTranslator.Plugin.BepIn/XUnity.AutoTranslator.Plugin.BepIn.csproj index bd8014d2..dfcf9420 100644 --- a/src/XUnity.AutoTranslator.Plugin.BepIn/XUnity.AutoTranslator.Plugin.BepIn.csproj +++ b/src/XUnity.AutoTranslator.Plugin.BepIn/XUnity.AutoTranslator.Plugin.BepIn.csproj @@ -2,7 +2,7 @@ net35 - 4.1.0 + 4.2.0 diff --git a/src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs b/src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs index 4ddef290..72e00b6e 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs @@ -126,7 +126,7 @@ public void Initialize() SpamChecker = new SpamChecker( TranslationManager ); // WORKAROUND: Initialize text parsers with delegate indicating if text should be translated - UnityTextParsers.Initialize( ( text, scope ) => TextCache.IsTranslatable( text, true, scope ) && IsBelowMaxLength( text ) ); + UnityTextParsers.Initialize( TextCache, ( text, scope ) => TextCache.IsTranslatable( text, true, scope ) && IsBelowMaxLength( text ) ); // resource redirectors InitializeResourceRedirector(); @@ -1341,6 +1341,7 @@ private string TranslateByParserResult( TranslationEndpointManager endpoint, Par { Dictionary translations = new Dictionary(); + var allowPartial = endpoint == null && result.AllowPartialTranslation; var context = new ParserTranslationContext( null, endpoint, translationResult, result ); if( isGlobal ) { @@ -1359,6 +1360,10 @@ private string TranslateByParserResult( TranslationEndpointManager endpoint, Par { translations.Add( variableName, textKey.Untemplate( partTranslation ) ); } + else if( allowPartial ) + { + translations.Add( variableName, textKey.Untemplate( textKey.TemplatedOriginal_Text ) ); + } else if( allowStartJob ) { // incomplete, must start job @@ -1409,6 +1414,10 @@ private string TranslateByParserResult( TranslationEndpointManager endpoint, Par { translations.Add( variableName, textKey.Untemplate( partTranslation ) ); } + else if( allowPartial ) + { + translations.Add( variableName, textKey.Untemplate( textKey.TemplatedOriginal_Text ) ); + } else if( allowStartJob ) { // incomplete, must start job @@ -1731,6 +1740,7 @@ private string TranslateOrQueueWebJobImmediateByParserResult( object ui, ParserR Dictionary translations = new Dictionary(); // attempt to lookup ALL strings immediately; return result if possible; queue operations + var allowPartial = TranslationManager.CurrentEndpoint == null && result.AllowPartialTranslation; var context = new ParserTranslationContext( ui, TranslationManager.CurrentEndpoint, null, result ); foreach( var kvp in result.Arguments ) { @@ -1747,6 +1757,10 @@ private string TranslateOrQueueWebJobImmediateByParserResult( object ui, ParserR { translations.Add( variableName, textKey.Untemplate( partTranslation ) ); } + else if( allowPartial ) + { + translations.Add( variableName, textKey.Untemplate( textKey.TemplatedOriginal_Text ) ); + } else if( allowStartJob ) { // incomplete, must start job diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs b/src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs index 9a4766e9..d4f35526 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs @@ -41,7 +41,7 @@ internal static class Settings public static string RedirectedResourcesPath; public static Dictionary Replacements = new Dictionary(); - public static List Patterns = new List(); + //public static List Patterns = new List(); public static bool SimulateError = false; public static bool SimulateDelayedError = false; @@ -66,13 +66,13 @@ internal static class Settings public static string FromLanguage; public static string OutputFile; public static string SubstitutionFile; - public static string PatternFile; + //public static string PatternFile; public static string TranslationDirectory; public static int MaxCharactersPerTranslation; public static bool EnableConsole; public static string AutoTranslationsFilePath; public static string SubstitutionFilePath; - public static string PatternFilePath; + //public static string PatternFilePath; public static bool EnableIMGUI; public static bool EnableUGUI; public static bool EnableNGUI; @@ -148,7 +148,6 @@ public static void Configure() TranslationDirectory = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "Directory", "Translation" ); OutputFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "OutputFile", @"Translation\_AutoGeneratedTranslations.{lang}.txt" ); SubstitutionFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "SubstitutionFile", @"Translation\_Substitutions.{lang}.txt" ); - PatternFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "PatternFile", @"Translation\_SplitterPatterns.{lang}.txt" ); EnableIMGUI = PluginEnvironment.Current.Preferences.GetOrDefault( "TextFrameworks", "EnableIMGUI", false ); EnableUGUI = PluginEnvironment.Current.Preferences.GetOrDefault( "TextFrameworks", "EnableUGUI", true ); @@ -236,7 +235,6 @@ public static void Configure() AutoTranslationsFilePath = Path.Combine( PluginEnvironment.Current.TranslationPath, OutputFile.Replace( "{lang}", Language ) ).Replace( "/", "\\" ).Parameterize(); SubstitutionFilePath = Path.Combine( PluginEnvironment.Current.TranslationPath, SubstitutionFile.Replace( "{lang}", Language ) ).Replace( "/", "\\" ).Parameterize(); - PatternFilePath = Path.Combine( PluginEnvironment.Current.TranslationPath, PatternFile.Replace( "{lang}", Language ) ).Replace( "/", "\\" ).Parameterize(); FromLanguageUsesWhitespaceBetweenWords = LanguageHelper.RequiresWhitespaceUponLineMerging( FromLanguage ); ToLanguageUsesWhitespaceBetweenWords = LanguageHelper.RequiresWhitespaceUponLineMerging( Language ); diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Constants/PluginData.cs b/src/XUnity.AutoTranslator.Plugin.Core/Constants/PluginData.cs index 39363fff..32e218bc 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Constants/PluginData.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Constants/PluginData.cs @@ -23,6 +23,6 @@ public static class PluginData /// /// Gets the version of the plugin. /// - public const string Version = "4.1.0"; + public const string Version = "4.2.0"; } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/GameLogTextParser.cs b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/GameLogTextParser.cs index 1f25e73d..c44c54cd 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/GameLogTextParser.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/GameLogTextParser.cs @@ -68,7 +68,7 @@ public ParserResult Parse( string input, int scope ) if( !containsTranslatable ) return null; if( !input.EndsWith( "\r\n" ) && !input.EndsWith( "\n" ) ) template.Remove( template.Length - 1, 1 ); - return new ParserResult( input, template.ToString(), false, false, true, args ); + return new ParserResult( input, template.ToString(), false, false, false, true, args ); } } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/ParserResult.cs b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/ParserResult.cs index 932420f6..85a47a07 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/ParserResult.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/ParserResult.cs @@ -1,38 +1,56 @@ using System.Collections.Generic; +using System.Linq; +using System.Text; namespace XUnity.AutoTranslator.Plugin.Core.Parsing { internal class ParserResult { - public ParserResult( string originalText, string template, bool cacheCombinedResult, bool persistCombinedResult, bool persistTokenResult, Dictionary args ) + public ParserResult( string originalText, string template, bool allowPartialTranslation, bool cacheCombinedResult, bool persistCombinedResult, bool persistTokenResult, Dictionary args ) { OriginalText = originalText; Template = template; + AllowPartialTranslation = allowPartialTranslation; CacheCombinedResult = cacheCombinedResult; PersistCombinedResult = persistCombinedResult; PersistTokenResult = persistTokenResult; Arguments = args; } - public string OriginalText { get; private set; } + public string OriginalText { get; } - public string Template { get; private set; } - public Dictionary Arguments { get; private set; } + public string Template { get; } + public Dictionary Arguments { get; } + + public bool AllowPartialTranslation { get; } public bool CacheCombinedResult { get; } - public bool PersistCombinedResult { get; private set; } + public bool PersistCombinedResult { get; } - public bool PersistTokenResult { get; private set; } + public bool PersistTokenResult { get; } public string Untemplate( Dictionary arguments ) { - string result = Template; - foreach( var kvp in arguments ) + // This is really not a nice fix... + if( arguments.Count > 9 ) + { + var result = new StringBuilder( Template ); + foreach( var kvp in arguments.OrderByDescending( x => x.Key.Length ) ) + { + result = result.Replace( kvp.Key, kvp.Value ); + } + return result.ToString(); + } + else { - result = result.Replace( kvp.Key, kvp.Value ); + var result = new StringBuilder( Template ); + foreach( var kvp in arguments ) + { + result = result.Replace( kvp.Key, kvp.Value ); + } + return result.ToString(); } - return result; } } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexSplittingTextParser.cs b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexSplittingTextParser.cs index 573f9c92..e41474b5 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexSplittingTextParser.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexSplittingTextParser.cs @@ -7,6 +7,13 @@ namespace XUnity.AutoTranslator.Plugin.Core.Parsing { internal class RegexSplittingTextParser : ITextParser { + private readonly TextTranslationCache _cache; + + public RegexSplittingTextParser( TextTranslationCache cache ) + { + _cache = cache; + } + public bool CanApply( object ui ) { return !ui.IsSpammingComponent(); @@ -14,28 +21,21 @@ public bool CanApply( object ui ) public ParserResult Parse( string input, int scope ) { - var patterns = Settings.Patterns; - var length = patterns.Count; - for( int i = 0; i < length; i++ ) + if( _cache.TryGetTranslationSplitter( input, scope, out var match, out var splitter ) ) { - var regex = patterns[ i ]; - var m = regex.CompiledRegex.Match( input ); - if( m.Success ) - { - var args = new Dictionary(); - - var groups = m.Groups; - var len = groups.Count; - for( int j = 1; j < len; j++ ) - { - var group = groups[ j ]; - var groupName = "$" + j; - var value = group.Value; - args.Add( groupName, value ); - } + var args = new Dictionary(); - return new ParserResult( input, regex.Translation, true, Settings.CacheRegexPatternResults, true, args ); + var groups = match.Groups; + var len = groups.Count; + for( int j = 1; j < len; j++ ) + { + var group = groups[ j ]; + var groupName = "$" + j; + var value = group.Value; + args.Add( groupName, value ); } + + return new ParserResult( input, splitter.Translation, true, true, Settings.CacheRegexPatternResults, true, args ); } return null; diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexTranslationSplitter.cs b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexTranslationSplitter.cs index af29e71b..715b649d 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexTranslationSplitter.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RegexTranslationSplitter.cs @@ -1,14 +1,63 @@ using System.Text.RegularExpressions; +using XUnity.Common.Logging; namespace XUnity.AutoTranslator.Plugin.Core.Parsing { internal class RegexTranslationSplitter { - public RegexTranslationSplitter( string regex, string translation ) + public RegexTranslationSplitter( string key, string value ) { - CompiledRegex = new Regex( regex ); - Translation = translation; - Original = regex; + Key = key; + Value = value; + + // remove sr: + if( key.StartsWith( "sr:" ) ) + { + key = key.Substring( 3, key.Length - 3 ); + } + + var startIdx = key.IndexOf( '"' ); + if( startIdx == -1 ) + { + // take entire string + } + else + { + startIdx++; + var endIdx = key.LastIndexOf( '"' ); + if( endIdx != startIdx ) + { + key = key.Substring( startIdx, endIdx - startIdx ); + } + } + + // remove sr: + if( value.StartsWith( "sr:" ) ) + { + value = value.Substring( 3, value.Length - 3 ); + } + + startIdx = value.IndexOf( '"' ); + if( startIdx == -1 ) + { + // take entire string + } + else + { + startIdx++; + var endIdx = value.LastIndexOf( '"' ); + if( endIdx != startIdx ) + { + value = value.Substring( startIdx, endIdx - startIdx ); + } + } + + if( !key.StartsWith( "^" ) ) key = "^" + key; + if( !key.EndsWith( "$" ) ) key = key + "$"; + + CompiledRegex = new Regex( key ); + Original = key; + Translation = value; } public Regex CompiledRegex { get; set; } @@ -16,5 +65,9 @@ public RegexTranslationSplitter( string regex, string translation ) public string Original { get; set; } public string Translation { get; set; } + + public string Key { get; set; } + + public string Value { get; set; } } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RichTextParser.cs b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RichTextParser.cs index d08ad21a..860c9a40 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RichTextParser.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/RichTextParser.cs @@ -147,7 +147,7 @@ public ParserResult Parse( string input, int scope ) var success = foundIgnoredTag || ( arg != 'A' && templateString.Length > 5 ); if( !success ) return null; - return new ParserResult( input, templateString, true, true, false, args ); + return new ParserResult( input, templateString, false, true, true, false, args ); } } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/UnityTextParsers.cs b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/UnityTextParsers.cs index 6a069001..d2bcc49a 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/Parsing/UnityTextParsers.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/Parsing/UnityTextParsers.cs @@ -12,10 +12,10 @@ internal static class UnityTextParsers public static RegexSplittingTextParser RegexSplittingTextParser; public static GameLogTextParser GameLogTextParser; - public static void Initialize( Func isTranslatable ) + public static void Initialize( TextTranslationCache cache, Func isTranslatable ) { RichTextParser = new RichTextParser(); - RegexSplittingTextParser = new RegexSplittingTextParser(); + RegexSplittingTextParser = new RegexSplittingTextParser( cache ); GameLogTextParser = new GameLogTextParser( isTranslatable ); } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/RegexTranslation.cs b/src/XUnity.AutoTranslator.Plugin.Core/RegexTranslation.cs index c9a1a272..c9300a49 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/RegexTranslation.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/RegexTranslation.cs @@ -16,14 +16,15 @@ public RegexTranslation( string key, string value ) key = key.Substring( 2, key.Length - 2 ); } - var startIdx = key.IndexOf( '"' ) + 1; + var startIdx = key.IndexOf( '"' ); if( startIdx == -1 ) { // take entire string } else { - var endIdx = key.LastIndexOf( '"', key.Length - 1 ); + startIdx++; + var endIdx = key.LastIndexOf( '"' ); if( endIdx != startIdx ) { key = key.Substring( startIdx, endIdx - startIdx ); @@ -36,14 +37,15 @@ public RegexTranslation( string key, string value ) value = value.Substring( 2, value.Length - 2 ); } - startIdx = value.IndexOf( '"' ) + 1; + startIdx = value.IndexOf( '"' ); if( startIdx == -1 ) { // take entire string } else { - var endIdx = value.LastIndexOf( '"', value.Length - 1 ); + startIdx++; + var endIdx = value.LastIndexOf( '"' ); if( endIdx != startIdx ) { value = value.Substring( startIdx, endIdx - startIdx ); diff --git a/src/XUnity.AutoTranslator.Plugin.Core/TextTranslationCache.cs b/src/XUnity.AutoTranslator.Plugin.Core/TextTranslationCache.cs index 3445acc2..de260eb4 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/TextTranslationCache.cs +++ b/src/XUnity.AutoTranslator.Plugin.Core/TextTranslationCache.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using UnityEngine; using XUnity.AutoTranslator.Plugin.Core.Configuration; using XUnity.AutoTranslator.Plugin.Core.Extensions; @@ -34,6 +35,9 @@ class TextTranslationCache private List _defaultRegexes = new List(); private HashSet _registeredRegexes = new HashSet(); + public List _splitterRegexes = new List(); + public HashSet _registeredSplitterRegexes = new HashSet(); + /// /// These are the new translations that has not yet been persisted to the file system. /// @@ -71,19 +75,18 @@ internal void LoadTranslationFiles() _partialTranslations.Clear(); _tokenTranslations.Clear(); _reverseTokenTranslations.Clear(); + _registeredSplitterRegexes.Clear(); + _splitterRegexes.Clear(); _scopedTranslations.Clear(); Settings.Replacements.Clear(); - Settings.Patterns.Clear(); var mainTranslationFile = Settings.AutoTranslationsFilePath; var substitutionFile = Settings.SubstitutionFilePath; - var patternFile = Settings.PatternFilePath; - LoadTranslationsInFile( mainTranslationFile, false, false, true ); - LoadTranslationsInFile( substitutionFile, true, false, false ); - LoadTranslationsInFile( patternFile, false, true, false ); - foreach( var fullFileName in GetTranslationFiles().Reverse().Except( new[] { mainTranslationFile, substitutionFile, patternFile } ) ) + LoadTranslationsInFile( mainTranslationFile, false, true ); + LoadTranslationsInFile( substitutionFile, true, false ); + foreach( var fullFileName in GetTranslationFiles().Reverse().Except( new[] { mainTranslationFile, substitutionFile } ) ) { - LoadTranslationsInFile( fullFileName, false, false, false ); + LoadTranslationsInFile( fullFileName, false, false ); } } var endTime = Time.realtimeSinceStartup; @@ -201,6 +204,7 @@ internal void LoadTranslationFiles() XuaLogger.AutoTranslator.Info( $"Global translations generated: {_translations.Count}" ); XuaLogger.AutoTranslator.Info( $"Global regex translations generated: {_defaultRegexes.Count}" ); + XuaLogger.AutoTranslator.Info( $"Global regex splitters generated: {_splitterRegexes.Count}" ); XuaLogger.AutoTranslator.Info( $"Global token translations generated: {_tokenTranslations.Count}" ); if( Settings.GeneratePartialTranslations ) { @@ -213,6 +217,7 @@ internal void LoadTranslationFiles() XuaLogger.AutoTranslator.Info( $"Scene {scope} translations generated: {dicts.Translations.Count}" ); XuaLogger.AutoTranslator.Info( $"Scene {scope} regex translations generated: {dicts.DefaultRegexes.Count}" ); + XuaLogger.AutoTranslator.Info( $"Scene {scope} regex splitter generated: {dicts.SplitterRegexes.Count}" ); XuaLogger.AutoTranslator.Info( $"Scene {scope} token translations generated: {dicts.TokenTranslations.Count}" ); } } @@ -306,16 +311,17 @@ public TranslationCharacterToken( char c, bool isVariable ) public bool IsVariable { get; set; } } - private void LoadTranslationsInFile( string fullFileName, bool isSubstitutionFile, bool isPatternFile, bool isOutputFile ) + private void LoadTranslationsInFile( string fullFileName, bool isSubstitutionFile, bool isOutputFile ) { var fileExists = File.Exists( fullFileName ); - if( fileExists || isSubstitutionFile || isPatternFile ) + if( fileExists || isSubstitutionFile ) { if( fileExists ) { XuaLogger.AutoTranslator.Debug( $"Loading texts: {fullFileName}." ); var context = new TranslationFileLoadingContext(); + var set = new HashSet(); string[] translations = File.ReadAllLines( fullFileName, Encoding.UTF8 ); foreach( string translatioOrDirective in translations ) @@ -346,21 +352,33 @@ private void LoadTranslationsInFile( string fullFileName, bool isSubstitutionFil { Settings.Replacements[ key ] = value; } - else if( isPatternFile ) - { - try - { - Settings.Patterns.Add( new RegexTranslationSplitter( key, value ) ); - } - catch( Exception e ) - { - XuaLogger.AutoTranslator.Warn( e, $"An error occurred while constructing regex splitter: '{translatioOrDirective}'." ); - } - } else { + if( key.StartsWith( "sr:" ) ) + { + try + { + var regex = new RegexTranslationSplitter( key, value ); - if( key.StartsWith( "r:" ) ) + var levels = context.GetLevels(); + if( levels.Count == 0 ) + { + AddTranslationSplitterRegex( regex, TranslationScopes.None ); + } + else + { + foreach( var level in levels ) + { + AddTranslationSplitterRegex( regex, level ); + } + } + } + catch( Exception e ) + { + XuaLogger.AutoTranslator.Warn( e, $"An error occurred while constructing regex translation splitter: '{translatioOrDirective}'." ); + } + } + else if( key.StartsWith( "r:" ) ) { try { @@ -413,14 +431,6 @@ private void LoadTranslationsInFile( string fullFileName, bool isSubstitutionFil stream.Close(); } } - else if( isPatternFile ) - { - using( var stream = File.Create( fullFileName ) ) - { - stream.Write( new byte[] { 0xEF, 0xBB, 0xBF }, 0, 3 ); // UTF-8 BOM - stream.Close(); - } - } } } @@ -470,22 +480,51 @@ private void SaveNewTranslationsToDisk() } } + private void AddTranslationSplitterRegex( RegexTranslationSplitter regex, int scope ) + { + if( scope != TranslationScopes.None ) + { + if( !_scopedTranslations.TryGetValue( scope, out var dicts ) ) + { + dicts = new TranslationDictionaries(); + _scopedTranslations.Add( scope, dicts ); + } + + if( !dicts.RegisteredSplitterRegexes.Contains( regex.Original ) ) + { + dicts.RegisteredSplitterRegexes.Add( regex.Original ); + dicts.SplitterRegexes.Add( regex ); + } + } + else + { + if( !_registeredSplitterRegexes.Contains( regex.Original ) ) + { + _registeredSplitterRegexes.Add( regex.Original ); + _splitterRegexes.Add( regex ); + } + } + } + private void AddTranslationRegex( RegexTranslation regex, int scope ) { - if( !_registeredRegexes.Contains( regex.Original ) ) + if( scope != TranslationScopes.None ) { - if( scope != TranslationScopes.None ) + if( !_scopedTranslations.TryGetValue( scope, out var dicts ) ) { - if( !_scopedTranslations.TryGetValue( scope, out var dicts ) ) - { - dicts = new TranslationDictionaries(); - _scopedTranslations.Add( scope, dicts ); - } + dicts = new TranslationDictionaries(); + _scopedTranslations.Add( scope, dicts ); + } + if( !dicts.RegisteredRegexes.Contains( regex.Original ) ) + { dicts.RegisteredRegexes.Add( regex.Original ); dicts.DefaultRegexes.Add( regex ); } - else + } + else + { + if( !_registeredRegexes.Contains( regex.Original ) ) { _registeredRegexes.Add( regex.Original ); _defaultRegexes.Add( regex ); @@ -635,6 +674,56 @@ internal void AddTranslationToCache( string key, string value, bool persistToDis } } + internal bool TryGetTranslationSplitter( string text, int scope, out Match match, out RegexTranslationSplitter splitter ) + { + if( scope != TranslationScopes.None && _scopedTranslations.TryGetValue( scope, out var dicts ) && dicts.SplitterRegexes.Count > 0 ) + { + for( int i = dicts.SplitterRegexes.Count - 1; i > -1; i-- ) + { + var regex = dicts.SplitterRegexes[ i ]; + try + { + match = regex.CompiledRegex.Match( text ); + splitter = regex; + if( match.Success ) + { + return true; + } + } + catch( Exception e ) + { + dicts.SplitterRegexes.RemoveAt( i ); + + XuaLogger.AutoTranslator.Error( e, $"Failed while attempting to replace or match text of splitter regex '{regex.Original}'. Removing that regex from the cache." ); + } + } + } + + for( int i = _splitterRegexes.Count - 1; i > -1; i-- ) + { + var regex = _splitterRegexes[ i ]; + try + { + match = regex.CompiledRegex.Match( text ); + splitter = regex; + if( match.Success ) + { + return true; + } + } + catch( Exception e ) + { + _splitterRegexes.RemoveAt( i ); + + XuaLogger.AutoTranslator.Error( e, $"Failed while attempting to replace or match text of splitter regex '{regex.Original}'. Removing that regex from the cache." ); + } + } + + match = null; + splitter = null; + return false; + } + internal bool TryGetTranslation( UntranslatedText key, bool allowRegex, bool allowToken, int scope, out string value ) { bool result; @@ -1184,6 +1273,8 @@ public TranslationDictionaries() RegisteredRegexes = new HashSet(); TokenTranslations = new Dictionary(); ReverseTokenTranslations = new Dictionary(); + SplitterRegexes = new List(); + RegisteredSplitterRegexes = new HashSet(); } public Dictionary TokenTranslations { get; } @@ -1192,6 +1283,8 @@ public TranslationDictionaries() public Dictionary ReverseTranslations { get; } public List DefaultRegexes { get; } public HashSet RegisteredRegexes { get; } + public List SplitterRegexes { get; } + public HashSet RegisteredSplitterRegexes { get; } } } } diff --git a/src/XUnity.AutoTranslator.Plugin.Core/XUnity.AutoTranslator.Plugin.Core.csproj b/src/XUnity.AutoTranslator.Plugin.Core/XUnity.AutoTranslator.Plugin.Core.csproj index 55a6e015..19773ef6 100644 --- a/src/XUnity.AutoTranslator.Plugin.Core/XUnity.AutoTranslator.Plugin.Core.csproj +++ b/src/XUnity.AutoTranslator.Plugin.Core/XUnity.AutoTranslator.Plugin.Core.csproj @@ -2,7 +2,7 @@ net35 - 4.1.0 + 4.2.0 latest diff --git a/src/XUnity.AutoTranslator.Plugin.IPA/XUnity.AutoTranslator.Plugin.IPA.csproj b/src/XUnity.AutoTranslator.Plugin.IPA/XUnity.AutoTranslator.Plugin.IPA.csproj index 25ae415f..93f735f6 100644 --- a/src/XUnity.AutoTranslator.Plugin.IPA/XUnity.AutoTranslator.Plugin.IPA.csproj +++ b/src/XUnity.AutoTranslator.Plugin.IPA/XUnity.AutoTranslator.Plugin.IPA.csproj @@ -2,7 +2,7 @@ net35 - 4.1.0 + 4.2.0 diff --git a/src/XUnity.AutoTranslator.Plugin.UnityInjector/XUnity.AutoTranslator.Plugin.UnityInjector.csproj b/src/XUnity.AutoTranslator.Plugin.UnityInjector/XUnity.AutoTranslator.Plugin.UnityInjector.csproj index 7ae5412b..469ead97 100644 --- a/src/XUnity.AutoTranslator.Plugin.UnityInjector/XUnity.AutoTranslator.Plugin.UnityInjector.csproj +++ b/src/XUnity.AutoTranslator.Plugin.UnityInjector/XUnity.AutoTranslator.Plugin.UnityInjector.csproj @@ -2,7 +2,7 @@ net35 - 4.1.0 + 4.2.0 diff --git a/src/XUnity.AutoTranslator.Setup/XUnity.AutoTranslator.Setup.csproj b/src/XUnity.AutoTranslator.Setup/XUnity.AutoTranslator.Setup.csproj index 4dca46b6..d3fe5670 100644 --- a/src/XUnity.AutoTranslator.Setup/XUnity.AutoTranslator.Setup.csproj +++ b/src/XUnity.AutoTranslator.Setup/XUnity.AutoTranslator.Setup.csproj @@ -4,7 +4,7 @@ Exe net40 SetupReiPatcherAndAutoTranslator - 4.1.0 + 4.2.0 icon.ico