Skip to content

Commit

Permalink
Version 4.2.0
Browse files Browse the repository at this point in the history
 * CHANGE - Changed how splitting regexes are configured to supports scopes + more.
  • Loading branch information
randoman committed Sep 3, 2019
1 parent ccb4bc4 commit 20a1a5a
Show file tree
Hide file tree
Showing 20 changed files with 292 additions and 96 deletions.
5 changes: 4 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
### 4.1.0
### 4.2.0
* CHANGE - Changed how splitting regexes are configured to supports scopes + more.

### 4.1.0
* FEATURE - Added new concept that allows splitting texts to be translated with a regex before trying to translate them
* MISC - Improved NGUI text resizing behaviour

Expand Down
21 changes: 18 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,6 @@ FromLanguage=ja ;The original language of the game
Directory=Translation ;Directory to search for cached translation files. Can use placeholder: {GameExeName}
OutputFile=Translation\_AutoGeneratedTranslations.{lang}.txt ;File to insert generated translations into. Can use placeholders: {GameExeName}, {lang}
SubstitutionFile=Translation\_Substitutions.{lang}.txt ;File that contains substitution applied before translations. Can use placeholders: {GameExeName}, {lang}
PatternFile=Translation\_SplitterPatterns.{lang}.txt ;File that contains regexes that are applied to translatable texts before performing a translation to split them up into individual parts

[TextFrameworks]
EnableUGUI=True ;Enable or disable UGUI translation
Expand Down Expand Up @@ -486,7 +485,21 @@ When creating manual translations, use this file as sparingly as you would use r

*NOTE: If the text to be translated includes rich text, it cannot currently be parameterized.*

### Regex Splitting
### Regex Usage
Text translation files support regexes as well. Always remember to use regexes as sparing as possible and scope them as much as possible to avoid performance issues.

Regexes can be applied to translations in two different ways. The following two sections describes these two ways:

#### Standard Regex Translation
Standard regex translation are simply regexes that applied directly onto a translatable text, if no direct lookup can be found.

```
r:"^シンプルリング ([0-9]+)$"=Simple Ring $1
```

These are identified by the untranslated text starting with 'r:'.

#### Splitter Regex
Sometimes games likes to combine texts before displaying them on screen. This means that it can sometimes be hard to know what text to add to the translation file because it appears in a number of different ways.

This section explores a solution to this by applying a regex to split the text to be translated into individual pieces before trying to make lookups for the specified texts.
Expand All @@ -498,11 +511,13 @@ However, if we split the translation before trying to make lookups it will allow
Simply place the following regex in the `PatternFile` file:

```
([0-9]{2}) ([\S\s]+)=$1 $2
sr:"^([0-9]{2}) ([\S\s]+)$"=$1 $2
```

This will split up the text to be translated into two parts, translate them individually and put them back together.

These are identified by the untranslated text starting with 'sr:'.

### Translation Scoping
The following two options are available when it comes to scoping translations to only part of the game:

Expand Down
2 changes: 1 addition & 1 deletion src/XUnity.AutoTranslator.Patcher/Patcher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public override string Version
{
get
{
return "4.1.0";
return "4.2.0";
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

<PropertyGroup>
<TargetFramework>net35</TargetFramework>
<Version>4.1.0</Version>
<Version>4.2.0</Version>
</PropertyGroup>

<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

<PropertyGroup>
<TargetFramework>net35</TargetFramework>
<Version>4.1.0</Version>
<Version>4.2.0</Version>
</PropertyGroup>

<ItemGroup>
Expand Down
16 changes: 15 additions & 1 deletion src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public void Initialize()
SpamChecker = new SpamChecker( TranslationManager );

// WORKAROUND: Initialize text parsers with delegate indicating if text should be translated
UnityTextParsers.Initialize( ( text, scope ) => TextCache.IsTranslatable( text, true, scope ) && IsBelowMaxLength( text ) );
UnityTextParsers.Initialize( TextCache, ( text, scope ) => TextCache.IsTranslatable( text, true, scope ) && IsBelowMaxLength( text ) );

// resource redirectors
InitializeResourceRedirector();
Expand Down Expand Up @@ -1341,6 +1341,7 @@ private string TranslateByParserResult( TranslationEndpointManager endpoint, Par
{
Dictionary<string, string> translations = new Dictionary<string, string>();

var allowPartial = endpoint == null && result.AllowPartialTranslation;
var context = new ParserTranslationContext( null, endpoint, translationResult, result );
if( isGlobal )
{
Expand All @@ -1359,6 +1360,10 @@ private string TranslateByParserResult( TranslationEndpointManager endpoint, Par
{
translations.Add( variableName, textKey.Untemplate( partTranslation ) );
}
else if( allowPartial )
{
translations.Add( variableName, textKey.Untemplate( textKey.TemplatedOriginal_Text ) );
}
else if( allowStartJob )
{
// incomplete, must start job
Expand Down Expand Up @@ -1409,6 +1414,10 @@ private string TranslateByParserResult( TranslationEndpointManager endpoint, Par
{
translations.Add( variableName, textKey.Untemplate( partTranslation ) );
}
else if( allowPartial )
{
translations.Add( variableName, textKey.Untemplate( textKey.TemplatedOriginal_Text ) );
}
else if( allowStartJob )
{
// incomplete, must start job
Expand Down Expand Up @@ -1731,6 +1740,7 @@ private string TranslateOrQueueWebJobImmediateByParserResult( object ui, ParserR
Dictionary<string, string> translations = new Dictionary<string, string>();

// attempt to lookup ALL strings immediately; return result if possible; queue operations
var allowPartial = TranslationManager.CurrentEndpoint == null && result.AllowPartialTranslation;
var context = new ParserTranslationContext( ui, TranslationManager.CurrentEndpoint, null, result );
foreach( var kvp in result.Arguments )
{
Expand All @@ -1747,6 +1757,10 @@ private string TranslateOrQueueWebJobImmediateByParserResult( object ui, ParserR
{
translations.Add( variableName, textKey.Untemplate( partTranslation ) );
}
else if( allowPartial )
{
translations.Add( variableName, textKey.Untemplate( textKey.TemplatedOriginal_Text ) );
}
else if( allowStartJob )
{
// incomplete, must start job
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ internal static class Settings
public static string RedirectedResourcesPath;

public static Dictionary<string, string> Replacements = new Dictionary<string, string>();
public static List<RegexTranslationSplitter> Patterns = new List<RegexTranslationSplitter>();
//public static List<RegexTranslationSplitter> Patterns = new List<RegexTranslationSplitter>();

public static bool SimulateError = false;
public static bool SimulateDelayedError = false;
Expand All @@ -66,13 +66,13 @@ internal static class Settings
public static string FromLanguage;
public static string OutputFile;
public static string SubstitutionFile;
public static string PatternFile;
//public static string PatternFile;
public static string TranslationDirectory;
public static int MaxCharactersPerTranslation;
public static bool EnableConsole;
public static string AutoTranslationsFilePath;
public static string SubstitutionFilePath;
public static string PatternFilePath;
//public static string PatternFilePath;
public static bool EnableIMGUI;
public static bool EnableUGUI;
public static bool EnableNGUI;
Expand Down Expand Up @@ -148,7 +148,6 @@ public static void Configure()
TranslationDirectory = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "Directory", "Translation" );
OutputFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "OutputFile", @"Translation\_AutoGeneratedTranslations.{lang}.txt" );
SubstitutionFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "SubstitutionFile", @"Translation\_Substitutions.{lang}.txt" );
PatternFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "PatternFile", @"Translation\_SplitterPatterns.{lang}.txt" );

EnableIMGUI = PluginEnvironment.Current.Preferences.GetOrDefault( "TextFrameworks", "EnableIMGUI", false );
EnableUGUI = PluginEnvironment.Current.Preferences.GetOrDefault( "TextFrameworks", "EnableUGUI", true );
Expand Down Expand Up @@ -236,7 +235,6 @@ public static void Configure()

AutoTranslationsFilePath = Path.Combine( PluginEnvironment.Current.TranslationPath, OutputFile.Replace( "{lang}", Language ) ).Replace( "/", "\\" ).Parameterize();
SubstitutionFilePath = Path.Combine( PluginEnvironment.Current.TranslationPath, SubstitutionFile.Replace( "{lang}", Language ) ).Replace( "/", "\\" ).Parameterize();
PatternFilePath = Path.Combine( PluginEnvironment.Current.TranslationPath, PatternFile.Replace( "{lang}", Language ) ).Replace( "/", "\\" ).Parameterize();

FromLanguageUsesWhitespaceBetweenWords = LanguageHelper.RequiresWhitespaceUponLineMerging( FromLanguage );
ToLanguageUsesWhitespaceBetweenWords = LanguageHelper.RequiresWhitespaceUponLineMerging( Language );
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ public static class PluginData
/// <summary>
/// Gets the version of the plugin.
/// </summary>
public const string Version = "4.1.0";
public const string Version = "4.2.0";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public ParserResult Parse( string input, int scope )
if( !containsTranslatable ) return null;

if( !input.EndsWith( "\r\n" ) && !input.EndsWith( "\n" ) ) template.Remove( template.Length - 1, 1 );
return new ParserResult( input, template.ToString(), false, false, true, args );
return new ParserResult( input, template.ToString(), false, false, false, true, args );
}
}
}
38 changes: 28 additions & 10 deletions src/XUnity.AutoTranslator.Plugin.Core/Parsing/ParserResult.cs
Original file line number Diff line number Diff line change
@@ -1,38 +1,56 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace XUnity.AutoTranslator.Plugin.Core.Parsing
{
internal class ParserResult
{
public ParserResult( string originalText, string template, bool cacheCombinedResult, bool persistCombinedResult, bool persistTokenResult, Dictionary<string, string> args )
public ParserResult( string originalText, string template, bool allowPartialTranslation, bool cacheCombinedResult, bool persistCombinedResult, bool persistTokenResult, Dictionary<string, string> args )
{
OriginalText = originalText;
Template = template;
AllowPartialTranslation = allowPartialTranslation;
CacheCombinedResult = cacheCombinedResult;
PersistCombinedResult = persistCombinedResult;
PersistTokenResult = persistTokenResult;
Arguments = args;
}

public string OriginalText { get; private set; }
public string OriginalText { get; }

public string Template { get; private set; }
public Dictionary<string, string> Arguments { get; private set; }
public string Template { get; }
public Dictionary<string, string> Arguments { get; }

public bool AllowPartialTranslation { get; }

public bool CacheCombinedResult { get; }

public bool PersistCombinedResult { get; private set; }
public bool PersistCombinedResult { get; }

public bool PersistTokenResult { get; private set; }
public bool PersistTokenResult { get; }

public string Untemplate( Dictionary<string, string> arguments )
{
string result = Template;
foreach( var kvp in arguments )
// This is really not a nice fix...
if( arguments.Count > 9 )
{
var result = new StringBuilder( Template );
foreach( var kvp in arguments.OrderByDescending( x => x.Key.Length ) )
{
result = result.Replace( kvp.Key, kvp.Value );
}
return result.ToString();
}
else
{
result = result.Replace( kvp.Key, kvp.Value );
var result = new StringBuilder( Template );
foreach( var kvp in arguments )
{
result = result.Replace( kvp.Key, kvp.Value );
}
return result.ToString();
}
return result;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,35 +7,35 @@ namespace XUnity.AutoTranslator.Plugin.Core.Parsing
{
internal class RegexSplittingTextParser : ITextParser
{
private readonly TextTranslationCache _cache;

public RegexSplittingTextParser( TextTranslationCache cache )
{
_cache = cache;
}

public bool CanApply( object ui )
{
return !ui.IsSpammingComponent();
}

public ParserResult Parse( string input, int scope )
{
var patterns = Settings.Patterns;
var length = patterns.Count;
for( int i = 0; i < length; i++ )
if( _cache.TryGetTranslationSplitter( input, scope, out var match, out var splitter ) )
{
var regex = patterns[ i ];
var m = regex.CompiledRegex.Match( input );
if( m.Success )
{
var args = new Dictionary<string, string>();

var groups = m.Groups;
var len = groups.Count;
for( int j = 1; j < len; j++ )
{
var group = groups[ j ];
var groupName = "$" + j;
var value = group.Value;
args.Add( groupName, value );
}
var args = new Dictionary<string, string>();

return new ParserResult( input, regex.Translation, true, Settings.CacheRegexPatternResults, true, args );
var groups = match.Groups;
var len = groups.Count;
for( int j = 1; j < len; j++ )
{
var group = groups[ j ];
var groupName = "$" + j;
var value = group.Value;
args.Add( groupName, value );
}

return new ParserResult( input, splitter.Translation, true, true, Settings.CacheRegexPatternResults, true, args );
}

return null;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,73 @@
using System.Text.RegularExpressions;
using XUnity.Common.Logging;

namespace XUnity.AutoTranslator.Plugin.Core.Parsing
{
internal class RegexTranslationSplitter
{
public RegexTranslationSplitter( string regex, string translation )
public RegexTranslationSplitter( string key, string value )
{
CompiledRegex = new Regex( regex );
Translation = translation;
Original = regex;
Key = key;
Value = value;

// remove sr:
if( key.StartsWith( "sr:" ) )
{
key = key.Substring( 3, key.Length - 3 );
}

var startIdx = key.IndexOf( '"' );
if( startIdx == -1 )
{
// take entire string
}
else
{
startIdx++;
var endIdx = key.LastIndexOf( '"' );
if( endIdx != startIdx )
{
key = key.Substring( startIdx, endIdx - startIdx );
}
}

// remove sr:
if( value.StartsWith( "sr:" ) )
{
value = value.Substring( 3, value.Length - 3 );
}

startIdx = value.IndexOf( '"' );
if( startIdx == -1 )
{
// take entire string
}
else
{
startIdx++;
var endIdx = value.LastIndexOf( '"' );
if( endIdx != startIdx )
{
value = value.Substring( startIdx, endIdx - startIdx );
}
}

if( !key.StartsWith( "^" ) ) key = "^" + key;
if( !key.EndsWith( "$" ) ) key = key + "$";

CompiledRegex = new Regex( key );
Original = key;
Translation = value;
}

public Regex CompiledRegex { get; set; }

public string Original { get; set; }

public string Translation { get; set; }

public string Key { get; set; }

public string Value { get; set; }
}
}
Loading

0 comments on commit 20a1a5a

Please sign in to comment.