Add SetIetfLanguageTags option.

PlexCleaner.json

@@ -84,10 +84,6 @@
         "Format": "mpeg4",
         "Codec": "dx50"
       },
-      {
-        "Format": "mpeg4",
-        "Codec": "xvid"
-      },
       {
         "Format": "msmpeg4v3",
         "Codec": "div3"
@@ -164,6 +160,8 @@
     "RemoveTags": true,
     // Enable removing of EIA-608 Closed Captions embedded in video streams
     "RemoveClosedCaptions": true,
+    // Set IETF language tags when not present
+    "SetIetfLanguageTags": true,
     // Speedup media re-processing by saving media info and processed state in sidecar files
     "UseSidecarFiles": true,
     // Invalidate sidecar files when tool versions change

PlexCleaner.schema.json

@@ -90,6 +90,9 @@
         "RemoveClosedCaptions": {
           "type": "boolean"
         },
+        "SetIetfLanguageTags": {
+          "type": "boolean"
+        },
         "KeepExtensions": {
           "type": "array",
           "items": {
@@ -196,6 +199,7 @@
       "required": [
         "KeepOriginalLanguage",
         "RemoveClosedCaptions",
+        "SetIetfLanguageTags",
         "KeepExtensions",
         "ReMuxExtensions",
         "ReEncodeVideo",

PlexCleaner/PlexCleaner.csproj

@@ -53,4 +53,8 @@
     <PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="PlexCleanerTests" />
+  </ItemGroup>
+
 </Project>

PlexCleaner/ProcessFile.cs

@@ -687,6 +687,7 @@ public bool DeInterlace(ref bool modified)
         }
 
         Log.Logger.Information("Deinterlacing interlaced video : {FileName}", FileInfo.Name);
+        videoInfo.State = TrackInfo.StateType.DeInterlace;
         videoInfo.WriteLine("Interlaced");
 
         // TODO: HandBrake will convert closed captions and subtitle tracks to ASS format
@@ -1233,26 +1234,14 @@ private bool VerifyHdrProfile()
         <HDR_Format_Compatibility>Blu-ray / HDR10</HDR_Format_Compatibility>
         */
 
-        // Use first video track, if any
-        // TODO: Only supports one video track
-        if (MediaInfoInfo.Video.Count == 0)
+        // Find video tracks with HDR10 as format
+        var hdrVideoTracks = MediaInfoInfo.Video.FindAll(videoItem => 
+            Hdr10Format.Any(hdrFormat => 
+            videoItem.FormatHdr.Contains(hdrFormat, StringComparison.OrdinalIgnoreCase)));
+        hdrVideoTracks.ForEach(videoItem =>
         {
-            return true;
-        }
-        VideoInfo videoInfo = MediaInfoInfo.Video.First();
-
-        // Test for HDR
-        if (string.IsNullOrEmpty(videoInfo.FormatHdr))
-        {
-            return true;
-        }
-
-        // Look for HDR10 format
-        bool hdr10 = Hdr10Format.Any(format => videoInfo.FormatHdr.Contains(format, StringComparison.OrdinalIgnoreCase));
-        if (!hdr10)
-        {
-            Log.Logger.Warning("Video lacks HDR10 compatibility : {Hdr} : {FileName}", videoInfo.FormatHdr, FileInfo.Name);
-        }
+            Log.Logger.Warning("Video lacks HDR10 compatibility : {Hdr} : {FileName}", videoItem.FormatHdr, FileInfo.Name);
+        });
 
         // Ignore the error
         return true;
@@ -1509,7 +1498,9 @@ private bool Refresh(bool modified)
 
     public bool VerifyMediaInfo()
     {
-        // TODO: Mixing anything other than MvMerge to MkvMerge requires the track id's to be the same
+        // TODO: Mixing anything other than MvMerge to MkvMerge requires the track numbers to be the same
+        // Id's are unique to the tool, numbers come from the matroska header
+        // FfProbe does not report numbers, only id's
 
         // Make sure the track counts match
         if (FfProbeInfo.Audio.Count != MkvMergeInfo.Audio.Count ||
@@ -1578,13 +1569,17 @@ public bool GetBitrateInfo(out BitrateInfo bitrateInfo)
             return false;
         }
 
+        // Use the default track, else the first track
+        var videoInfo = FfProbeInfo.Video.Find(item => item.Flags.HasFlag(TrackInfo.FlagsType.Default));
+        videoInfo ??= FfProbeInfo.Video.First();
+        var audioInfo = FfProbeInfo.Audio.Find(item => item.Flags.HasFlag(TrackInfo.FlagsType.Default));
+        audioInfo ??= FfProbeInfo.Audio.First();
+
         // Compute bitrate from packets
-        // Use the first video and audio track for calculation
-        // TODO: Use default tracks not the first track
         bitrateInfo = new BitrateInfo();
         bitrateInfo.Calculate(packetList,
-            FfProbeInfo.Video.Count > 0 ? FfProbeInfo.Video.First().Id : -1,
-            FfProbeInfo.Audio.Count > 0 ? FfProbeInfo.Audio.First().Id : -1,
+            videoInfo == null ? -1 : videoInfo.Id,
+            audioInfo == null ? -1 : audioInfo.Id,
             Program.Config.VerifyOptions.MaximumBitrate / 8);
 
         return true;
@@ -1647,10 +1642,17 @@ public SelectMediaInfo FindNeedReEncode()
         // Keep all subtitles
         selectMediaInfo.Add(FfProbeInfo.Subtitle, false);
 
-        // TODO: If we are encoding audio, the video track may need to be reencode
+        // If we are encoding audio, the video track may need to be reencoded at the same time
         // [matroska @ 00000195b3585c80] Timestamps are unset in a packet for stream 0.
         // [matroska @ 00000195b3585c80] Can't write packet with unknown timestamp
         // av_interleaved_write_frame(): Invalid argument
+        if (selectMediaInfo.Selected.Audio.Count > 0 &&
+            selectMediaInfo.Selected.Video.Count == 0)
+        {
+            // If the video is not H264 or H265 (by experimentation), then tag the video to also be reencoded
+            var reEncodevideo = selectMediaInfo.Selected.Video.FindAll(item => !ReEncodeVideoOnAudioReEncode.Contains(item.Codec));
+            selectMediaInfo.Move(reEncodevideo, true);
+        }
 
         // Selected is ReEncode
         // NotSelected is Keep
@@ -1790,4 +1792,5 @@ static AudioInfo FindPreferredAudio(IEnumerable<TrackInfo> trackInfoList)
 
     private const int RefreshWaitTime = 5;
     private static readonly string[] Hdr10Format = { "SMPTE ST 2086", "SMPTE ST 2094" };
+    private static readonly string[] ReEncodeVideoOnAudioReEncode = { "h264", "hevc" };
 }

PlexCleaner/ProcessOptions.cs

@@ -212,26 +212,16 @@ protected void Upgrade(ProcessOptions2 processOptions2)
         // Upgrade from v1
         Upgrade(processOptions2 as ProcessOptions1);
 
-        // Default
-        KeepOriginalLanguage = true;
-        RemoveClosedCaptions = true;
-
         // Convert ISO 639-2 to RFC 5646 language tags
-        if (!string.IsNullOrEmpty(DefaultLanguage))
-        {
-            // Not found, default to English
-            DefaultLanguage = Language.GetIetfTag(DefaultLanguage, true) ?? Language.English;
-        }
+        DefaultLanguage = Language.GetIetfTag(DefaultLanguage, true) ?? Language.English;
         List<string> oldList = KeepLanguages.ToList();
         KeepLanguages.Clear();
-        oldList.ForEach(item => 
-        {
-            var ietfLanguage = Language.GetIetfTag(item, true);
-            if (ietfLanguage != null)
-            { 
-                KeepLanguages.Add(ietfLanguage); 
-            }
-        });
+        oldList.ForEach(item => KeepLanguages.Add(Language.GetIetfTag(item, true) ?? Language.English));
+
+        // Defaults
+        KeepOriginalLanguage = true;
+        RemoveClosedCaptions = true;
+        SetIetfLanguageTags = true;
     }
 
     [Required]
@@ -240,6 +230,9 @@ protected void Upgrade(ProcessOptions2 processOptions2)
     [Required]
     public bool RemoveClosedCaptions { get; set; }
 
+    [Required]
+    public bool SetIetfLanguageTags { get; set; }
+
     public void SetDefaults()
     {
         DeleteEmptyFolders = true;
@@ -249,6 +242,7 @@ public void SetDefaults()
         ReEncode = true;
         SetUnknownLanguage = true;
         RemoveUnwantedLanguageTracks = true;
+        SetIetfLanguageTags = true;
         RemoveTags = true;
         UseSidecarFiles = true;
         SidecarUpdateOnToolChange = false;
@@ -292,7 +286,6 @@ public void SetDefaults()
             new() { Format = "indeo5" },
             new() { Format = "h264", Profile = "Constrained Baseline@30" },
             new() { Format = "mpeg4", Codec = "dx50" },
-            new() { Format = "mpeg4", Codec = "xvid" },
             new() { Format = "msmpeg4v2", Codec = "mp42" },
             new() { Format = "msmpeg4v3", Codec = "div3" }
         };

PlexCleaner/Program.cs

@@ -537,8 +537,8 @@ public static CancellationToken CancelToken()
         return CancelSource.Token;
     }
 
-    public static CommandLineOptions Options { get; private set; }
-    public static ConfigFileJsonSchema Config { get; private set; }
+    public static CommandLineOptions Options { get; internal set; }
+    public static ConfigFileJsonSchema Config { get; internal set; }
 
     private static readonly CancellationTokenSource CancelSource = new();
 

PlexCleaner/TrackInfo.cs

@@ -97,9 +97,13 @@ internal TrackInfo(MkvToolJsonSchema.Track trackJson)
                 Log.Logger.Warning("MkvToolJsonSchema : IETF language not set, converting ISO639-3 to IETF : {Language} -> {IetfLanguage}", Language, lookupLanguage);
                 LanguageIetf = lookupLanguage;
 
-                // Set track error and recommend remux
-                HasErrors = true;
-                State = StateType.ReMux;
+                // Conditionally flag as track error to be corrected with remuxing
+                if (Program.Config.ProcessOptions.SetIetfLanguageTags)
+                {
+                    // Set track error and recommend remux
+                    HasErrors = true;
+                    State = StateType.ReMux;
+                }
             }
         }
 

PlexCleanerTests/ConfigFileTests.cs

@@ -3,15 +3,15 @@
 
 namespace PlexCleanerTests;
 
-public class ConfigFileTests
+public class ConfigFileTests : IClassFixture<PlexCleanerTests>
 {
     [Theory]
     [InlineData("PlexCleaner.v1.json")]
     [InlineData("PlexCleaner.v2.json")]
     [InlineData("PlexCleaner.v3.json")]
     public void Open_OldSchemas_Opens(string fileName)
     {
-        ConfigFileJsonSchema configFileJsonSchema = ConfigFileJsonSchema.FromFile(SampleFiles.GetSampleFilePath(fileName));
+        ConfigFileJsonSchema configFileJsonSchema = ConfigFileJsonSchema.FromFile(PlexCleanerTests.GetSampleFilePath(fileName));
         Assert.NotNull(configFileJsonSchema);
     }
 }

PlexCleanerTests/LanguageTests.cs

@@ -2,7 +2,7 @@
 
 namespace PlexCleanerTests;
 
-public class LanguageTests
+public class LanguageTests : IClassFixture<PlexCleanerTests>
 {
     [Theory]
     [InlineData("afr", "af")]

PlexCleanerTests/SampleFiles.cs → PlexCleanerTests/PlexCleanerTests.cs

@@ -1,11 +1,24 @@
-using System.Diagnostics;
+using System;
+using System.Diagnostics;
 using System.IO;
 using System.Reflection;
 
 namespace PlexCleanerTests;
 
-public static class SampleFiles
+public class PlexCleanerTests : IDisposable
 {
+    public PlexCleanerTests()
+    {
+        // Create defaults for Program Options and Config
+        PlexCleaner.Program.Options = new();
+        PlexCleaner.Program.Config = new();
+        PlexCleaner.Program.Config.SetDefaults();
+    }
+
+    public void Dispose()
+    {
+    }
+
     public static FileInfo GetSampleFileInfo(string fileName)
     {
         return new FileInfo(GetSampleFilePath(fileName));

PlexCleanerTests/SidecarFileTests.cs

@@ -12,9 +12,9 @@ public class SidecarFileTests
     [InlineData("Sidecar.v4.mkv")]
     public void Open_OldSchemas_Opens(string fileName)
     {
-        SidecarFile sidecarFile = new(SampleFiles.GetSampleFileInfo(fileName));
+        SidecarFile sidecarFile = new(PlexCleanerTests.GetSampleFileInfo(fileName));
         // Read the JSON file but do not verify the MKV media attributes
-        // TODO: Use a media file that matches the JSON
+        // TODO: Use media files that match the JSON, currently dummy files
         Assert.True(sidecarFile.Read(out _, false));
     }
 }

README.md

@@ -43,6 +43,7 @@ Docker images are published on [Docker Hub](https://hub.docker.com/u/ptr727/plex
       - On v3 schema upgrade old `ConvertOptions` settings will be upgrade to equivalent settings.
     - Added `ProcessOptions:KeepOriginalLanguage` to keep tracks marked as [original language](https://www.ietf.org/archive/id/draft-ietf-cellar-matroska-15.html#name-original-flag).
     - Added `ProcessOptions:RemoveClosedCaptions` to conditionally vs. always remove closed captions.
+    - Added `ProcessOptions.SetIetfLanguageTags` to conditionally remux files using MkvMerge to apply IETF language tags when not set.
     - Language tags now support [RFC 5646 / BCP 47](https://en.wikipedia.org/wiki/IETF_language_tag) format.
       - See the [Language Matching](#language-matching) section for matching details.
       - IETF language tags allows for greater flexibility in Matroska player [language matching](https://gitlab.com/mbunkus/mkvtoolnix/-/wikis/Languages-in-Matroska-and-MKVToolNix).
@@ -210,15 +211,8 @@ docker run \
 
 ### Linux
 
-- Automatic downloading of Linux 3rd party tools are not currently supported, consider using the [Docker](#docker) build instead.
-- Manually install the 3rd party tools by following steps similar to the [Docker](./Docker/Dockerfile) file `RUN` commands. (Note, steps are for Ubuntu, adjust as appropriate for your distribution.)
-  - Install prerequisites.
-  - Install [.NET Runtime](https://docs.microsoft.com/en-us/dotnet/core/install/linux).
-  - Install [MediaInfo](https://mediaarea.net/en/MediaInfo/Download/Ubuntu).
-  - Install [MKVToolNix](https://mkvtoolnix.download/downloads.html#ubuntu).
-  - Install [FFmpeg](https://launchpad.net/~savoury1/+archive/ubuntu/ffmpeg5).
-  - Install [HandBrake](https://launchpad.net/~savoury1/+archive/ubuntu/handbrake).
-- Keep the 3rd party tools updated by periodically running `sudo apt update && sudo apt upgrade -y`.
+- Automatic downloading of Linux 3rd party tools are not supported, consider using the [Docker](#docker) build instead.
+- Manually install the 3rd party tools, e.g. following steps similar to the [Docker](./Docker/Dockerfile) file commands.
 - Download [PlexCleaner](https://github.com/ptr727/PlexCleaner/releases/latest) and extract the pre-compiled binaries.
 - Or compile from [code](https://github.com/ptr727/PlexCleaner.git) using the [.NET SDK](https://docs.microsoft.com/en-us/dotnet/core/install/linux).
 - Create a default JSON settings file using the `defaultsettings` command:
@@ -319,10 +313,6 @@ Following is the [default JSON settings](./PlexCleaner.json) with usage comments
         "Format": "mpeg4",
         "Codec": "dx50"
       },
-      {
-        "Format": "mpeg4",
-        "Codec": "xvid"
-      },
       {
         "Format": "msmpeg4v3",
         "Codec": "div3"
@@ -399,6 +389,8 @@ Following is the [default JSON settings](./PlexCleaner.json) with usage comments
     "RemoveTags": true,
     // Enable removing of EIA-608 Closed Captions embedded in video streams
     "RemoveClosedCaptions": true,
+    // Set IETF language tags when not present
+    "SetIetfLanguageTags": true,
     // Speedup media re-processing by saving media info and processed state in sidecar files
     "UseSidecarFiles": true,
     // Invalidate sidecar files when tool versions change
@@ -528,8 +520,8 @@ The default `HandBrakeOptions:Audio` configuration is set to `copy --audio-fallb
 ## Language Matching
 
 Language tag matching now supports [IETF / RFC 5646 / BCP 47](https://en.wikipedia.org/wiki/IETF_language_tag) tag formats as implemented by [MkvMerge](https://gitlab.com/mbunkus/mkvtoolnix/-/wikis/Languages-in-Matroska-and-MKVToolNix).  
-During processing the absence of IETF language tags will treated as a track warning, and an IETF language will be assigned using from ISO639-3-2 tag.  
-On remediation MkvMerge will be used to remux the file using the `--normalize-language-ietf extlang` option, see the [MkvMerge docs](https://mkvtoolnix.download/doc/mkvpropedit.html#:~:text=%2D%2Dnormalize%2Dlanguage%2Dietf%20mode) for more details.
+During processing the absence of IETF language tags will treated as a track warning, and an IETF language will be assigned from the ISO639-3-2 tag.  
+If `ProcessOptions.SetIetfLanguageTags` is enabled MkvMerge will be used to remux the file using the `--normalize-language-ietf extlang` option, see the [MkvMerge docs](https://mkvtoolnix.download/doc/mkvpropedit.html#:~:text=%2D%2Dnormalize%2Dlanguage%2Dietf%20mode) for more details.
 
 Tags are in the form of `language-extlang-script-region-variant-extension-privateuse`, and matching happens left to right.  
 E.g. `pt` will match `pt` Portuguese, or `pt-BR` Brazilian Portuguese, or `pt-BR` Portugal Portuguese.