Skip to content

Commit

Permalink
media file value migrations fix + extended logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@tkrch
tkrch committed Nov 27, 2024
1 parent 695eb6d commit dd12b88
Show file tree
Hide file tree
Showing 9 changed files with 257 additions and 70 deletions.
29 changes: 25 additions & 4 deletions KVA/Migration.Tool.Source/Handlers/MigratePagesCommandHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,8 @@ public async Task<CommandResult> Handle(MigratePagesCommand request, Cancellatio
? (Guid?)null
: spoiledGuidContext.EnsureNodeGuid(ksNodeParent);

DataClassInfo targetClass = null!;
var classMapping = classMappingProvider.GetMapping(ksNodeClass.ClassName);
targetClass = classMapping != null
var targetClass = classMapping != null
? DataClassInfoProvider.ProviderObject.Get(classMapping.TargetClassName)
: DataClassInfoProvider.ProviderObject.Get(ksNodeClass.ClassGUID);

Expand All @@ -210,6 +209,7 @@ public async Task<CommandResult> Handle(MigratePagesCommand request, Cancellatio
var commonDataInfos = new List<ContentItemCommonDataInfo>();
foreach (var umtModel in results)
{
logger.LogTrace("UMT-M: {UMT}", umtModel.PrintMe());
switch (await importer.ImportAsync(umtModel))
{
case { Success: false } result:
Expand All @@ -233,9 +233,23 @@ public async Task<CommandResult> Handle(MigratePagesCommand request, Cancellatio
webPageItemInfo = wp;
break;
}

case {} importResult:
{
logger.LogTrace("Unexpected state {UMT} Result: {Result}", umtModel.PrintMe(), new
{
importResult.Success,
importResult.PrimaryKey,
importResult.Imported,
importResult.Exception,
validationResults = JsonConvert.SerializeObject(importResult.ModelValidationResults ?? [])
});
break;
}
default:
{
logger.LogTrace("Unexpected state {UMT}", umtModel.PrintMe());
break;
}
}
}

Expand Down Expand Up @@ -270,7 +284,14 @@ await MigratePageUrlPaths(ksSite.SiteGUID,
}
else
{
logger.LogTrace("No webpage item produced for '{NodeAliasPath}'", ksNode.NodeAliasPath);
if (nodeParentGuid is {} npg && ContentItemInfo.Provider.Get(npg) is {ContentItemIsReusable:true})
{
logger.LogTrace("No webpage item produced for '{NodeAliasPath}' - parent is reusable, possibly converted with mapping?", ksNode.NodeAliasPath);
}
else
{
logger.LogTrace("No webpage item produced for '{NodeAliasPath}'", ksNode.NodeAliasPath);
}
}
}

Expand Down
11 changes: 8 additions & 3 deletions KVA/Migration.Tool.Source/Mappers/ContentItemMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,22 @@ protected override IEnumerable<IUmtModel> MapInternal(CmsTreeMapperSource source
var sourceNodeClass = modelFacade.SelectById<ICmsClass>(cmsTree.NodeClassID) ?? throw new InvalidOperationException($"Fatal: node class is missing, class id '{cmsTree.NodeClassID}'");
var mapping = classMappingProvider.GetMapping(sourceNodeClass.ClassName);
var targetClassGuid = sourceNodeClass.ClassGUID;
DataClassInfo targetClassInfo = null;
var targetClassInfo = DataClassInfoProvider.ProviderObject.Get(sourceNodeClass.ClassName);
if (mapping != null)
{
targetClassInfo = DataClassInfoProvider.ProviderObject.Get(mapping.TargetClassName) ?? throw new InvalidOperationException($"Unable to find target class '{mapping.TargetClassName}'");
targetClassGuid = targetClassInfo.ClassGUID;
}

bool migratedAsContentFolder = sourceNodeClass.ClassName.Equals("cms.folder", StringComparison.InvariantCultureIgnoreCase) && !configuration.UseDeprecatedFolderPageType.GetValueOrDefault(false);

var contentItemGuid = spoiledGuidContext.EnsureNodeGuid(cmsTree.NodeGUID, cmsTree.NodeSiteID, cmsTree.NodeID);
bool isMappedTypeReusable = (targetClassInfo?.ClassContentTypeType is ClassContentTypeType.REUSABLE) || configuration.ClassNamesConvertToContentHub.Contains(sourceNodeClass.ClassName);
if (isMappedTypeReusable)
{
logger.LogTrace("Target is reusable {Info}", new { cmsTree.NodeAliasPath, targetClassInfo?.ClassName });
}

yield return new ContentItemModel
{
ContentItemGUID = contentItemGuid,
Expand Down Expand Up @@ -670,7 +675,7 @@ IClassMapping mapping
!configuration.MigrateMediaToMediaLibrary)
{
var mediaLinkService = mediaLinkServiceFactory.Create();
var htmlProcessor = new HtmlProcessor(html, mediaLinkService);
var htmlProcessor = new HtmlProcessor(html, mediaLinkService, logger);

target[targetColumnName] = await htmlProcessor.ProcessHtml(site.SiteID, async (result, original) =>
{
Expand Down
108 changes: 58 additions & 50 deletions KVA/Migration.Tool.Source/Services/MediaFileMigrator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class MediaFileMigrator(
PrimaryKeyMappingContext primaryKeyMappingContext,
EntityIdentityFacade entityIdentityFacade,
IProtocol protocol
) : IMediaFileMigrator
) : IMediaFileMigrator
{
public async Task<CommandResult> Handle(MigrateMediaLibrariesCommand request, CancellationToken cancellationToken)
{
Expand Down Expand Up @@ -84,6 +84,7 @@ SELECT LibraryName FROM Media_Library GROUP BY LibraryName HAVING COUNT(*) > 1
}
catch (Exception ex)
{
logger.LogTrace($"Failed {ksMediaLibrary}");
protocol.Append(HandbookReferences
.ErrorCreatingTargetInstance<MediaLibraryInfo>(ex)
.NeedsManualAction()
Expand Down Expand Up @@ -136,70 +137,77 @@ private void RequireMigratedMediaFiles(List<(IMediaLibrary sourceLibrary, ICmsSi

foreach (var ksMediaFile in ksMediaFiles)
{
protocol.FetchedSource(ksMediaFile);

bool found = false;
IUploadedFile? uploadedFile = null;
string? fullMediaPath = "<uninitialized>";
if (loadMediaFileData)
try
{
(found, uploadedFile, fullMediaPath) = LoadMediaFileBinary(sourceMediaLibraryPath, ksMediaFile.FilePath, ksMediaFile.FileMimeType);
if (!found)
protocol.FetchedSource(ksMediaFile);

bool found = false;
IUploadedFile? uploadedFile = null;
string? fullMediaPath = "<uninitialized>";
if (loadMediaFileData)
{
// report missing file (currently reported in mapper)
(found, uploadedFile, fullMediaPath) = LoadMediaFileBinary(sourceMediaLibraryPath, ksMediaFile.FilePath, ksMediaFile.FileMimeType);
if (!found)
{
// report missing file (currently reported in mapper)
}
}
}

string? librarySubfolder = Path.GetDirectoryName(ksMediaFile.FilePath);
string? librarySubfolder = Path.GetDirectoryName(ksMediaFile.FilePath);

(bool isFixed, var safeMediaFileGuid) = entityIdentityFacade.Translate(ksMediaFile);
if (isFixed)
{
logger.LogWarning("MediaFile {File} has non-unique guid, new guid {Guid} was required", new { ksMediaFile.FileGUID, ksMediaFile.FileName, ksMediaFile.FileSiteID }, safeMediaFileGuid);
}

var kxoMediaFile = mediaFileFacade.GetMediaFile(safeMediaFileGuid);
(bool isFixed, var safeMediaFileGuid) = entityIdentityFacade.Translate(ksMediaFile);
if (isFixed)
{
logger.LogWarning("MediaFile {File} has non-unique guid, new guid {Guid} was required", new { ksMediaFile.FileGUID, ksMediaFile.FileName, ksMediaFile.FileSiteID }, safeMediaFileGuid);
}

protocol.FetchedTarget(kxoMediaFile);
var kxoMediaFile = mediaFileFacade.GetMediaFile(safeMediaFileGuid);

var source = new MediaFileInfoMapperSource(fullMediaPath, ksMediaFile, targetMediaLibrary.LibraryID, found ? uploadedFile : null,
librarySubfolder, toolConfiguration.MigrateOnlyMediaFileInfo.GetValueOrDefault(false), safeMediaFileGuid);
var mapped = mediaFileInfoMapper.Map(source, kxoMediaFile);
protocol.MappedTarget(mapped);
protocol.FetchedTarget(kxoMediaFile);

if (mapped is { Success: true } result)
{
(var mf, bool newInstance) = result;
ArgumentNullException.ThrowIfNull(mf, nameof(mf));
var source = new MediaFileInfoMapperSource(fullMediaPath, ksMediaFile, targetMediaLibrary.LibraryID, found ? uploadedFile : null,
librarySubfolder, toolConfiguration.MigrateOnlyMediaFileInfo.GetValueOrDefault(false), safeMediaFileGuid);
var mapped = mediaFileInfoMapper.Map(source, kxoMediaFile);
protocol.MappedTarget(mapped);

try
if (mapped is { Success: true } result)
{
if (newInstance)
(var mf, bool newInstance) = result;
ArgumentNullException.ThrowIfNull(mf, nameof(mf));

try
{
mediaFileFacade.EnsureMediaFilePathExistsInLibrary(mf, targetMediaLibrary.LibraryID);
}
if (newInstance)
{
mediaFileFacade.EnsureMediaFilePathExistsInLibrary(mf, targetMediaLibrary.LibraryID);
}

mediaFileFacade.SetMediaFile(mf, newInstance);
mediaFileFacade.SetMediaFile(mf, newInstance);

protocol.Success(ksMediaFile, mf, mapped);
logger.LogEntitySetAction(newInstance, mf);
}
catch (Exception ex)
{
protocol.Append(HandbookReferences
.ErrorCreatingTargetInstance<MediaLibraryInfo>(ex)
.NeedsManualAction()
.WithIdentityPrint(mf)
protocol.Success(ksMediaFile, mf, mapped);
logger.LogEntitySetAction(newInstance, mf);
}
catch (Exception ex)
{
protocol.Append(HandbookReferences
.ErrorCreatingTargetInstance<MediaLibraryInfo>(ex)
.NeedsManualAction()
.WithIdentityPrint(mf)
);
logger.LogEntitySetError(ex, newInstance, mf);
continue;
}

primaryKeyMappingContext.SetMapping<MediaFileInfo>(
r => r.FileID,
ksMediaFile.FileID,
mf.FileID
);
logger.LogEntitySetError(ex, newInstance, mf);
continue;
}

primaryKeyMappingContext.SetMapping<MediaFileInfo>(
r => r.FileID,
ksMediaFile.FileID,
mf.FileID
);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed {MediaFile}", ksMediaFile);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion Migration.Tool.Common/Commands.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public record MigratePageTypesCommand : IRequest<CommandResult>, ICommand

public record MigratePagesCommand : IRequest<CommandResult>, ICommand
{
public static readonly int Rank = 1 + MigrateSitesCommand.Rank + MigrateUsersCommand.Rank + MigratePageTypesCommand.Rank;
public static readonly int Rank = 1 + MigrateSitesCommand.Rank + MigrateUsersCommand.Rank + MigratePageTypesCommand.Rank + MigrateMediaLibrariesCommand.Rank;

public static string Moniker => "pages";
public static string MonikerFriendly => "Pages";
Expand Down
20 changes: 18 additions & 2 deletions Migration.Tool.Common/Helpers/HtmlProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;

namespace Migration.Tool.Common.Helpers;

public class HtmlProcessor
{
private readonly MediaLinkService mediaLinkService;
private readonly ILogger? logger;
private readonly HtmlDocument document;
private readonly string html;

public HtmlProcessor(string html, MediaLinkService mediaLinkService)
public HtmlProcessor(string html, MediaLinkService mediaLinkService, ILogger? logger)
{
this.mediaLinkService = mediaLinkService;
this.logger = logger;

var doc = new HtmlDocument();
doc.LoadHtml(html);
Expand All @@ -25,7 +28,20 @@ public IEnumerable<MatchMediaLinkResult> GetImages(int currentSiteId)
{
if (imgNode?.Attributes["src"].Value is { } src)
{
yield return mediaLinkService.MatchMediaLink(src, currentSiteId);
MatchMediaLinkResult? mediaLinkMatch = null;
try
{
mediaLinkMatch = mediaLinkService.MatchMediaLink(src, currentSiteId);
}
catch (Exception ex)
{
logger?.LogError(ex, "Failed to match media link for value '{SourceValue}'", src);
}

if (mediaLinkMatch != null)
{
yield return mediaLinkMatch;
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion Migration.Tool.Common/Services/CommandParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public List<ICommand> Parse(Queue<string> args, ref bool bypassDependencyCheck,
}
}

return commands;
return commands.OrderBy(c => c.Rank).ToList();
}

private void PrintCommandDescriptions()
Expand Down
50 changes: 42 additions & 8 deletions Migration.Tool.Extensions/DefaultMigrations/AssetMigration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,20 @@ public async Task<FieldMigrationResult> MigrateValue(object? sourceValue, FieldM

List<object> mfis = [];
bool hasMigratedAsset = false;
if (sourceValue is string link &&
mediaLinkServiceFactory.Create().MatchMediaLink(link, cmsSite.SiteID) is (true, var mediaLinkKind, var mediaKind, var path, var mediaGuid, _, _) result)
MatchMediaLinkResult? mediaLinkMatch = null;
try
{
if (sourceValue is string link)
{
mediaLinkMatch = mediaLinkServiceFactory.Create().MatchMediaLink(link, cmsSite.SiteID);
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to match media link for value '{SourceValue}'", sourceValue);
}

if (mediaLinkMatch is (true, var mediaLinkKind, var mediaKind, var path, var mediaGuid, _, _) result)
{
if (mediaLinkKind == MediaLinkKind.Path)
{
Expand Down Expand Up @@ -107,10 +119,21 @@ public async Task<FieldMigrationResult> MigrateValue(object? sourceValue, FieldM
{
if (configuration.MigrateMediaToMediaLibrary)
{
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mf && mediaFileFacade.GetMediaFile(mf.Identity) is { } x)
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mfi)
{
mfis = [new AssetRelatedItem { Identifier = x.FileGUID, Dimensions = new AssetDimensions { Height = x.FileImageHeight, Width = x.FileImageWidth }, Name = x.FileName, Size = x.FileSize }];
hasMigratedAsset = true;
if (mediaFileFacade.GetMediaFile(mfi.Identity) is { } mf)
{
mfis = [new AssetRelatedItem { Identifier = mf.FileGUID, Dimensions = new AssetDimensions { Height = mf.FileImageHeight, Width = mf.FileImageWidth }, Name = mf.FileName, Size = mf.FileSize }];
hasMigratedAsset = true;
}
else
{
logger.LogError("Media file {MediaFileIdentity} not found", mfi);
}
}
else
{
logger.LogError("Unable to determine identity for {MediaFile}", sourceMediaFile);
}
}
else
Expand Down Expand Up @@ -165,10 +188,21 @@ public async Task<FieldMigrationResult> MigrateValue(object? sourceValue, FieldM
{
if (configuration.MigrateMediaToMediaLibrary)
{
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mf && mediaFileFacade.GetMediaFile(mf.Identity) is { } x)
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mfi)
{
mfis = [new AssetRelatedItem { Identifier = x.FileGUID, Dimensions = new AssetDimensions { Height = x.FileImageHeight, Width = x.FileImageWidth }, Name = x.FileName, Size = x.FileSize }];
hasMigratedAsset = true;
if (mediaFileFacade.GetMediaFile(mfi.Identity) is { } mf)
{
mfis = [new AssetRelatedItem { Identifier = mf.FileGUID, Dimensions = new AssetDimensions { Height = mf.FileImageHeight, Width = mf.FileImageWidth }, Name = mf.FileName, Size = mf.FileSize }];
hasMigratedAsset = true;
}
else
{
logger.LogError("Media file {MediaFileIdentity} not found", mfi);
}
}
else
{
logger.LogError("Unable to determine identity for {MediaFile}", sourceMediaFile);
}
}
else
Expand Down
2 changes: 1 addition & 1 deletion Migration.Tool.Tests/HtmlProcessorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public void TestHtmlFragment1()
}
);

var processor = new HtmlProcessor(HtmlFragmentSample1, mediaLinkService);
var processor = new HtmlProcessor(HtmlFragmentSample1, mediaLinkService, null);

var actual = processor.GetImages(2).ToArray();
Assert.Collection(actual,
Expand Down
Loading

0 comments on commit dd12b88

Please sign in to comment.