MetalImageView render issue

[binarysai]

Great job
but i found something wired

the texture rendered by MetalImageView does not look good

If i change the MetalImageView to MTKView, and implement the drawRect
It works perfectly

[jzting]

Does this happen for all filters or just some of them? I'm also noticing something weird with MIJFAVoronoiFilter.

[binarysai]

Does this happen for all filters or just some of them? I'm also noticing something weird with MIJFAVoronoiFilter.

The filters work as expected, i guess it's the MetalImageView problem.
Change the MetalImageView subclass to MTKView, and render texture in drawRect method
It works perfectly

[jzting]

Are you able to create a Gist of what your MetalImageView.m looks like after your changes? Thanks!

[binarysai]

Are you able to create a Gist of what your MetalImageView.m looks like after your changes? Thanks!

#import "MetalImageView.h"
#import "MetalImageContext.h"
#import "MetalImageTypes.h"
#import "MetalImageFunction.h"
#import "MetalDevice.h"
#import "UIImage+Texture.h"

@import AVFoundation;

@interface MetalImageView () {
MTLFloat4 imageScalingVertics[4];
MTLUInt2 inputImageSize;
}

@Property (nonatomic, strong) id verticsBuffer;
@Property (nonatomic, strong) id coordBuffer;
@Property (nonatomic, strong) id pipelineState;
@Property (nonatomic, strong) id depthStencilState;
@Property (nonatomic, strong) MTLRenderPassDescriptor *renderPassDescriptor;

• (void)buildUp;
• (void)tearDown;

@EnD

@implementation MetalImageView

//+(Class)layerClass
//{
// return [CAMetalLayer class];
//}

• (instancetype)init {
  if (self = [super init]) {
  [self buildUp];
  }
  return self;
  }

• (instancetype)initWithFrame:(CGRect)frame device:(nullable id)device{
  if (self = [super initWithFrame:frame device:device]) {
  [self buildUp];
  }
  return self;
  }

• (instancetype)initWithCoder:(NSCoder *)aDecoder {
  if (self = [super initWithCoder:aDecoder]) {
  [self buildUp];
  }

  return self;
  }

• (void)dealloc {
  [self tearDown];
  }

• (void)buildUp {
  self.opaque = YES;
  self.backgroundColor = [UIColor clearColor];
  self.bgClearColor = MTLClearColorMake(0.0f, 0.0f, 0.0f, 1.0f);

  id device = [MetalDevice sharedMTLDevice];
  self.device = device;
  self.paused = YES;
  self.enableSetNeedsDisplay = NO;
  // CAMetalLayer *metalLayer = (CAMetalLayer )self.layer;
  // metalLayer = (CAMetalLayer) self.layer;
  // metalLayer.device = device;
  // metalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm;
  // metalLayer.framebufferOnly = YES;

  inputRotationMode = kMetalImageNoRotation;

  _fillMode = kMetalImageFillModePreserveAspectRatio;
  [self fillTextureVerticsWithWidthScaling:1.0f heightScaling:1.0f];

  [self updateTextureVertics];
  [self updateTextureCoordinates];
  [self prepareRenderPipeline];
  [self prepareRenderDepthStencilState];
  [self prepareRenderPassDescriptor];
  }

• (void)tearDown {

}

• (void)updateTextureVertics {
  id device = [MetalDevice sharedMTLDevice];
  _verticsBuffer = [device newBufferWithBytes:imageScalingVertics
  length:MetalImageDefaultRenderVetexCount*sizeof(MTLFloat4)
  options:MTLResourceOptionCPUCacheModeDefault];
  if(!_verticsBuffer) {
  NSLog(@">> ERROR: Failed creating a vertex buffer for a quad!");
  return ;
  }
  _verticsBuffer.label = @"quad vertices";
  }

• (void)updateTextureCoordinates {
  id device = [MetalDevice sharedMTLDevice];
  const MTLFloat2 textureCoordinates = TextureCoordinatesForRotation(inputRotationMode);
  _coordBuffer = [device newBufferWithBytes:textureCoordinates
  length:MetalImageDefaultRenderVetexCountsizeof(MTLFloat2)
  options:MTLResourceOptionCPUCacheModeDefault];
  if (!_coordBuffer) {
  NSLog(@">> ERROR: Failed creating a 2d texture coordinate buffer!");
  return;
  }
  _coordBuffer.label = @"quad texcoords";
  }

• (BOOL)prepareRenderPipeline
  {
  id device = [MetalDevice sharedMTLDevice];
  id renderLibrary = [MetalDevice MetalImageLibrary];
  if (!renderLibrary && device) {
  NSError *liberr = nil;
  renderLibrary = [device newLibraryWithFile:@"CommonStruct.metallib" error:&liberr];
  NSAssert(liberr == nil, @"Create library failed...%@", liberr);
  }

  id vertexProgram = [renderLibrary newFunctionWithName:@"vertex_common"];
  id fragmentProgram = [renderLibrary newFunctionWithName:@"fragment_common"];
  if (!vertexProgram || !fragmentProgram) {
  return NO;
  }

  MTLRenderPipelineDescriptor *pQuadPipelineStateDescriptor = [MTLRenderPipelineDescriptor new];
  pQuadPipelineStateDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
  pQuadPipelineStateDescriptor.stencilAttachmentPixelFormat = MTLPixelFormatInvalid;
  pQuadPipelineStateDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
  pQuadPipelineStateDescriptor.sampleCount = 1;
  pQuadPipelineStateDescriptor.vertexFunction = vertexProgram;
  pQuadPipelineStateDescriptor.fragmentFunction = fragmentProgram;

  NSError *pError = nil;
  _pipelineState = [device newRenderPipelineStateWithDescriptor:pQuadPipelineStateDescriptor error:&pError];
  if(pError) {
  NSLog(@">> ERROR: Failed acquiring pipeline state descriptor: %@", pError);
  }

  return _pipelineState != nil;
  }

• (BOOL)prepareRenderDepthStencilState
  {
  id device = [MetalDevice sharedMTLDevice];
  MTLDepthStencilDescriptor *pDepthStateDesc = [MTLDepthStencilDescriptor new];

  if(!pDepthStateDesc){
  NSLog(@">> ERROR: Failed creating a depth stencil descriptor!");
  }

  pDepthStateDesc.depthCompareFunction = MTLCompareFunctionAlways;
  pDepthStateDesc.depthWriteEnabled = NO;
  _depthStencilState = [device newDepthStencilStateWithDescriptor:pDepthStateDesc];

  return _depthStencilState != nil;
  }

• (BOOL)prepareRenderPassDescriptor {

  _renderPassDescriptor = [[MTLRenderPassDescriptor alloc] init];
  MTLRenderPassColorAttachmentDescriptor *colorAttachment = _renderPassDescriptor.colorAttachments[0];
  colorAttachment.loadAction = MTLLoadActionClear;
  colorAttachment.clearColor = _bgClearColor;
  colorAttachment.storeAction = MTLStoreActionStore;

  return _renderPassDescriptor != nil;

}

#pragma mark - Fill mode

• (void)setFillMode:(MetalImageFillModeType)newValue;
  {
  _fillMode = newValue;
  [self updateViewGeometry];
  }

• (void)updateViewGeometry
  {
  __block CGRect frame;
  __block CGSize currentViewSize;
  runMetalOnMainQueueWithoutDeadlocking(^{
  frame = self.bounds;
  currentViewSize = self.bounds.size;
  });
  runMetalSynchronouslyOnVideoProcessingQueue(^{
  CGFloat heightScaling, widthScaling;
  CGSize imageSize = CGSizeMake(inputImageSize.x, inputImageSize.y);
  CGRect insetRect = AVMakeRectWithAspectRatioInsideRect(imageSize, frame);

    switch(_fillMode)
    {
        case kMetalImageFillModeStretch:
        {
            widthScaling = 1.0;
            heightScaling = 1.0;
        }
            break;
            
        case kMetalImageFillModePreserveAspectRatio:
        {
            widthScaling = insetRect.size.width / currentViewSize.width;
            heightScaling = insetRect.size.height / currentViewSize.height;
        }
            break;
            
        case kMetalImageFillModePreserveAspectRatioAndFill:
        {
            widthScaling = currentViewSize.height / insetRect.size.height;
            heightScaling = currentViewSize.width / insetRect.size.width;
        }
            break;
    }
    
    [self fillTextureVerticsWithWidthScaling:widthScaling heightScaling:heightScaling];
  

  });
  }

• (void)fillTextureVerticsWithWidthScaling:(CGFloat)widthScaling heightScaling:(CGFloat)heightScaling {
  imageScalingVertics[0] = MTLFloat4Make(-widthScaling, -heightScaling, 0.0f, 1.0f);
  imageScalingVertics[1] = MTLFloat4Make( widthScaling, -heightScaling, 0.0f, 1.0f);
  imageScalingVertics[2] = MTLFloat4Make(-widthScaling, heightScaling, 0.0f, 1.0f);
  imageScalingVertics[3] = MTLFloat4Make( widthScaling, heightScaling, 0.0f, 1.0f);
  }

• (void)updateInputImageSize {
  if (firstInputTexture) {

    MTLUInt2 newSize = [firstInputTexture size];
    if (MetalImageRotationSwapsWidthAndHeight(inputRotationMode)) {
        MTLUInt2Swap(&newSize);
    }
    if (!MTLUInt2Equal(inputImageSize, newSize)) {
        inputImageSize = newSize;
        if (!MTLUInt2IsZero(newSize)) {
            [self updateViewGeometry];
            [self updateTextureVertics];
        }
    }
  

  }
  }

#pragma mark - MetalImageInput Protocol

• (void)setInputRotation:(MetalImageRotationMode)newInputRotation atIndex:(NSInteger)textureIndex {
  if (inputRotationMode != newInputRotation) {
  inputRotationMode = newInputRotation;
  [self updateTextureCoordinates];

    NSLog(@"update rotaion mode:%ld", (unsigned long)newInputRotation);
  

  }
  }

• (void)setInputTexture:(MetalImageTexture *)newInputTexture atIndex:(NSInteger)textureIndex {
  firstInputTexture = newInputTexture;
  [firstInputTexture lock];
  [self updateInputImageSize];
  }

• (void)drawRect:(CGRect)rect{
  if (firstInputTexture == nil)
  return;

  CAMetalLayer *metalLayer = (CAMetalLayer *)self.layer;
  id commandBuffer = [MetalDevice sharedCommandBuffer];
  id currentDrawable = [metalLayer nextDrawable];
  if (currentDrawable == nil) {
  NSLog(@"Obtain drawable failed...");
  return;
  }

  MTLRenderPassColorAttachmentDescriptor *colorAttachment = _renderPassDescriptor.colorAttachments[0];
  colorAttachment.texture = [currentDrawable texture];

  id renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
  [renderEncoder setDepthStencilState:_depthStencilState];
  [renderEncoder setRenderPipelineState:_pipelineState];
  [renderEncoder setVertexBuffer:_verticsBuffer offset:0 atIndex:0];
  [renderEncoder setVertexBuffer:_coordBuffer offset:0 atIndex:1];
  [renderEncoder setFragmentTexture:[firstInputTexture texture] atIndex:0];
  [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:MetalImageDefaultRenderVetexCount instanceCount:1];
  [renderEncoder endEncoding];

  [commandBuffer presentDrawable:currentDrawable];

  [MetalDevice commitCommandBufferWaitUntilDone:NO];

  currentDrawable = nil;

  [firstInputTexture unlock];
  firstInputTexture = nil;
  }

• (void)newTextureReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex {
  runMetalOnMainQueueWithoutDeadlocking(^{
  [self draw];
  });
  }

[jzting]

Thanks! I'm still seeing issues with the voronoi filter, so it must be an issue particular to that.