feat: add rendering panel

packages/backend/src/assets/gpuTextures/textures.ts

@@ -1,6 +1,6 @@
 import type { TextureDataState } from '@devtool/frontend/pages/assets/assets';
 import type { PixiDevtools } from '../../pixi';
-import type { TextureSource, GlTexture, CanvasSource } from 'pixi.js';
+import type { TextureSource, GlTexture, CanvasSource, WebGLRenderer, WebGPURenderer } from 'pixi.js';
 
 const gpuTextureFormatSize: Record<string, number> = {
   r8unorm: 1,
@@ -84,22 +84,21 @@ export class Textures {
 
   public get() {
     const currentTextures = this._devtool.renderer.texture.managedTextures;
-    // @ts-expect-error - private property
-    const glTextures = this._devtool.renderer.texture['_glTextures'] as Record<number, GlTexture | GPUTexture>;
+    const glTextures = this.getWebTextures();
 
     const data: TextureDataState[] = [];
     currentTextures.forEach((texture) => {
       if (!texture.resource) return;
 
       if (!this._textures.get(texture.uid)) {
-        const res = this._getTextureSource(texture);
+        const res = this.getTextureSource(texture);
         if (res) {
           this._textures.set(texture.uid, res);
         }
       }
 
       if (!this._gpuTextureSize.has(texture.uid)) {
-        const size = this._getMemorySize(texture, glTextures[texture.uid]);
+        const size = this.getMemorySize(texture, glTextures[texture.uid]);
         if (size) {
           this._gpuTextureSize.set(texture.uid, size);
         }
@@ -129,7 +128,17 @@ export class Textures {
     return data;
   }
 
-  private _getTextureSource(texture: TextureSource) {
+  public getWebTextures() {
+    const glRenderer = this._devtool.renderer as WebGLRenderer;
+    const gpuRenderer = this._devtool.renderer as WebGPURenderer;
+
+    const glTextures: Record<number, GlTexture | GPUTexture> =
+      glRenderer.texture['_glTextures'] || gpuRenderer.texture['_gpuSources'];
+
+    return glTextures;
+  }
+
+  public getTextureSource(texture: TextureSource) {
     if (
       texture.resource instanceof ImageBitmap ||
       texture.resource instanceof HTMLImageElement ||
@@ -175,7 +184,7 @@ export class Textures {
     return canvas.toDataURL('image/png');
   }
 
-  private _getMemorySize(texture: TextureSource, webTexture: GlTexture | GPUTexture): number | null {
+  public getMemorySize(texture: TextureSource, webTexture: GlTexture | GPUTexture): number | null {
     if (!webTexture) return null;
     if (Array.isArray(texture.resource) && texture.resource[0] instanceof Uint8Array) {
       return texture.resource.reduce((acc, res) => acc + res.byteLength, 0);

packages/backend/src/pixi.ts

@@ -1,7 +1,7 @@
 import type { DevtoolState } from '@devtool/frontend/types';
 import { DevtoolMessage } from '@devtool/frontend/types';
 import type { Devtools } from '@pixi/devtools';
-import type { Application, Container, Renderer } from 'pixi.js';
+import type { Application, Container, Renderer, WebGLRenderer } from 'pixi.js';
 import { extensions } from './extensions/Extensions';
 import { Overlay } from './scene/overlay/overlay';
 import { overlayExtension } from './scene/overlay/overlayExtension';
@@ -19,6 +19,8 @@ import { loop } from './utils/loop';
 import { Throttle } from './utils/throttle';
 import { Textures } from './assets/gpuTextures/textures';
 import type { TextureState } from '@devtool/frontend/pages/assets/assets';
+import { Rendering } from './rendering/rendering';
+import type { RenderingState } from '@devtool/frontend/pages/rendering/rendering';
 
 /**
  * PixiWrapper is a class that wraps around the PixiJS library.
@@ -31,7 +33,14 @@ class PixiWrapper {
   };
   public state: Omit<
     DevtoolState,
-    'active' | 'setActive' | 'bridge' | 'setBridge' | 'chromeProxy' | 'setChromeProxy' | keyof TextureState
+    | 'active'
+    | 'setActive'
+    | 'bridge'
+    | 'setBridge'
+    | 'chromeProxy'
+    | 'setChromeProxy'
+    | keyof TextureState
+    | keyof RenderingState
   > = {
     version: null,
     setVersion: function (version: DevtoolState['version']) {
@@ -88,6 +97,7 @@ class PixiWrapper {
   public properties = new Properties(this);
   public overlay = new Overlay(this);
   public textures = new Textures(this);
+  public rendering = new Rendering(this);
   // Private properties
   private _devtools: Devtools | undefined;
   private _app: Application | undefined;
@@ -230,6 +240,10 @@ class PixiWrapper {
 
   public get majorVersion() {
     if (this.version === '') {
+      if (!this.stage) {
+        return null;
+      }
+
       // lets try and find the version
       const stage = this.stage;
       if (stage.effects != null && Array.isArray(stage.effects) && '_updateFlags' in stage) {
@@ -248,6 +262,15 @@ class PixiWrapper {
     return this.app || (this.stage && this.renderer) ? DevtoolMessage.active : DevtoolMessage.inactive;
   }
 
+  public get rendererType(): 'webgl' | 'webgl2' | 'webgpu' | null {
+    if (!this.renderer) return null;
+    return this.renderer.type === 0b10
+      ? 'webgpu'
+      : (this.renderer as WebGLRenderer).context.webGLVersion === 1
+        ? 'webgl'
+        : 'webgl2';
+  }
+
   public inject() {
     // eslint-disable-next-line @typescript-eslint/no-this-alias
     const that = this;
@@ -284,6 +307,7 @@ class PixiWrapper {
     // TODO: tree: 300ms, stats: 300ms, properties: 300ms, overlay: 50ms
 
     this.overlay.update();
+    this.rendering.update();
     if (this._updateThrottle.shouldExecute(this.settings.throttle)) {
       this.preupdate();
 
@@ -314,6 +338,7 @@ class PixiWrapper {
     this.stats.init();
     this.tree.init();
     this.textures.init();
+    this.rendering.init();
     this._initialized = true;
   }
 

packages/backend/src/rendering/instructions.ts

@@ -0,0 +1,245 @@
+import type {
+  BatchInstruction,
+  FilterInstruction,
+  GraphicsInstruction,
+  MaskInstruction,
+  MeshInstruction,
+  NineSliceInstruction,
+  TilingSpriteInstruction,
+} from '@devtool/frontend/pages/rendering/instructions/Instructions';
+import type {
+  Batch,
+  BlurFilter,
+  Filter,
+  GlProgram,
+  GpuProgram,
+  Graphics,
+  Mesh,
+  NineSliceSprite,
+  FilterInstruction as PixiFilterInstruction,
+  Sprite,
+  StencilMaskInstruction,
+  TilingSprite,
+  RenderContainer,
+} from 'pixi.js';
+import { getRenderableData, getStateData, getTextureData } from './renderableData';
+import { PixiDevtools } from '../pixi';
+import { getProgramSource } from './program';
+
+type TextureCache = Parameters<typeof getTextureData>[1];
+export function getBatchInstruction(instruction: Batch, textureCache: TextureCache): BatchInstruction {
+  const textures: BatchInstruction['textures'] = [];
+  instruction.textures.textures.forEach((texture) => {
+    if (texture == null) return;
+    const tex = getTextureData(texture, textureCache);
+    if (tex) {
+      textures.push(tex);
+    } else {
+      console.log('Texture not found', texture);
+    }
+  });
+
+  return {
+    type: 'batch',
+    action: instruction.action,
+    blendMode: instruction.blendMode,
+    size: instruction.size,
+    start: instruction.start,
+    drawCalls: 0,
+    drawTextures: [],
+    textures,
+  };
+}
+
+export function getCustomRenderableInstruction(instruction: RenderContainer, _textureCache: TextureCache) {
+  return {
+    type: 'customRender',
+    action: 'custom',
+    renderable: {
+      ...getRenderableData(instruction),
+    },
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}
+
+export function getGraphicsInstruction(instruction: Graphics, _textureCache: TextureCache): GraphicsInstruction {
+  return {
+    type: 'graphics',
+    action: 'draw',
+    renderable: {
+      ...getRenderableData(instruction),
+    },
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}
+
+export function getNineSliceInstruction(
+  instruction: NineSliceSprite,
+  textureCache: TextureCache,
+): NineSliceInstruction {
+  return {
+    type: 'nineSlice',
+    action: 'draw',
+    renderable: {
+      topHeight: instruction.topHeight,
+      bottomHeight: instruction.bottomHeight,
+      leftWidth: instruction.leftWidth,
+      rightWidth: instruction.rightWidth,
+      texture: instruction.texture ? getTextureData(instruction.texture._source, textureCache) : null,
+      originalWidth: instruction.originalWidth,
+      originalHeight: instruction.originalHeight,
+      ...getRenderableData(instruction),
+    },
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}
+
+export function getMaskInstruction(instruction: StencilMaskInstruction, _textureCache: TextureCache): MaskInstruction {
+  const mask = instruction.mask?.mask;
+  return {
+    type: instruction.renderPipeId,
+    action: instruction.action,
+    mask: mask
+      ? {
+          ...getRenderableData(mask),
+        }
+      : null,
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}
+
+export function getFilterInstruction(
+  instruction: PixiFilterInstruction,
+  textureCache: TextureCache,
+): FilterInstruction {
+  const rendererType = PixiDevtools.renderer.name === 'webgl' ? 'webgl' : 'webgpu';
+  const getProgramSource = (filter: Filter, shader: 'fragment' | 'vertex'): string => {
+    const type = rendererType === 'webgl' ? 'glProgram' : 'gpuProgram';
+    let program: GlProgram | GpuProgram;
+
+    // if blur filter then you need to get the blurX/blurY shaders
+    if ((filter as BlurFilter).blurXFilter) {
+      program = (filter as BlurFilter).blurXFilter[type];
+    } else {
+      program = filter[type];
+    }
+
+    if (!program) {
+      return '';
+    }
+
+    const source = program[shader];
+
+    if (!source) {
+      return '';
+    }
+
+    if (typeof source === 'string') {
+      return source;
+    }
+
+    return source.source;
+  };
+
+  return {
+    type: 'filter',
+    action: instruction.action,
+    filter: instruction.filterEffect?.filters?.map((filter) => {
+      return {
+        type: filter.constructor.name,
+        padding: filter.padding,
+        resolution: filter.resolution,
+        antialias: filter.antialias,
+        blendMode: filter.blendMode,
+        program: {
+          fragment: getProgramSource(filter, 'fragment'),
+          vertex: getProgramSource(filter, 'vertex'),
+        },
+        state: getStateData(filter._state),
+      };
+    }),
+    renderables:
+      instruction.renderables?.map((renderable) => ({
+        texture: (renderable as Sprite).texture
+          ? getTextureData((renderable as Sprite).texture.source, textureCache)
+          : null,
+        ...getRenderableData(renderable),
+      })) || ([] as FilterInstruction['renderables']),
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}
+
+interface PixiMeshObjectInstruction {
+  renderPipeId: string;
+  mesh: Mesh;
+}
+
+export function getMeshInstruction(
+  instruction: Mesh | PixiMeshObjectInstruction,
+  textureCache: TextureCache,
+): MeshInstruction {
+  const mesh = (instruction as PixiMeshObjectInstruction)?.mesh || instruction;
+  const rendererType = PixiDevtools.renderer.name === 'webgl' ? 'webgl' : 'webgpu';
+  let shader = mesh.shader;
+  const program = {
+    fragment: null as string | null,
+    vertex: null as string | null,
+  };
+
+  if (!shader) {
+    shader = PixiDevtools.renderer.renderPipes.mesh['_adaptor']._shader;
+    program.fragment = getProgramSource(shader, 'fragment', rendererType);
+    program.vertex = getProgramSource(shader, 'vertex', rendererType);
+  } else {
+    program.fragment = getProgramSource(shader, 'fragment', rendererType);
+    program.vertex = getProgramSource(shader, 'vertex', rendererType);
+  }
+  return {
+    type: 'mesh',
+    action: 'draw',
+    renderable: {
+      texture: mesh.texture ? getTextureData(mesh.texture.source, textureCache) : null,
+      program,
+      state: getStateData(mesh.state),
+      geometry: {
+        indices: Array.from(mesh.geometry.indices),
+        positions: Array.from(mesh.geometry.positions),
+        uvs: Array.from(mesh.geometry.uvs),
+      },
+      ...getRenderableData(mesh),
+    },
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}
+
+export function getTileSpriteInstruction(
+  tilingSprite: TilingSprite,
+  textureCache: TextureCache,
+): TilingSpriteInstruction {
+  return {
+    type: 'tilingSprite',
+    action: 'draw',
+    renderable: {
+      ...getRenderableData(tilingSprite),
+      texture: tilingSprite.texture ? getTextureData(tilingSprite.texture.source, textureCache) : null,
+      tilePosition: {
+        x: tilingSprite.tilePosition.x,
+        y: tilingSprite.tilePosition.y,
+      },
+      tileScale: {
+        x: tilingSprite.tileScale.x,
+        y: tilingSprite.tileScale.y,
+      },
+      tileRotation: tilingSprite.tileRotation,
+      clampMargin: tilingSprite.clampMargin,
+    },
+    drawCalls: 0,
+    drawTextures: [],
+  };
+}