fix overlay resolution

packages/api/global.d.ts

@@ -36,6 +36,14 @@ declare global {
       __devtoolIgnoreChildren?: string;
       __devtoolLocked?: boolean;
     }
+
+    interface WebGLSystems {
+      __devtoolInjected?: boolean;
+    }
+
+    interface WebGPUSystems {
+      __devtoolInjected?: boolean;
+    }
   }
 
   namespace GlobalMixins {

packages/backend/src/assets/gpuTextures/textures.ts

@@ -1,6 +1,6 @@
 import type { TextureDataState } from '@devtool/frontend/pages/assets/assets';
 import type { PixiDevtools } from '../../pixi';
-import type { TextureSource, GlTexture } from 'pixi.js';
+import type { TextureSource, GlTexture, CanvasSource } from 'pixi.js';
 
 const gpuTextureFormatSize: Record<string, number> = {
   r8unorm: 1,
@@ -67,6 +67,7 @@ export class Textures {
   private _devtool: typeof PixiDevtools;
   private _textures: Map<number, string> = new Map();
   private _gpuTextureSize: Map<number, number> = new Map();
+  private _canvas = document.createElement('canvas');
 
   constructor(devtool: typeof PixiDevtools) {
     this._devtool = devtool;
@@ -88,6 +89,8 @@ export class Textures {
 
     const data: TextureDataState[] = [];
     currentTextures.forEach((texture) => {
+      if (!texture.resource) return;
+
       if (!this._textures.get(texture.uid)) {
         const res = this._getTextureSource(texture);
         if (res) {
@@ -127,27 +130,42 @@ export class Textures {
   }
 
   private _getTextureSource(texture: TextureSource) {
-    if (texture.resource instanceof ImageBitmap) {
+    if (
+      texture.resource instanceof ImageBitmap ||
+      texture.resource instanceof HTMLImageElement ||
+      texture.resource instanceof HTMLVideoElement
+    ) {
       return this._imageBitmapToString(texture.resource);
     } else if (texture.resource instanceof HTMLCanvasElement) {
       return this._canvasToString(texture.resource);
     }
+
+    // in an iframe instanceof does not work due to different window objects
+    // we now try using the textures uploadID
+    if (texture.uploadMethodId === 'image' || texture.uploadMethodId === 'video') {
+      if ((texture as CanvasSource).resizeCanvas) {
+        return this._canvasToString(texture.resource);
+      }
+      return this._imageBitmapToString(texture.resource);
+    }
+
+    // TODO buffer resource and compressed texture
+
     return null;
   }
 
   private _imageBitmapToString(imageBitmap: ImageBitmap | HTMLImageElement | HTMLVideoElement): string {
     // Create a canvas element
-    const canvas = document.createElement('canvas');
-    canvas.width = imageBitmap.width;
-    canvas.height = imageBitmap.height;
+    this._canvas.width = imageBitmap.width;
+    this._canvas.height = imageBitmap.height;
 
     // Get the context of the canvas
-    const ctx = canvas.getContext('2d')!;
+    const ctx = this._canvas.getContext('2d')!;
 
     // Draw the ImageBitmap on the canvas
     ctx.drawImage(imageBitmap, 0, 0);
 
-    const res = this._canvasToString(canvas);
+    const res = this._canvasToString(this._canvas);
 
     // Convert the canvas to a blob
     return res;

packages/backend/src/pixi.ts

@@ -251,17 +251,17 @@ class PixiWrapper {
   public inject() {
     // eslint-disable-next-line @typescript-eslint/no-this-alias
     const that = this;
-    if (this.renderer) {
+    if (this.renderer && !this.renderer.__devtoolInjected) {
+      this.renderer.__devtoolInjected = true;
       this.renderer.render = new Proxy(this.renderer.render, {
         apply(target, thisArg, ...args) {
           that.update();
           // @ts-expect-error - TODO: fix this type
           return target.apply(thisArg, ...args);
         },
       });
+      window.postMessage({ method: DevtoolMessage.active, data: {} }, '*');
     }
-
-    window.postMessage({ method: DevtoolMessage.active, data: {} }, '*');
   }
 
   public reset() {
@@ -273,6 +273,7 @@ class PixiWrapper {
     this._pixi = undefined;
     this._version = undefined;
     this._initialized = false;
+    window.postMessage({ method: DevtoolMessage.pulse, data: {} }, '*');
   }
 
   public update() {

packages/backend/src/scene/overlay/overlay.ts

@@ -195,10 +195,13 @@ export class Overlay {
   private _updateOverlay() {
     const canvas = this._devtool.canvas!;
     const renderer = this._devtool.renderer!;
+    const version = this._devtool.majorVersion;
+
+    const res = version === '8' ? 1 : renderer.resolution;
 
     Object.assign(this._overlay!.style, {
-      width: `${renderer.width / renderer.resolution}px`,
-      height: `${renderer.height / renderer.resolution}px`,
+      width: `${renderer.width / res}px`,
+      height: `${renderer.height / res}px`,
     });
     const cBounds = canvas.getBoundingClientRect();
     this._overlay!.style.transform = '';

packages/backend/src/utils/poller.ts

@@ -9,15 +9,22 @@ let pixiPollingInterval: number | undefined;
 let tryCount = 0;
 
 // Function to start polling for PixiJS
-export function pollPixi(foundCallback: () => void) {
+export function pollPixi(foundCallback: () => void, loopCallback?: () => void, stopCallback?: () => void) {
   // Start a new interval
   pixiPollingInterval = window.setInterval(() => {
     // If we've tried more than the max tries, stop polling
     if (tryCount > MAX_TRIES) {
       stopPolling();
+      if (stopCallback) {
+        stopCallback();
+      }
       return;
     }
 
+    // If a loop callback is provided, call it
+    if (loopCallback) {
+      loopCallback();
+    }
     // Try to detect Pixi
     tryDetectPixi(foundCallback);
 
@@ -45,6 +52,8 @@ function tryDetectPixi(foundCallback: () => void) {
     if (pixiDetectionResult === DevtoolMessage.active) {
       stopPolling();
       foundCallback();
+    } else {
+      window.postMessage({ method: pixiDetectionResult, data: JSON.stringify({}) }, '*');
     }
   } catch (error) {
     // If an error occurred, stop polling

packages/devtool-chrome/src/inject/index.ts

@@ -1,21 +1,30 @@
 // Note: this file is compiled to `dist/content-inject/index.js` and is used by the content script
 
-import { PixiDevtools } from '@devtool/backend/pixi';
 import { pollPixi } from '@devtool/backend/utils/poller';
-import { convertPostMessage } from '../messageUtils';
 import { DevtoolMessage } from '@devtool/frontend/types';
+import { convertPostMessage } from '../messageUtils';
 import type { Application, Renderer } from 'pixi.js';
 
 function attach() {
-  const renderer = PixiDevtools.renderer;
-
-  if (renderer) {
-    window.postMessage(convertPostMessage(DevtoolMessage.active, {}), '*');
-  }
+  window.postMessage(convertPostMessage(DevtoolMessage.active, {}), '*');
 }
+function injectIFrames() {
+  if (window.frames) {
+    for (let i = 0; i < window.frames.length; i += 1) {
+      try {
+        const frame = window.frames[i] as Window & typeof globalThis;
+        frame.__PIXI_APP_INIT__ = init;
+        frame.__PIXI_RENDERER_INIT__ = init;
+      } catch (_) {
+        // access to iframe was denied
+      }
+    }
+  }
+  window.__PIXI_APP_INIT__ = init;
+  window.__PIXI_RENDERER_INIT__ = init;
 
-pollPixi(attach);
-
+  return null;
+}
 function init(arg: Application | Renderer) {
   const stage = (arg as Application).stage;
   const renderer = stage ? (arg as Application).renderer : (arg as Renderer);
@@ -26,5 +35,13 @@ function init(arg: Application | Renderer) {
   };
   window.__PIXI_DEVTOOLS_WRAPPER__?.reset();
 }
-window.__PIXI_APP_INIT__ = init;
-window.__PIXI_RENDERER_INIT__ = init;
+
+// try injecting iframes using requestAnimationFrame until poller is stopped
+const inject = () => {
+  injectIFrames();
+  requestAnimationFrame(inject);
+};
+
+inject();
+
+pollPixi(attach);

packages/frontend/src/App.tsx

@@ -92,6 +92,10 @@ const App: React.FC<AppProps> = ({ bridge, chromeProxy }) => {
             setActive(true);
           }
           break;
+        case 'pixi-pulse': {
+          bridge('window.__PIXI_DEVTOOLS_WRAPPER__.inject()');
+          break;
+        }
         case 'pixi-inactive':
           {
             setActive(false);

packages/frontend/src/pages/assets/gpuTextures/TextureViewer.tsx

@@ -10,7 +10,7 @@ interface TextureViewerProps extends TextureDataState {
   selected?: boolean;
 }
 export const TextureViewer: React.FC<TextureViewerProps> = memo(
-  ({ blob, width, height, name, isLoaded, onClick, selected, gpuSize }) => {
+  ({ blob, pixelWidth, pixelHeight, name, isLoaded, onClick, selected, gpuSize }) => {
     const { theme } = useTheme();
     const [imageSize, setImageSize] = useState<number | null>(null);
     useEffect(() => {
@@ -47,7 +47,7 @@ export const TextureViewer: React.FC<TextureViewerProps> = memo(
           <div className={`${bg} group-hover:bg-secondary rounded-b-sm`}>
             <div className={`w-full truncate px-1 py-0.5 pb-2 text-center text-xs text-white`}>{sanitizedName}</div>
             <div className={`h-auto w-full truncate px-1 py-0.5 text-left text-xs text-white`}>
-              Size: {formatNumber(width, 1)} x {formatNumber(height, 1)}
+              Size: {formatNumber(pixelWidth, 1)} x {formatNumber(pixelHeight, 1)}
             </div>
 
             <div className="h-auto w-full truncate px-1 py-0.5 text-left text-xs text-white">

packages/frontend/src/types.ts

@@ -7,6 +7,7 @@ export enum DevtoolMessage {
   active = 'pixi-active',
   inactive = 'pixi-inactive',
   stateUpdate = 'pixi-state-update',
+  pulse = 'pixi-pulse',
 }
 
 export type SceneGraphEntry = {