RenderMesh

- Camera model is now generic

integration/boofcv-swing/src/main/java/boofcv/gui/mesh/MeshViewerPanel.java

@@ -18,11 +18,13 @@
 
 package boofcv.gui.mesh;
 
+import boofcv.alg.distort.pinhole.LensDistortionPinhole;
 import boofcv.alg.geo.PerspectiveOps;
 import boofcv.gui.image.SaveImageOnClick;
 import boofcv.gui.image.VisualizeImageData;
 import boofcv.io.image.ConvertBufferedImage;
 import boofcv.misc.BoofMiscOps;
+import boofcv.struct.calib.CameraPinhole;
 import boofcv.struct.image.GrayF32;
 import boofcv.struct.image.ImageDimension;
 import boofcv.struct.image.InterleavedU8;
@@ -115,6 +117,8 @@ public class MeshViewerPanel extends JPanel implements VerbosePrint, KeyEventDis
 
 	@Nullable PrintStream verbose = null;
 
+	CameraPinhole intrinsics = new CameraPinhole();
+
 	/**
 	 * Convenience constructor that calls {@link #setMesh(VertexMesh, boolean)} and the default constructor.
 	 *
@@ -303,11 +307,13 @@ private void render() {
 				return;
 			}
 
-			PerspectiveOps.createIntrinsic(dimension.width, dimension.height, hfov, -1, renderer.getIntrinsics());
+			PerspectiveOps.createIntrinsic(dimension.width, dimension.height, hfov, -1, intrinsics);
+			var factory = new LensDistortionPinhole(intrinsics);
+			renderer.setCamera(factory, dimension.width, dimension.height);
 		}
 
 		synchronized (controls) {
-			activeControl.setCamera(renderer.getIntrinsics());
+			activeControl.setCamera(intrinsics);
 			renderer.worldToView.setTo(activeControl.getWorldToCamera());
 		}
 

main/boofcv-io/src/benchmark/java/boofcv/visualize/BenchmarkRenderMesh.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Peter Abeles. All Rights Reserved.
+ * Copyright (c) 2024, Peter Abeles. All Rights Reserved.
  *
  * This file is part of BoofCV (http://boofcv.org).
  *
@@ -18,6 +18,7 @@
 
 package boofcv.visualize;
 
+import boofcv.alg.distort.pinhole.LensDistortionPinhole;
 import boofcv.alg.geo.PerspectiveOps;
 import boofcv.struct.calib.CameraPinhole;
 import boofcv.struct.mesh.VertexMesh;
@@ -54,7 +55,7 @@ public class BenchmarkRenderMesh {
 
 		createFlatSquareScene(intrinsics, rand);
 
-		renderer.getIntrinsics().setTo(intrinsics);
+		renderer.setCamera(new LensDistortionPinhole(intrinsics), intrinsics.width, intrinsics.height);
 	}
 
 	private void createFlatSquareScene( CameraPinhole intrinsics, Random rand ) {

main/boofcv-io/src/main/java/boofcv/io/points/PointCloudIO.java

@@ -27,16 +27,15 @@
 import boofcv.struct.mesh.VertexMesh;
 import georegression.struct.point.Point3D_F32;
 import georegression.struct.point.Point3D_F64;
+import org.apache.commons.io.FilenameUtils;
 import org.ddogleg.struct.DogArray;
 import org.ddogleg.struct.DogArray_I32;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
+import java.io.*;
 import java.nio.ByteOrder;
 import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 
 /**
  * Code for reading different point cloud formats
@@ -146,6 +145,29 @@ public static void load( Format format, InputStream input, PointCloudWriter outp
 		}
 	}
 
+	/**
+	 * Loads the mesh from a file. File type is determined by the file's extension.
+	 *
+	 * @param file Which file it should load
+	 * @param mesh (Output) storage for the mesh
+	 * @param colors (Output) Storage for vertex colors
+	 */
+	public static void load( File file, VertexMesh mesh, DogArray_I32 colors ) throws IOException {
+		String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(Locale.ENGLISH);
+		var type = switch (extension) {
+			case "ply" -> PointCloudIO.Format.PLY;
+			case "stl" -> PointCloudIO.Format.STL;
+			case "obj" -> PointCloudIO.Format.OBJ;
+			default -> {
+				throw new RuntimeException("Unknown file type: " + extension);
+			}
+		};
+
+		try (var input = new FileInputStream(file)) {
+			PointCloudIO.load(type, input, mesh, colors);
+		}
+	}
+
 	/**
 	 * Reads a 3D mesh from the input stream in the specified format and writes it to the output.
 	 *

main/boofcv-io/src/main/java/boofcv/visualize/RenderMesh.java

@@ -18,13 +18,20 @@
 
 package boofcv.visualize;
 
+import boofcv.alg.distort.LensDistortionNarrowFOV;
+import boofcv.alg.distort.brown.LensDistortionBrown;
+import boofcv.alg.distort.pinhole.LensDistortionPinhole;
+import boofcv.alg.geo.PerspectiveOps;
 import boofcv.alg.interpolate.InterpolatePixelMB;
 import boofcv.alg.misc.ImageMiscOps;
 import boofcv.factory.interpolate.FactoryInterpolation;
 import boofcv.misc.BoofMiscOps;
 import boofcv.struct.border.BorderType;
 import boofcv.struct.calib.CameraPinhole;
+import boofcv.struct.calib.CameraPinholeBrown;
+import boofcv.struct.distort.Point2Transform2_F64;
 import boofcv.struct.image.GrayF32;
+import boofcv.struct.image.ImageDimension;
 import boofcv.struct.image.InterleavedU8;
 import boofcv.struct.mesh.VertexMesh;
 import georegression.geometry.UtilPolygons2D_F64;
@@ -54,12 +61,13 @@
  * <ul>
  *     <li>{@link #defaultColorRgba} Specifies what color the background is.</li>
  *     <li>{@link #surfaceColor} Function which returns the color of a shape. The shape's index is passed.</li>
- *     <li>{@link #intrinsics} Camera intrinsics. This must be set before use.</li>
+ *     <li>{@link #setCamera(CameraPinholeBrown)} This must be set before use.</li>
  *     <li>{@link #worldToView} Transform from work to the current view.</li>
  * </ul>
  *
  * @author Peter Abeles
  */
+@SuppressWarnings({"NullAway.Init"})
 public class RenderMesh implements VerbosePrint {
 	/** What color background pixels are set to by default in RGBA. Default value is white */
 	public @Getter @Setter int defaultColorRgba = 0xFFFFFF;
@@ -73,9 +81,6 @@ public class RenderMesh implements VerbosePrint {
 	/** Rendered color image. Pixels are in RGBA format. */
 	public @Getter final InterleavedU8 rgbImage = new InterleavedU8(1, 1, 3);
 
-	/** Pinhole camera model needed to go from depth image to 3D point */
-	public @Getter final CameraPinhole intrinsics = new CameraPinhole();
-
 	/** Transform from world (what the mesh is in) to the camera view */
 	public @Getter final Se3_F64 worldToView = new Se3_F64();
 
@@ -90,6 +95,11 @@ public class RenderMesh implements VerbosePrint {
 	private InterpolatePixelMB<InterleavedU8> textureInterp = FactoryInterpolation.bilinearPixelMB(textureImage, BorderType.EXTENDED);
 	private float[] textureValues = new float[3];
 
+	// Intrinsic camera model
+	protected Point2Transform2_F64 pixelToNorm;
+	protected Point2Transform2_F64 normToPixel;
+	protected ImageDimension resolution = new ImageDimension();
+
 	//---------- Workspace variables
 	private final Point3D_F64 camera = new Point3D_F64();
 	private final Point2D_F64 point = new Point2D_F64();
@@ -113,15 +123,55 @@ public void setTextureImage( InterleavedU8 textureImage ) {
 		textureValues = new float[textureImage.numBands];
 	}
 
+	/**
+	 * Specifies a pinhole camera from its fov and image shape.
+	 *
+	 * @param hfov Horizontal field of view. Degrees.
+	 */
+	public void setCamera( double hfov, int width, int height ) {
+		var model = new CameraPinhole();
+		PerspectiveOps.createIntrinsic(width, height, hfov, -1, model);
+		var factory = new LensDistortionPinhole(model);
+		setCamera(factory, model.width, model.height);
+	}
+
+	/**
+	 * Specifies the intrinsic camera model
+	 */
+	public void setCamera( CameraPinholeBrown model ) {
+		var factory = new LensDistortionBrown(model);
+		setCamera(factory, model.width, model.height);
+	}
+
+	/**
+	 * Specifies the intrinsic camera model
+	 */
+	public void setCamera( LensDistortionNarrowFOV factory, int width, int height ) {
+		setCamera(factory.undistort_F64(true, false),
+				factory.distort_F64(false, true),
+				width, height);
+	}
+
+	/**
+	 * Specifies the intrinsic camera model
+	 */
+	public void setCamera( Point2Transform2_F64 pixelToNorm,
+						   Point2Transform2_F64 normToPixel,
+						   int width, int height ) {
+		this.pixelToNorm = pixelToNorm;
+		this.normToPixel = normToPixel;
+		this.resolution.setTo(width, height);
+	}
+
 	/**
 	 * Renders the mesh onto an image. Produces an RGB image and depth image. Must have configured
-	 * {@link #intrinsics} already and set {@link #worldToView}.
+	 * {@link #setCamera(CameraPinholeBrown)} already and set {@link #worldToView}.
 	 *
 	 * @param mesh The mesh that's going to be rendered.
 	 */
 	public void render( VertexMesh mesh ) {
 		// Sanity check to see if intrinsics has been configured
-		BoofMiscOps.checkTrue(intrinsics.width > 0 && intrinsics.height > 0, "Intrinsics not set");
+		BoofMiscOps.checkTrue(resolution.width > 0 && resolution.height > 0, "Intrinsics not set");
 
 		// Make sure there are normals if it's configured to use them
 		if (checkFaceNormal && mesh.faceNormals.size() == 0)
@@ -130,11 +180,6 @@ public void render( VertexMesh mesh ) {
 		// Initialize output images
 		initializeImages();
 
-		final double fx = intrinsics.fx;
-		final double fy = intrinsics.fy;
-		final double cx = intrinsics.cx;
-		final double cy = intrinsics.cy;
-
 		// Keep track of how many meshes were rendered
 		int shapesRenderedCount = 0;
 
@@ -182,11 +227,8 @@ public void render( VertexMesh mesh ) {
 				double normX = camera.x/camera.z;
 				double normY = camera.y/camera.z;
 
-				// Project onto the image
-				double pixelX = normX*fx + cx;
-				double pixelY = normY*fy + cy;
-
-				polygonProj.vertexes.grow().setTo(pixelX, pixelY);
+				// Compute pixel coordinates of this observation
+				normToPixel.compute(normX, normY, polygonProj.vertexes.grow());
 				meshCam.grow().setTo(camera);
 			}
 
@@ -230,8 +272,8 @@ static boolean isFrontVisible( VertexMesh mesh, int shapeIdx, int idx0, Point3D_
 	}
 
 	void initializeImages() {
-		depthImage.reshape(intrinsics.width, intrinsics.height);
-		rgbImage.reshape(intrinsics.width, intrinsics.height);
+		depthImage.reshape(resolution.width, resolution.height);
+		rgbImage.reshape(resolution.width, resolution.height);
 		ImageMiscOps.fill(rgbImage, defaultColorRgba);
 		ImageMiscOps.fill(depthImage, Float.NaN);
 	}
@@ -269,7 +311,7 @@ void projectSurfaceColor( FastAccess<Point3D_F64> mesh, Polygon2D_F64 polyProj,
 		// The entire surface will have one color
 		int color = surfaceColor.surfaceRgb(shapeIdx);
 
-		computeBoundingBox(intrinsics.width, intrinsics.height, polyProj, aabb);
+		computeBoundingBox(resolution.width, resolution.height, polyProj, aabb);
 
 		// Go through all pixels and see if the points are inside the polygon. If so
 		for (int pixelY = aabb.y0; pixelY < aabb.y1; pixelY++) {
@@ -303,7 +345,7 @@ void projectSurfaceColor( FastAccess<Point3D_F64> mesh, Polygon2D_F64 polyProj,
 	void projectSurfaceTexture( FastAccess<Point3D_F64> mesh, Polygon2D_F64 polyProj, Polygon2D_F32 polyText ) {
 
 		// Scale factor to normalize image pixels from 0 to 1.0
-		float scale = Math.max(intrinsics.width, intrinsics.height);
+		float scale = Math.max(resolution.width, resolution.height);
 
 		// If the mesh has more than 3 sides, break it up into triangles using the first vertex as a pivot
 		// This works because the mesh has to be convex
@@ -339,7 +381,7 @@ void projectSurfaceTexture( FastAccess<Point3D_F64> mesh, Polygon2D_F64 polyProj
 			// TODO look at vertexes and get min/max depth. Use that to quickly reject pixels based on depth without
 			//      convex intersection or computing the depth at that pixel on this surface
 
-			computeBoundingBox(intrinsics.width, intrinsics.height, workTri, aabb);
+			computeBoundingBox(resolution.width, resolution.height, workTri, aabb);
 
 			// Go through all pixels and see if the points are inside the polygon. If so
 			for (int pixelY = aabb.y0; pixelY < aabb.y1; pixelY++) {

main/boofcv-io/src/test/java/boofcv/visualize/TestRenderMesh.java

@@ -19,6 +19,7 @@
 package boofcv.visualize;
 
 import boofcv.alg.geo.PerspectiveOps;
+import boofcv.struct.calib.CameraPinholeBrown;
 import boofcv.struct.mesh.VertexMesh;
 import boofcv.testing.BoofStandardJUnit;
 import georegression.struct.point.Point2D_F64;
@@ -51,15 +52,17 @@ public class TestRenderMesh extends BoofStandardJUnit {
 
 		// turn off checking with normals to simply this test
 		alg.setCheckFaceNormal(false);
-		PerspectiveOps.createIntrinsic(300, 200, 90, -1, alg.intrinsics);
+		var intrinsics = new CameraPinholeBrown();
+		PerspectiveOps.createIntrinsic(300, 200, 90, -1, intrinsics);
+		alg.setCamera(intrinsics);
 
 		// Render
 		alg.render(mesh);
 
 		// See if it did anything
 		int count = 0;
-		for (int y = 0; y < alg.intrinsics.height; y++) {
-			for (int x = 0; x < alg.intrinsics.width; x++) {
+		for (int y = 0; y < alg.resolution.height; y++) {
+			for (int x = 0; x < alg.resolution.width; x++) {
 				if (alg.rgbImage.get24(x, y) != 0xFFFFFF)
 					count++;
 			}
@@ -97,7 +100,7 @@ public class TestRenderMesh extends BoofStandardJUnit {
 	 */
 	@Test void projectSurfaceColor() {
 		var alg = new RenderMesh();
-		alg.intrinsics.fsetShape(100, 120);
+		alg.resolution.setTo(100, 120);
 		alg.initializeImages();
 
 		// Polygon of projected shape on to the image. Make is an AABB, but smaller than the one above
@@ -117,8 +120,8 @@ public class TestRenderMesh extends BoofStandardJUnit {
 		// Verify by counting the number of projected points
 		int countDepth = 0;
 		int countRgb = 0;
-		for (int y = 0; y < alg.intrinsics.height; y++) {
-			for (int x = 0; x < alg.intrinsics.width; x++) {
+		for (int y = 0; y < alg.resolution.height; y++) {
+			for (int x = 0; x < alg.resolution.width; x++) {
 				if (alg.depthImage.get(x, y) == 10)
 					countDepth++;
 				if (alg.rgbImage.get24(x, y) != 0xFFFFFF)
@@ -137,7 +140,7 @@ public class TestRenderMesh extends BoofStandardJUnit {
 				return 1;
 			}
 		};
-		alg.intrinsics.fsetShape(100, 120);
+		alg.resolution.setTo(100, 120);
 		alg.initializeImages();
 
 		// Polygon of projected shape on to the image. Make is an AABB, but smaller than the one above
@@ -164,8 +167,8 @@ public class TestRenderMesh extends BoofStandardJUnit {
 		// Verify by counting the number of projected points
 		int countDepth = 0;
 		int countRgb = 0;
-		for (int y = 0; y < alg.intrinsics.height; y++) {
-			for (int x = 0; x < alg.intrinsics.width; x++) {
+		for (int y = 0; y < alg.resolution.height; y++) {
+			for (int x = 0; x < alg.resolution.width; x++) {
 				if (!Float.isNaN(alg.depthImage.get(x, y))) {
 					countDepth++;
 				}