diff --git a/integration/boofcv-swing/src/main/java/boofcv/gui/mesh/MeshViewerPanel.java b/integration/boofcv-swing/src/main/java/boofcv/gui/mesh/MeshViewerPanel.java index 4db79689a4..bea83e8e25 100644 --- a/integration/boofcv-swing/src/main/java/boofcv/gui/mesh/MeshViewerPanel.java +++ b/integration/boofcv-swing/src/main/java/boofcv/gui/mesh/MeshViewerPanel.java @@ -18,11 +18,13 @@ package boofcv.gui.mesh; +import boofcv.alg.distort.pinhole.LensDistortionPinhole; import boofcv.alg.geo.PerspectiveOps; import boofcv.gui.image.SaveImageOnClick; import boofcv.gui.image.VisualizeImageData; import boofcv.io.image.ConvertBufferedImage; import boofcv.misc.BoofMiscOps; +import boofcv.struct.calib.CameraPinhole; import boofcv.struct.image.GrayF32; import boofcv.struct.image.ImageDimension; import boofcv.struct.image.InterleavedU8; @@ -115,6 +117,8 @@ public class MeshViewerPanel extends JPanel implements VerbosePrint, KeyEventDis @Nullable PrintStream verbose = null; + CameraPinhole intrinsics = new CameraPinhole(); + /** * Convenience constructor that calls {@link #setMesh(VertexMesh, boolean)} and the default constructor. * @@ -303,11 +307,13 @@ private void render() { return; } - PerspectiveOps.createIntrinsic(dimension.width, dimension.height, hfov, -1, renderer.getIntrinsics()); + PerspectiveOps.createIntrinsic(dimension.width, dimension.height, hfov, -1, intrinsics); + var factory = new LensDistortionPinhole(intrinsics); + renderer.setCamera(factory, dimension.width, dimension.height); } synchronized (controls) { - activeControl.setCamera(renderer.getIntrinsics()); + activeControl.setCamera(intrinsics); renderer.worldToView.setTo(activeControl.getWorldToCamera()); } diff --git a/main/boofcv-io/src/benchmark/java/boofcv/visualize/BenchmarkRenderMesh.java b/main/boofcv-io/src/benchmark/java/boofcv/visualize/BenchmarkRenderMesh.java index fdcba7f19b..5eca47a560 100644 --- a/main/boofcv-io/src/benchmark/java/boofcv/visualize/BenchmarkRenderMesh.java +++ b/main/boofcv-io/src/benchmark/java/boofcv/visualize/BenchmarkRenderMesh.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Peter Abeles. All Rights Reserved. + * Copyright (c) 2024, Peter Abeles. All Rights Reserved. * * This file is part of BoofCV (http://boofcv.org). * @@ -18,6 +18,7 @@ package boofcv.visualize; +import boofcv.alg.distort.pinhole.LensDistortionPinhole; import boofcv.alg.geo.PerspectiveOps; import boofcv.struct.calib.CameraPinhole; import boofcv.struct.mesh.VertexMesh; @@ -54,7 +55,7 @@ public class BenchmarkRenderMesh { createFlatSquareScene(intrinsics, rand); - renderer.getIntrinsics().setTo(intrinsics); + renderer.setCamera(new LensDistortionPinhole(intrinsics), intrinsics.width, intrinsics.height); } private void createFlatSquareScene( CameraPinhole intrinsics, Random rand ) { diff --git a/main/boofcv-io/src/main/java/boofcv/io/points/PointCloudIO.java b/main/boofcv-io/src/main/java/boofcv/io/points/PointCloudIO.java index 9c314dcb62..565bf5a38c 100644 --- a/main/boofcv-io/src/main/java/boofcv/io/points/PointCloudIO.java +++ b/main/boofcv-io/src/main/java/boofcv/io/points/PointCloudIO.java @@ -27,16 +27,15 @@ import boofcv.struct.mesh.VertexMesh; import georegression.struct.point.Point3D_F32; import georegression.struct.point.Point3D_F64; +import org.apache.commons.io.FilenameUtils; import org.ddogleg.struct.DogArray; import org.ddogleg.struct.DogArray_I32; import org.jetbrains.annotations.Nullable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; +import java.io.*; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.Locale; /** * Code for reading different point cloud formats @@ -146,6 +145,29 @@ public static void load( Format format, InputStream input, PointCloudWriter outp } } + /** + * Loads the mesh from a file. File type is determined by the file's extension. + * + * @param file Which file it should load + * @param mesh (Output) storage for the mesh + * @param colors (Output) Storage for vertex colors + */ + public static void load( File file, VertexMesh mesh, DogArray_I32 colors ) throws IOException { + String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(Locale.ENGLISH); + var type = switch (extension) { + case "ply" -> PointCloudIO.Format.PLY; + case "stl" -> PointCloudIO.Format.STL; + case "obj" -> PointCloudIO.Format.OBJ; + default -> { + throw new RuntimeException("Unknown file type: " + extension); + } + }; + + try (var input = new FileInputStream(file)) { + PointCloudIO.load(type, input, mesh, colors); + } + } + /** * Reads a 3D mesh from the input stream in the specified format and writes it to the output. * diff --git a/main/boofcv-io/src/main/java/boofcv/visualize/RenderMesh.java b/main/boofcv-io/src/main/java/boofcv/visualize/RenderMesh.java index cc27c3ff88..cac6aa9ab6 100644 --- a/main/boofcv-io/src/main/java/boofcv/visualize/RenderMesh.java +++ b/main/boofcv-io/src/main/java/boofcv/visualize/RenderMesh.java @@ -18,13 +18,20 @@ package boofcv.visualize; +import boofcv.alg.distort.LensDistortionNarrowFOV; +import boofcv.alg.distort.brown.LensDistortionBrown; +import boofcv.alg.distort.pinhole.LensDistortionPinhole; +import boofcv.alg.geo.PerspectiveOps; import boofcv.alg.interpolate.InterpolatePixelMB; import boofcv.alg.misc.ImageMiscOps; import boofcv.factory.interpolate.FactoryInterpolation; import boofcv.misc.BoofMiscOps; import boofcv.struct.border.BorderType; import boofcv.struct.calib.CameraPinhole; +import boofcv.struct.calib.CameraPinholeBrown; +import boofcv.struct.distort.Point2Transform2_F64; import boofcv.struct.image.GrayF32; +import boofcv.struct.image.ImageDimension; import boofcv.struct.image.InterleavedU8; import boofcv.struct.mesh.VertexMesh; import georegression.geometry.UtilPolygons2D_F64; @@ -54,12 +61,13 @@ * * * @author Peter Abeles */ +@SuppressWarnings({"NullAway.Init"}) public class RenderMesh implements VerbosePrint { /** What color background pixels are set to by default in RGBA. Default value is white */ public @Getter @Setter int defaultColorRgba = 0xFFFFFF; @@ -73,9 +81,6 @@ public class RenderMesh implements VerbosePrint { /** Rendered color image. Pixels are in RGBA format. */ public @Getter final InterleavedU8 rgbImage = new InterleavedU8(1, 1, 3); - /** Pinhole camera model needed to go from depth image to 3D point */ - public @Getter final CameraPinhole intrinsics = new CameraPinhole(); - /** Transform from world (what the mesh is in) to the camera view */ public @Getter final Se3_F64 worldToView = new Se3_F64(); @@ -90,6 +95,11 @@ public class RenderMesh implements VerbosePrint { private InterpolatePixelMB textureInterp = FactoryInterpolation.bilinearPixelMB(textureImage, BorderType.EXTENDED); private float[] textureValues = new float[3]; + // Intrinsic camera model + protected Point2Transform2_F64 pixelToNorm; + protected Point2Transform2_F64 normToPixel; + protected ImageDimension resolution = new ImageDimension(); + //---------- Workspace variables private final Point3D_F64 camera = new Point3D_F64(); private final Point2D_F64 point = new Point2D_F64(); @@ -113,15 +123,55 @@ public void setTextureImage( InterleavedU8 textureImage ) { textureValues = new float[textureImage.numBands]; } + /** + * Specifies a pinhole camera from its fov and image shape. + * + * @param hfov Horizontal field of view. Degrees. + */ + public void setCamera( double hfov, int width, int height ) { + var model = new CameraPinhole(); + PerspectiveOps.createIntrinsic(width, height, hfov, -1, model); + var factory = new LensDistortionPinhole(model); + setCamera(factory, model.width, model.height); + } + + /** + * Specifies the intrinsic camera model + */ + public void setCamera( CameraPinholeBrown model ) { + var factory = new LensDistortionBrown(model); + setCamera(factory, model.width, model.height); + } + + /** + * Specifies the intrinsic camera model + */ + public void setCamera( LensDistortionNarrowFOV factory, int width, int height ) { + setCamera(factory.undistort_F64(true, false), + factory.distort_F64(false, true), + width, height); + } + + /** + * Specifies the intrinsic camera model + */ + public void setCamera( Point2Transform2_F64 pixelToNorm, + Point2Transform2_F64 normToPixel, + int width, int height ) { + this.pixelToNorm = pixelToNorm; + this.normToPixel = normToPixel; + this.resolution.setTo(width, height); + } + /** * Renders the mesh onto an image. Produces an RGB image and depth image. Must have configured - * {@link #intrinsics} already and set {@link #worldToView}. + * {@link #setCamera(CameraPinholeBrown)} already and set {@link #worldToView}. * * @param mesh The mesh that's going to be rendered. */ public void render( VertexMesh mesh ) { // Sanity check to see if intrinsics has been configured - BoofMiscOps.checkTrue(intrinsics.width > 0 && intrinsics.height > 0, "Intrinsics not set"); + BoofMiscOps.checkTrue(resolution.width > 0 && resolution.height > 0, "Intrinsics not set"); // Make sure there are normals if it's configured to use them if (checkFaceNormal && mesh.faceNormals.size() == 0) @@ -130,11 +180,6 @@ public void render( VertexMesh mesh ) { // Initialize output images initializeImages(); - final double fx = intrinsics.fx; - final double fy = intrinsics.fy; - final double cx = intrinsics.cx; - final double cy = intrinsics.cy; - // Keep track of how many meshes were rendered int shapesRenderedCount = 0; @@ -182,11 +227,8 @@ public void render( VertexMesh mesh ) { double normX = camera.x/camera.z; double normY = camera.y/camera.z; - // Project onto the image - double pixelX = normX*fx + cx; - double pixelY = normY*fy + cy; - - polygonProj.vertexes.grow().setTo(pixelX, pixelY); + // Compute pixel coordinates of this observation + normToPixel.compute(normX, normY, polygonProj.vertexes.grow()); meshCam.grow().setTo(camera); } @@ -230,8 +272,8 @@ static boolean isFrontVisible( VertexMesh mesh, int shapeIdx, int idx0, Point3D_ } void initializeImages() { - depthImage.reshape(intrinsics.width, intrinsics.height); - rgbImage.reshape(intrinsics.width, intrinsics.height); + depthImage.reshape(resolution.width, resolution.height); + rgbImage.reshape(resolution.width, resolution.height); ImageMiscOps.fill(rgbImage, defaultColorRgba); ImageMiscOps.fill(depthImage, Float.NaN); } @@ -269,7 +311,7 @@ void projectSurfaceColor( FastAccess mesh, Polygon2D_F64 polyProj, // The entire surface will have one color int color = surfaceColor.surfaceRgb(shapeIdx); - computeBoundingBox(intrinsics.width, intrinsics.height, polyProj, aabb); + computeBoundingBox(resolution.width, resolution.height, polyProj, aabb); // Go through all pixels and see if the points are inside the polygon. If so for (int pixelY = aabb.y0; pixelY < aabb.y1; pixelY++) { @@ -303,7 +345,7 @@ void projectSurfaceColor( FastAccess mesh, Polygon2D_F64 polyProj, void projectSurfaceTexture( FastAccess mesh, Polygon2D_F64 polyProj, Polygon2D_F32 polyText ) { // Scale factor to normalize image pixels from 0 to 1.0 - float scale = Math.max(intrinsics.width, intrinsics.height); + float scale = Math.max(resolution.width, resolution.height); // If the mesh has more than 3 sides, break it up into triangles using the first vertex as a pivot // This works because the mesh has to be convex @@ -339,7 +381,7 @@ void projectSurfaceTexture( FastAccess mesh, Polygon2D_F64 polyProj // TODO look at vertexes and get min/max depth. Use that to quickly reject pixels based on depth without // convex intersection or computing the depth at that pixel on this surface - computeBoundingBox(intrinsics.width, intrinsics.height, workTri, aabb); + computeBoundingBox(resolution.width, resolution.height, workTri, aabb); // Go through all pixels and see if the points are inside the polygon. If so for (int pixelY = aabb.y0; pixelY < aabb.y1; pixelY++) { diff --git a/main/boofcv-io/src/test/java/boofcv/visualize/TestRenderMesh.java b/main/boofcv-io/src/test/java/boofcv/visualize/TestRenderMesh.java index ace9add16e..def2d5bf70 100644 --- a/main/boofcv-io/src/test/java/boofcv/visualize/TestRenderMesh.java +++ b/main/boofcv-io/src/test/java/boofcv/visualize/TestRenderMesh.java @@ -19,6 +19,7 @@ package boofcv.visualize; import boofcv.alg.geo.PerspectiveOps; +import boofcv.struct.calib.CameraPinholeBrown; import boofcv.struct.mesh.VertexMesh; import boofcv.testing.BoofStandardJUnit; import georegression.struct.point.Point2D_F64; @@ -51,15 +52,17 @@ public class TestRenderMesh extends BoofStandardJUnit { // turn off checking with normals to simply this test alg.setCheckFaceNormal(false); - PerspectiveOps.createIntrinsic(300, 200, 90, -1, alg.intrinsics); + var intrinsics = new CameraPinholeBrown(); + PerspectiveOps.createIntrinsic(300, 200, 90, -1, intrinsics); + alg.setCamera(intrinsics); // Render alg.render(mesh); // See if it did anything int count = 0; - for (int y = 0; y < alg.intrinsics.height; y++) { - for (int x = 0; x < alg.intrinsics.width; x++) { + for (int y = 0; y < alg.resolution.height; y++) { + for (int x = 0; x < alg.resolution.width; x++) { if (alg.rgbImage.get24(x, y) != 0xFFFFFF) count++; } @@ -97,7 +100,7 @@ public class TestRenderMesh extends BoofStandardJUnit { */ @Test void projectSurfaceColor() { var alg = new RenderMesh(); - alg.intrinsics.fsetShape(100, 120); + alg.resolution.setTo(100, 120); alg.initializeImages(); // Polygon of projected shape on to the image. Make is an AABB, but smaller than the one above @@ -117,8 +120,8 @@ public class TestRenderMesh extends BoofStandardJUnit { // Verify by counting the number of projected points int countDepth = 0; int countRgb = 0; - for (int y = 0; y < alg.intrinsics.height; y++) { - for (int x = 0; x < alg.intrinsics.width; x++) { + for (int y = 0; y < alg.resolution.height; y++) { + for (int x = 0; x < alg.resolution.width; x++) { if (alg.depthImage.get(x, y) == 10) countDepth++; if (alg.rgbImage.get24(x, y) != 0xFFFFFF) @@ -137,7 +140,7 @@ public class TestRenderMesh extends BoofStandardJUnit { return 1; } }; - alg.intrinsics.fsetShape(100, 120); + alg.resolution.setTo(100, 120); alg.initializeImages(); // Polygon of projected shape on to the image. Make is an AABB, but smaller than the one above @@ -164,8 +167,8 @@ public class TestRenderMesh extends BoofStandardJUnit { // Verify by counting the number of projected points int countDepth = 0; int countRgb = 0; - for (int y = 0; y < alg.intrinsics.height; y++) { - for (int x = 0; x < alg.intrinsics.width; x++) { + for (int y = 0; y < alg.resolution.height; y++) { + for (int x = 0; x < alg.resolution.width; x++) { if (!Float.isNaN(alg.depthImage.get(x, y))) { countDepth++; }