diff --git a/demo/bev_demo.py b/demo/bev_demo.py new file mode 100644 index 00000000..07a8cc01 --- /dev/null +++ b/demo/bev_demo.py @@ -0,0 +1,95 @@ +import argparse +import os +import numpy as np + +from infer import Infer + +from paddle3d.apis.config import Config +from paddle3d.slim import get_qat_config +from paddle3d.utils.checkpoint import load_pretrained_model + +def parse_args(): + """ + """ + parser = argparse.ArgumentParser(description='Model evaluation') + # params of training + parser.add_argument( + "--config", dest="cfg", help="The config file.", default=None, type=str) + parser.add_argument( + '--batch_size', + dest='batch_size', + help='Mini batch size of one gpu or cpu', + type=int, + default=None) + parser.add_argument( + '--model', + dest='model', + help='pretrained parameters of the model', + type=str, + default=None) + parser.add_argument( + '--num_workers', + dest='num_workers', + help='Num workers for data loader', + type=int, + default=2) + parser.add_argument( + '--quant_config', + dest='quant_config', + help='Config for quant model.', + default=None, + type=str) + + return parser.parse_args() + + +def worker_init_fn(worker_id): + np.random.seed(1024) + +def main(args): + """ + """ + if args.cfg is None: + raise RuntimeError("No configuration file specified!") + + if not os.path.exists(args.cfg): + raise RuntimeError("Config file `{}` does not exist!".format(args.cfg)) + + cfg = Config(path=args.cfg, batch_size=args.batch_size) + print(args.cfg) + if cfg.val_dataset is None: + raise RuntimeError( + 'The validation dataset is not specified in the configuration file!' + ) + elif len(cfg.val_dataset) == 0: + raise ValueError( + 'The length of validation dataset is 0. Please check if your dataset is valid!' + ) + + dic = cfg.to_dict() + batch_size = dic.pop('batch_size') + dic.update({ + 'dataloader_fn': { + 'batch_size': batch_size, + 'num_workers': args.num_workers, + 'worker_init_fn': worker_init_fn + } + }) + + if args.quant_config: + quant_config = get_qat_config(args.quant_config) + cfg.model.build_slim_model(quant_config['quant_config']) + + if args.model is not None: + load_pretrained_model(cfg.model, args.model) + dic['checkpoint'] = None + dic['resume'] = False + else: + dic['resume'] = True + + infer = Infer(**dic) + infer.infer('bev') + +if __name__ == '__main__': + args = parse_args() + main(args) diff --git a/demo/bev_demo_deploy.py b/demo/bev_demo_deploy.py new file mode 100644 index 00000000..e8955454 --- /dev/null +++ b/demo/bev_demo_deploy.py @@ -0,0 +1,174 @@ +import argparse +import numpy as np +import paddle +from paddle.inference import Config, create_predictor +from paddle3d.ops.iou3d_nms_cuda import nms_gpu +from utils import preprocess, Calibration, show_bev_with_boxes + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_file", + type=str, + help="Model filename, Specify this when your model is a combined model.", + required=True) + parser.add_argument( + "--params_file", + type=str, + help= + "Parameter filename, Specify this when your model is a combined model.", + required=True) + parser.add_argument( + '--lidar_file', type=str, help='The lidar path.', required=True) + parser.add_argument( + '--calib_file', type=str, help='The lidar path.', required=True) + parser.add_argument( + "--num_point_dim", + type=int, + default=4, + help="Dimension of a point in the lidar file.") + parser.add_argument( + "--point_cloud_range", + dest='point_cloud_range', + nargs='+', + help="Range of point cloud for voxelize operation.", + type=float, + default=None) + parser.add_argument( + "--voxel_size", + dest='voxel_size', + nargs='+', + help="Size of voxels for voxelize operation.", + type=float, + default=None) + parser.add_argument( + "--max_points_in_voxel", + type=int, + default=100, + help="Maximum number of points in a voxel.") + parser.add_argument( + "--max_voxel_num", + type=int, + default=12000, + help="Maximum number of voxels.") + parser.add_argument("--gpu_id", type=int, default=0, help="GPU card id.") + parser.add_argument( + "--use_trt", + type=int, + default=0, + help="Whether to use tensorrt to accelerate when using gpu.") + parser.add_argument( + "--trt_precision", + type=int, + default=0, + help="Precision type of tensorrt, 0: kFloat32, 1: kHalf.") + parser.add_argument( + "--trt_use_static", + type=int, + default=0, + help="Whether to load the tensorrt graph optimization from a disk path." + ) + parser.add_argument( + "--trt_static_dir", + type=str, + help="Path of a tensorrt graph optimization directory.") + parser.add_argument( + "--collect_shape_info", + type=int, + default=0, + help="Whether to collect dynamic shape before using tensorrt.") + parser.add_argument( + "--dynamic_shape_file", + type=str, + default="", + help="Path of a dynamic shape file for tensorrt.") + + return parser.parse_args() + + +def init_predictor(model_file, + params_file, + gpu_id=0, + use_trt=False, + trt_precision=0, + trt_use_static=False, + trt_static_dir=None, + collect_shape_info=False, + dynamic_shape_file=None): + config = Config(model_file, params_file) + config.enable_memory_optim() + config.enable_use_gpu(1000, gpu_id) + if use_trt: + precision_mode = paddle.inference.PrecisionType.Float32 + if trt_precision == 1: + precision_mode = paddle.inference.PrecisionType.Half + config.enable_tensorrt_engine( + workspace_size=1 << 30, + max_batch_size=1, + min_subgraph_size=10, + precision_mode=precision_mode, + use_static=trt_use_static, + use_calib_mode=False) + if collect_shape_info: + config.collect_shape_range_info(dynamic_shape_file) + else: + config.enable_tuned_tensorrt_dynamic_shape(dynamic_shape_file, True) + if trt_use_static: + config.set_optim_cache_dir(trt_static_dir) + + predictor = create_predictor(config) + return predictor + + +def run(predictor, voxels, coords, num_points_per_voxel): + input_names = predictor.get_input_names() + for i, name in enumerate(input_names): + input_tensor = predictor.get_input_handle(name) + if name == "voxels": + input_tensor.reshape(voxels.shape) + input_tensor.copy_from_cpu(voxels.copy()) + elif name == "coords": + input_tensor.reshape(coords.shape) + input_tensor.copy_from_cpu(coords.copy()) + elif name == "num_points_per_voxel": + input_tensor.reshape(num_points_per_voxel.shape) + input_tensor.copy_from_cpu(num_points_per_voxel.copy()) + + # do the inference + predictor.run() + + # get out data from output tensor + output_names = predictor.get_output_names() + for i, name in enumerate(output_names): + output_tensor = predictor.get_output_handle(name) + if i == 0: + box3d_lidar = output_tensor.copy_to_cpu() + elif i == 1: + label_preds = output_tensor.copy_to_cpu() + elif i == 2: + scores = output_tensor.copy_to_cpu() + return box3d_lidar, label_preds, scores + + +if __name__ == '__main__': + args = parse_args() + + predictor = init_predictor(args.model_file, args.params_file, args.gpu_id, + args.use_trt, args.trt_precision, + args.trt_use_static, args.trt_static_dir, + args.collect_shape_info, args.dynamic_shape_file) + voxels, coords, num_points_per_voxel = preprocess( + args.lidar_file, args.num_point_dim, args.point_cloud_range, + args.voxel_size, args.max_points_in_voxel, args.max_voxel_num) + box3d_lidar, label_preds, scores = run(predictor, voxels, coords, + num_points_per_voxel) + + scan = np.fromfile(args.lidar_file, dtype=np.float32) + pc_velo = scan.reshape((-1, 4)) + + # Obtain calibration information about Kitti + calib = Calibration(args.calib_file) + + # Plot box in lidar cloud + show_bev_with_boxes(pc_velo, box3d_lidar, scores, calib) diff --git a/demo/data/calib/000008.txt b/demo/data/calib/000008.txt new file mode 100755 index 00000000..f8a223db --- /dev/null +++ b/demo/data/calib/000008.txt @@ -0,0 +1,8 @@ +P0: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 0.000000000000e+00 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 0.000000000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 0.000000000000e+00 +P1: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 -3.875744000000e+02 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 0.000000000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 0.000000000000e+00 +P2: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 4.485728000000e+01 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 2.163791000000e-01 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 2.745884000000e-03 +P3: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 -3.395242000000e+02 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 2.199936000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 2.729905000000e-03 +R0_rect: 9.999239000000e-01 9.837760000000e-03 -7.445048000000e-03 -9.869795000000e-03 9.999421000000e-01 -4.278459000000e-03 7.402527000000e-03 4.351614000000e-03 9.999631000000e-01 +Tr_velo_to_cam: 7.533745000000e-03 -9.999714000000e-01 -6.166020000000e-04 -4.069766000000e-03 1.480249000000e-02 7.280733000000e-04 -9.998902000000e-01 -7.631618000000e-02 9.998621000000e-01 7.523790000000e-03 1.480755000000e-02 -2.717806000000e-01 +Tr_imu_to_velo: 9.999976000000e-01 7.553071000000e-04 -2.035826000000e-03 -8.086759000000e-01 -7.854027000000e-04 9.998898000000e-01 -1.482298000000e-02 3.195559000000e-01 2.024406000000e-03 1.482454000000e-02 9.998881000000e-01 -7.997231000000e-01 + diff --git a/demo/data/image_2/000008.png b/demo/data/image_2/000008.png new file mode 100755 index 00000000..4ea06065 Binary files /dev/null and b/demo/data/image_2/000008.png differ diff --git a/demo/data/velodyne/000008.bin b/demo/data/velodyne/000008.bin new file mode 100644 index 00000000..1ae36f75 Binary files /dev/null and b/demo/data/velodyne/000008.bin differ diff --git a/demo/img/bev.png b/demo/img/bev.png new file mode 100644 index 00000000..88a6bde0 Binary files /dev/null and b/demo/img/bev.png differ diff --git a/demo/img/mono.png b/demo/img/mono.png new file mode 100644 index 00000000..f0a3d8ee Binary files /dev/null and b/demo/img/mono.png differ diff --git a/demo/img/pc.png b/demo/img/pc.png new file mode 100644 index 00000000..2b101bac Binary files /dev/null and b/demo/img/pc.png differ diff --git a/demo/infer.py b/demo/infer.py new file mode 100644 index 00000000..f61686e6 --- /dev/null +++ b/demo/infer.py @@ -0,0 +1,432 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +import cv2 +import sys +import numpy as np +from collections import defaultdict +from typing import Callable, Optional, Union + +import paddle +from visualdl import LogWriter + +import paddle3d.env as env +from paddle3d.apis.checkpoint import Checkpoint, CheckpointABC +from paddle3d.apis.pipeline import training_step, validation_step +from paddle3d.apis.scheduler import Scheduler, SchedulerABC +from paddle3d.utils.logger import logger +from paddle3d.utils.shm_utils import _get_shared_memory_size_in_M +from paddle3d.utils.timer import Timer + +from paddle3d.datasets.kitti.kitti_utils import camera_record_to_object + +from utils import Calibration, show_lidar_with_boxes, total_imgpred_by_conf_to_kitti_records, \ + make_imgpts_list, draw_mono_3d, show_bev_with_boxes + + +def default_dataloader_build_fn(**kwargs) -> paddle.io.DataLoader: + """ + """ + + def _generate_loader(dataset: paddle.io.Dataset, model: paddle.nn.Layer): + args = kwargs.copy() + batch_size = args.pop('batch_size', 1) + shuffle = False if not dataset.is_train_mode else True + drop_last = args.pop('drop_last', + False if not dataset.is_train_mode else True) + + if dataset.is_train_mode: + BatchSampler = paddle.io.DistributedBatchSampler + else: + # Do eval in single device + BatchSampler = paddle.io.BatchSampler + + batch_sampler = BatchSampler( + dataset, + batch_size=batch_size, + shuffle=shuffle, + drop_last=drop_last) + + if hasattr(model, 'collate_fn'): + collate_fn = model.collate_fn + else: + collate_fn = getattr(dataset, 'collate_fn', None) + + # DataLoader do not start sub-process in Windows and Mac + # system, do not need to use shared memory + use_shared_memory = sys.platform not in ['win32', 'darwin'] + # check whether shared memory size is bigger than 1G(1024M) + if use_shared_memory: + shm_size = _get_shared_memory_size_in_M() + if shm_size is not None and shm_size < 1024.: + logger.warning("Shared memory size is less than 1G, " + "disable shared_memory in DataLoader") + use_shared_memory = False + + return paddle.io.DataLoader( + dataset=dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + use_shared_memory=use_shared_memory, + **args) + + return _generate_loader + + +def default_checkpoint_build_fn(**kwargs) -> Checkpoint: + """ + """ + kwargs = kwargs.copy() + kwargs.setdefault('save_dir', 'output') + kwargs.setdefault('keep_checkpoint_max', 5) + kwargs.setdefault('overwrite', True) + return Checkpoint(**kwargs) + + +def default_scheduler_build_fn(**kwargs) -> Scheduler: + """ + """ + kwargs = kwargs.copy() + kwargs.setdefault('log_interval', 10) + kwargs.setdefault('do_eval', False) + + if kwargs.get('train_by_epoch'): + kwargs.setdefault('save_interval', 5) + else: + kwargs.setdefault('save_interval', 1000) + + return Scheduler(**kwargs) + + +class Infer: + """ + """ + + def __init__( + self, + model: paddle.nn.Layer, + optimizer: paddle.optimizer.Optimizer, + iters: Optional[int] = None, + epochs: Optional[int] = None, + train_dataset: Optional[paddle.io.Dataset] = None, + val_dataset: Optional[paddle.io.Dataset] = None, + resume: bool = False, + # TODO: Default parameters should not use mutable objects, there is a risk + checkpoint: Union[dict, CheckpointABC] = dict(), + scheduler: Union[dict, SchedulerABC] = dict(), + dataloader_fn: Union[dict, Callable] = dict(), + amp_cfg: Optional[dict] = None): + + self.model = model + self.optimizer = optimizer + + _dataloader_build_fn = default_dataloader_build_fn( + **dataloader_fn) if isinstance(dataloader_fn, + dict) else dataloader_fn + + self.train_dataloader = _dataloader_build_fn(train_dataset, self.model) + self.eval_dataloader = _dataloader_build_fn( + val_dataset, self.model) if val_dataset else None + self.val_dataset = val_dataset + + self.resume = resume + vdl_file_name = None + self.iters_per_epoch = len(self.train_dataloader) + + if iters is None: + self.epochs = epochs + self.iters = epochs * self.iters_per_epoch + self.train_by_epoch = True + else: + self.iters = iters + self.epochs = (iters - 1) // self.iters_per_epoch + 1 + self.train_by_epoch = False + + def set_lr_scheduler_iters_per_epoch(lr_scheduler, + iters_per_epoch, + warmup_iters=0): + if isinstance(lr_scheduler, paddle.optimizer.lr.LinearWarmup): + return set_lr_scheduler_iters_per_epoch( + lr_scheduler.learning_rate, iters_per_epoch, + lr_scheduler.warmup_steps) + elif hasattr(lr_scheduler, 'learning_rate') and isinstance( + lr_scheduler.learning_rate, + paddle.optimizer.lr.LRScheduler): + return set_lr_scheduler_iters_per_epoch( + lr_scheduler.learning_rate, iters_per_epoch) + + if hasattr(lr_scheduler, 'iters_per_epoch'): + print('set lr scheduler {} iters_per_epoch={}, warmup_iters={}'.format(lr_scheduler.__class__.__name__, \ + iters_per_epoch, warmup_iters)) + lr_scheduler.iters_per_epoch = iters_per_epoch + lr_scheduler.warmup_iters = warmup_iters + + if hasattr(optimizer, '_learning_rate'): + set_lr_scheduler_iters_per_epoch(optimizer._learning_rate, + self.iters_per_epoch) + + self.cur_iter = 0 + self.cur_epoch = 0 + + if self.optimizer.__class__.__name__ == 'OneCycleAdam': + self.optimizer.before_run(max_iters=self.iters) + + self.checkpoint = default_checkpoint_build_fn( + **checkpoint) if isinstance(checkpoint, dict) else checkpoint + + if isinstance(scheduler, dict): + scheduler.setdefault('train_by_epoch', self.train_by_epoch) + scheduler.setdefault('iters_per_epoch', self.iters_per_epoch) + self.scheduler = default_scheduler_build_fn(**scheduler) + else: + self.scheduler = scheduler + + if self.checkpoint is None: + return + + if not self.checkpoint.empty: + if not resume: + raise RuntimeError( + 'The checkpoint {} is not emtpy! Set `resume=True` to continue training or use another dir as checkpoint' + .format(self.checkpoint.rootdir)) + + if self.checkpoint.meta.get( + 'train_by_epoch') != self.train_by_epoch: + raise RuntimeError( + 'Unable to resume training since the train_by_epoch is inconsistent with that saved in the checkpoint' + ) + + params_dict, opt_dict = self.checkpoint.get() + self.model.set_dict(params_dict) + self.optimizer.set_state_dict(opt_dict) + self.cur_iter = self.checkpoint.meta.get('iters') + self.cur_epoch = self.checkpoint.meta.get('epochs') + self.scheduler.step(self.cur_iter) + + logger.info( + 'Resume model from checkpoint {}, current iter set to {}'. + format(self.checkpoint.rootdir, self.cur_iter)) + vdl_file_name = self.checkpoint.meta['vdl_file_name'] + elif resume: + logger.warning( + "Attempt to restore parameters from an empty checkpoint") + + if env.local_rank == 0: + self.log_writer = LogWriter( + logdir=self.checkpoint.rootdir, file_name=vdl_file_name) + self.checkpoint.record('vdl_file_name', + os.path.basename(self.log_writer.file_name)) + self.checkpoint.record('train_by_epoch', self.train_by_epoch) + + self.scaler = None + self.amp_cfg = None + + if amp_cfg is not None: + scaler_cfg_ = dict(init_loss_scaling=2. ** 15) + scaler_cfg_.update(**amp_cfg.pop('scaler', dict())) + self.scaler = paddle.amp.GradScaler(**scaler_cfg_) + + self.amp_cfg = amp_cfg + + amp_cfg_ = copy.deepcopy(amp_cfg) + amp_cfg_.pop('enable', False) + self.model.amp_cfg_ = amp_cfg_ + logger.info( + 'Use AMP train, AMP config: {}, Scaler config: {}'.format( + amp_cfg_, scaler_cfg_)) + + def train(self): + """ + """ + + sync_bn = (getattr(self.model, 'sync_bn', False) and env.nranks > 1) + if sync_bn: + sparse_conv = False + for layer in self.model.sublayers(): + if 'sparse' in str(type(layer)): + sparse_conv = True + break + if sparse_conv: + self.model = paddle.sparse.nn.SyncBatchNorm.convert_sync_batchnorm( + self.model) + else: + self.model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( + self.model) + + model = self.model + if env.nranks > 1: + if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized( + ): + paddle.distributed.init_parallel_env() + model = paddle.DataParallel(self.model) + + losses_sum = defaultdict(float) + timer = Timer(iters=self.iters - self.cur_iter) + + while self.cur_iter < self.iters: + + for sample in self.train_dataloader: + self.cur_iter += 1 + + if self.cur_iter % self.iters_per_epoch == 1: + self.cur_epoch += 1 + + if self.cur_iter > self.iters: + break + + lr = self.optimizer.get_lr() + output = training_step( + model, + self.optimizer, + sample, + self.cur_iter, + scaler=self.scaler, + amp_cfg=self.amp_cfg) + + if isinstance(output['loss'], dict): + for k, v in output['loss'].items(): + losses_sum[k] += float(v) + + losses_sum['total_loss'] += float(output['total_loss']) + + timer.step() + status = self.scheduler.step() + + if status.do_log and env.local_rank == 0: + + loss_log = '' + + self.log_writer.add_scalar( + tag='Training/learning_rate', + value=lr, + step=self.cur_iter) + + for k, v in losses_sum.items(): + loss_val = v / self.scheduler.log_interval + loss_log += ', {}={:.6f}'.format(k, loss_val) + self.log_writer.add_scalar( + tag='Training/' + k, + value=loss_val, + step=self.cur_iter) + + logger.info( + '[TRAIN] epoch={}/{}, iter={}/{} {}, lr={:.6f} | ETA {}' + .format(self.cur_epoch, self.epochs, self.cur_iter, + self.iters, loss_log, lr, timer.eta)) + + losses_sum.clear() + + if status.do_eval and env.local_rank == 0: + # TODO: whether to save a checkpoint based on the metric + metrics = self.evaluate() + for k, v in metrics.items(): + if not isinstance(v, paddle.Tensor) or v.numel() != 1: + continue + + self.log_writer.add_scalar( + tag='Evaluation/{}'.format(k), + value=float(v), + step=self.cur_iter) + + if status.save_checkpoint and env.local_rank == 0: + if self.train_by_epoch: + tag = 'epoch_{}'.format(self.cur_epoch) + else: + tag = 'iter_{}'.format(self.cur_iter) + + self.checkpoint.push( + tag=tag, + params_dict=self.model.state_dict(), + opt_dict=self.optimizer.state_dict(), + verbose=True) + + self.checkpoint.record('iters', self.cur_iter) + self.checkpoint.record('epochs', self.cur_epoch) + + logger.info('Training is complete.') + + if env.local_rank == 0: + if self.train_by_epoch: + tag = 'epoch_{}'.format(self.epochs) + else: + tag = 'iter_{}'.format(self.iters) + + if not self.checkpoint.have(tag): + self.checkpoint.push( + tag=tag, + params_dict=self.model.state_dict(), + opt_dict=self.optimizer.state_dict(), + verbose=True) + + self.checkpoint.record('iters', self.iters) + self.checkpoint.record('epochs', self.epochs) + + def infer(self, mode) -> float: + """ + """ + sync_bn = (getattr(self.model, 'sync_bn', False) and env.nranks > 1) + if sync_bn: + sparse_conv = False + for layer in self.model.sublayers(): + if 'sparse' in str(type(layer)): + sparse_conv = True + break + if sparse_conv: + self.model = paddle.sparse.nn.SyncBatchNorm.convert_sync_batchnorm( + self.model) + else: + self.model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( + self.model) + + if self.val_dataset is None: + raise RuntimeError('No evaluation dataset specified!') + msg = 'evaluate on validate dataset' + metric_obj = self.val_dataset.metric + + for idx, sample in logger.enumerate(self.eval_dataloader, msg=msg): + results = validation_step(self.model, sample) + + if mode == 'pcd': + for result in results: + scan = np.fromfile(result['path'], dtype=np.float32) + pc_velo = scan.reshape((-1, 4)) + # Obtain calibration information about Kitti + calib = Calibration(result['path'].replace('velodyne', 'calib').replace('bin', 'txt')) + # Plot box in lidar cloud + # show_lidar_with_boxes(pc_velo, result['bboxes_3d'], result['confidences'], calib) + show_lidar_with_boxes(pc_velo, result['bboxes_3d'], result['confidences'], calib) + + if mode == 'image': + for result in results: + kitti_records = total_imgpred_by_conf_to_kitti_records(result, 0.3) + bboxes_2d, bboxes_3d, labels = camera_record_to_object(kitti_records) + # read origin image + img_origin = cv2.imread(result['path']) + # to 8 points on image + K = np.array(result['meta']['camera_intrinsic']) + imgpts_list = make_imgpts_list(bboxes_3d, K) + # draw smoke result to photo + draw_mono_3d(img_origin, imgpts_list) + + if mode == 'bev': + for result in results: + scan = np.fromfile(result['path'], dtype=np.float32) + pc_velo = scan.reshape((-1, 4)) + # Obtain calibration information about Kitti + calib = Calibration(result['path'].replace('velodyne', 'calib').replace('bin', 'txt')) + # Plot box in lidar cloud (bev) + show_bev_with_boxes(pc_velo, result['bboxes_3d'], result['confidences'], calib) + diff --git a/demo/mono_demo.py b/demo/mono_demo.py new file mode 100644 index 00000000..63c4cd0f --- /dev/null +++ b/demo/mono_demo.py @@ -0,0 +1,99 @@ +import argparse +import os +import cv2 +import numpy as np + +from infer import Infer +from utils import make_imgpts_list, draw_mono_3d, total_imgpred_by_conf_to_kitti_records + +from paddle3d.apis.config import Config +from paddle3d.slim import get_qat_config +from paddle3d.utils.checkpoint import load_pretrained_model +from paddle3d.datasets.kitti.kitti_utils import camera_record_to_object + +def parse_args(): + """ + """ + parser = argparse.ArgumentParser(description='Model evaluation') + # params of training + parser.add_argument( + "--config", dest="cfg", help="The config file.", default=None, type=str) + parser.add_argument( + '--batch_size', + dest='batch_size', + help='Mini batch size of one gpu or cpu', + type=int, + default=None) + parser.add_argument( + '--model', + dest='model', + help='pretrained parameters of the model', + type=str, + default=None) + parser.add_argument( + '--num_workers', + dest='num_workers', + help='Num workers for data loader', + type=int, + default=2) + parser.add_argument( + '--quant_config', + dest='quant_config', + help='Config for quant model.', + default=None, + type=str) + + return parser.parse_args() + + +def worker_init_fn(worker_id): + np.random.seed(1024) + +def main(args): + """ + """ + if args.cfg is None: + raise RuntimeError("No configuration file specified!") + + if not os.path.exists(args.cfg): + raise RuntimeError("Config file `{}` does not exist!".format(args.cfg)) + + cfg = Config(path=args.cfg, batch_size=args.batch_size) + print(args.cfg) + if cfg.val_dataset is None: + raise RuntimeError( + 'The validation dataset is not specified in the configuration file!' + ) + elif len(cfg.val_dataset) == 0: + raise ValueError( + 'The length of validation dataset is 0. Please check if your dataset is valid!' + ) + + dic = cfg.to_dict() + batch_size = dic.pop('batch_size') + dic.update({ + 'dataloader_fn': { + 'batch_size': batch_size, + 'num_workers': args.num_workers, + 'worker_init_fn': worker_init_fn + } + }) + + if args.quant_config: + quant_config = get_qat_config(args.quant_config) + cfg.model.build_slim_model(quant_config['quant_config']) + + if args.model is not None: + load_pretrained_model(cfg.model, args.model) + dic['checkpoint'] = None + dic['resume'] = False + else: + dic['resume'] = True + + infer = Infer(**dic) + infer.infer('image') + + +if __name__ == '__main__': + args = parse_args() + main(args) diff --git a/demo/mono_demo_deploy.py b/demo/mono_demo_deploy.py new file mode 100644 index 00000000..a33205ef --- /dev/null +++ b/demo/mono_demo_deploy.py @@ -0,0 +1,126 @@ +import argparse + +import cv2 +import numpy as np + +from paddle.inference import Config, PrecisionType, create_predictor +from paddle3d.datasets.kitti.kitti_utils import camera_record_to_object +from utils import get_img, get_ratio, total_pred_by_conf_to_kitti_records, make_imgpts_list, draw_mono_3d + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_file", + type=str, + help="Model filename, Specify this when your model is a combined model.", + required=True) + parser.add_argument( + "--params_file", + type=str, + help= + "Parameter filename, Specify this when your model is a combined model.", + required=True) + parser.add_argument('--image', + dest='image', + help='The image path', + type=str, + required=True) + parser.add_argument("--use_gpu", + action='store_true', + help="Whether use gpu.") + parser.add_argument("--use_trt", + action='store_true', + help="Whether use trt.") + parser.add_argument( + "--collect_dynamic_shape_info", + action='store_true', + help="Whether to collect dynamic shape before using tensorrt.") + parser.add_argument("--dynamic_shape_file", + dest='dynamic_shape_file', + help='The image path', + type=str, + default="dynamic_shape_info.txt") + return parser.parse_args() + + +def init_predictor(args): + config = Config(args.model_file, args.params_file) + config.enable_memory_optim() + if args.use_gpu: + config.enable_use_gpu(1000, 0) + else: + # If not specific mkldnn, you can set the blas thread. + # The thread num should not be greater than the number of cores in the CPU. + config.set_cpu_math_library_num_threads(4) + config.enable_mkldnn() + + if args.collect_dynamic_shape_info: + config.collect_shape_range_info(args.dynamic_shape_file) + elif args.use_trt: + allow_build_at_runtime = True + config.enable_tuned_tensorrt_dynamic_shape(args.dynamic_shape_file, + allow_build_at_runtime) + + config.enable_tensorrt_engine(workspace_size=1 << 20, + max_batch_size=1, + min_subgraph_size=3, + precision_mode=PrecisionType.Float32) + + predictor = create_predictor(config) + return predictor + + +def run(predictor, image, K, down_ratio): + # copy img data to input tensor + input_names = predictor.get_input_names() + for i, name in enumerate(input_names): + input_tensor = predictor.get_input_handle(name) + if name == "images": + input_tensor.reshape(image.shape) + input_tensor.copy_from_cpu(image.copy()) + elif name == "trans_cam_to_img": + input_tensor.reshape(K.shape) + input_tensor.copy_from_cpu(K.copy()) + elif name == "down_ratios": + input_tensor.reshape(down_ratio.shape) + input_tensor.copy_from_cpu(down_ratio.copy()) + + # do the inference + predictor.run() + + results = [] + # get out data from output tensor + output_names = predictor.get_output_names() + for i, name in enumerate(output_names): + output_tensor = predictor.get_output_handle(name) + output_data = output_tensor.copy_to_cpu() + results.append(output_data) + + return results + + +if __name__ == '__main__': + args = parse_args() + pred = init_predictor(args) + # Listed below are camera intrinsic parameter of the kitti dataset + # If the model is trained on other datasets, please replace the relevant data + K = np.array([[[721.53771973, 0., 609.55932617], + [0., 721.53771973, 172.85400391], [0, 0, 1]]], np.float32) + + img, ori_img_size, output_size = get_img(args.image) + ratio = get_ratio(ori_img_size, output_size) + + results = run(pred, img, K, ratio) + + total_pred = results[0] + print(total_pred) + # convert pred to bboxes_2d, bboxes_3d + kitti_records = total_pred_by_conf_to_kitti_records(total_pred, conf=0.5) + bboxes_2d, bboxes_3d, labels = camera_record_to_object(kitti_records) + # read origin image + img_origin = cv2.imread(args.image) + # to 8 points on image + imgpts_list = make_imgpts_list(bboxes_3d, K[0]) + # draw smoke result to photo + draw_mono_3d(img_origin, imgpts_list) diff --git a/demo/pcd_demo.py b/demo/pcd_demo.py new file mode 100644 index 00000000..4659c5d8 --- /dev/null +++ b/demo/pcd_demo.py @@ -0,0 +1,98 @@ +import argparse +import os +import cv2 +import numpy as np + +from infer import Infer +from utils import Calibration, show_lidar_with_boxes + +from paddle3d.apis.config import Config +from paddle3d.slim import get_qat_config +from paddle3d.utils.checkpoint import load_pretrained_model +from paddle3d.datasets.kitti.kitti_utils import camera_record_to_object + +def parse_args(): + """ + """ + parser = argparse.ArgumentParser(description='Model evaluation') + # params of training + parser.add_argument( + "--config", dest="cfg", help="The config file.", default=None, type=str) + parser.add_argument( + '--batch_size', + dest='batch_size', + help='Mini batch size of one gpu or cpu', + type=int, + default=None) + parser.add_argument( + '--model', + dest='model', + help='pretrained parameters of the model', + type=str, + default=None) + parser.add_argument( + '--num_workers', + dest='num_workers', + help='Num workers for data loader', + type=int, + default=2) + parser.add_argument( + '--quant_config', + dest='quant_config', + help='Config for quant model.', + default=None, + type=str) + + return parser.parse_args() + + +def worker_init_fn(worker_id): + np.random.seed(1024) + +def main(args): + """ + """ + if args.cfg is None: + raise RuntimeError("No configuration file specified!") + + if not os.path.exists(args.cfg): + raise RuntimeError("Config file `{}` does not exist!".format(args.cfg)) + + cfg = Config(path=args.cfg, batch_size=args.batch_size) + print(args.cfg) + if cfg.val_dataset is None: + raise RuntimeError( + 'The validation dataset is not specified in the configuration file!' + ) + elif len(cfg.val_dataset) == 0: + raise ValueError( + 'The length of validation dataset is 0. Please check if your dataset is valid!' + ) + + dic = cfg.to_dict() + batch_size = dic.pop('batch_size') + dic.update({ + 'dataloader_fn': { + 'batch_size': batch_size, + 'num_workers': args.num_workers, + 'worker_init_fn': worker_init_fn + } + }) + + if args.quant_config: + quant_config = get_qat_config(args.quant_config) + cfg.model.build_slim_model(quant_config['quant_config']) + + if args.model is not None: + load_pretrained_model(cfg.model, args.model) + dic['checkpoint'] = None + dic['resume'] = False + else: + dic['resume'] = True + + infer = Infer(**dic) + infer.infer('pcd') + +if __name__ == '__main__': + args = parse_args() + main(args) diff --git a/demo/pcd_demo_deploy.py b/demo/pcd_demo_deploy.py new file mode 100644 index 00000000..cf018003 --- /dev/null +++ b/demo/pcd_demo_deploy.py @@ -0,0 +1,174 @@ +import argparse +import numpy as np +import paddle +from paddle.inference import Config, create_predictor +from paddle3d.ops.iou3d_nms_cuda import nms_gpu +from utils import preprocess, Calibration, show_lidar_with_boxes + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_file", + type=str, + help="Model filename, Specify this when your model is a combined model.", + required=True) + parser.add_argument( + "--params_file", + type=str, + help= + "Parameter filename, Specify this when your model is a combined model.", + required=True) + parser.add_argument( + '--lidar_file', type=str, help='The lidar path.', required=True) + parser.add_argument( + '--calib_file', type=str, help='The lidar path.', required=True) + parser.add_argument( + "--num_point_dim", + type=int, + default=4, + help="Dimension of a point in the lidar file.") + parser.add_argument( + "--point_cloud_range", + dest='point_cloud_range', + nargs='+', + help="Range of point cloud for voxelize operation.", + type=float, + default=None) + parser.add_argument( + "--voxel_size", + dest='voxel_size', + nargs='+', + help="Size of voxels for voxelize operation.", + type=float, + default=None) + parser.add_argument( + "--max_points_in_voxel", + type=int, + default=100, + help="Maximum number of points in a voxel.") + parser.add_argument( + "--max_voxel_num", + type=int, + default=12000, + help="Maximum number of voxels.") + parser.add_argument("--gpu_id", type=int, default=0, help="GPU card id.") + parser.add_argument( + "--use_trt", + type=int, + default=0, + help="Whether to use tensorrt to accelerate when using gpu.") + parser.add_argument( + "--trt_precision", + type=int, + default=0, + help="Precision type of tensorrt, 0: kFloat32, 1: kHalf.") + parser.add_argument( + "--trt_use_static", + type=int, + default=0, + help="Whether to load the tensorrt graph optimization from a disk path." + ) + parser.add_argument( + "--trt_static_dir", + type=str, + help="Path of a tensorrt graph optimization directory.") + parser.add_argument( + "--collect_shape_info", + type=int, + default=0, + help="Whether to collect dynamic shape before using tensorrt.") + parser.add_argument( + "--dynamic_shape_file", + type=str, + default="", + help="Path of a dynamic shape file for tensorrt.") + + return parser.parse_args() + + +def init_predictor(model_file, + params_file, + gpu_id=0, + use_trt=False, + trt_precision=0, + trt_use_static=False, + trt_static_dir=None, + collect_shape_info=False, + dynamic_shape_file=None): + config = Config(model_file, params_file) + config.enable_memory_optim() + config.enable_use_gpu(1000, gpu_id) + if use_trt: + precision_mode = paddle.inference.PrecisionType.Float32 + if trt_precision == 1: + precision_mode = paddle.inference.PrecisionType.Half + config.enable_tensorrt_engine( + workspace_size=1 << 30, + max_batch_size=1, + min_subgraph_size=10, + precision_mode=precision_mode, + use_static=trt_use_static, + use_calib_mode=False) + if collect_shape_info: + config.collect_shape_range_info(dynamic_shape_file) + else: + config.enable_tuned_tensorrt_dynamic_shape(dynamic_shape_file, True) + if trt_use_static: + config.set_optim_cache_dir(trt_static_dir) + + predictor = create_predictor(config) + return predictor + + +def run(predictor, voxels, coords, num_points_per_voxel): + input_names = predictor.get_input_names() + for i, name in enumerate(input_names): + input_tensor = predictor.get_input_handle(name) + if name == "voxels": + input_tensor.reshape(voxels.shape) + input_tensor.copy_from_cpu(voxels.copy()) + elif name == "coords": + input_tensor.reshape(coords.shape) + input_tensor.copy_from_cpu(coords.copy()) + elif name == "num_points_per_voxel": + input_tensor.reshape(num_points_per_voxel.shape) + input_tensor.copy_from_cpu(num_points_per_voxel.copy()) + + # do the inference + predictor.run() + + # get out data from output tensor + output_names = predictor.get_output_names() + for i, name in enumerate(output_names): + output_tensor = predictor.get_output_handle(name) + if i == 0: + box3d_lidar = output_tensor.copy_to_cpu() + elif i == 1: + label_preds = output_tensor.copy_to_cpu() + elif i == 2: + scores = output_tensor.copy_to_cpu() + return box3d_lidar, label_preds, scores + + +if __name__ == '__main__': + args = parse_args() + + predictor = init_predictor(args.model_file, args.params_file, args.gpu_id, + args.use_trt, args.trt_precision, + args.trt_use_static, args.trt_static_dir, + args.collect_shape_info, args.dynamic_shape_file) + voxels, coords, num_points_per_voxel = preprocess( + args.lidar_file, args.num_point_dim, args.point_cloud_range, + args.voxel_size, args.max_points_in_voxel, args.max_voxel_num) + box3d_lidar, label_preds, scores = run(predictor, voxels, coords, + num_points_per_voxel) + + scan = np.fromfile(args.lidar_file, dtype=np.float32) + pc_velo = scan.reshape((-1, 4)) + + # Obtain calibration information about Kitti + calib = Calibration(args.calib_file) + + # Plot box in lidar cloud + show_lidar_with_boxes(pc_velo, box3d_lidar, scores, calib) diff --git a/demo/readme.md b/demo/readme.md new file mode 100644 index 00000000..ef021786 --- /dev/null +++ b/demo/readme.md @@ -0,0 +1,80 @@ +We use `mayavi` for lidar points visualization +``` +conda create -n pp3d python=3.7 -y +pip install paddlepaddle-gpu==2.4.1 -i https://mirror.baidu.com/pypi/simple +pip install -r requirements.txt +pip install -e . +pip install vtk==8.1.2 +pip install mayavi==4.7.4 +pip install PyQt5 +``` +--- +## Single-Frame Visualizaion +### For mono-image +``` +cd demo/ +python mono_demo_deploy.py \ + --model_file model/smoke.pdmodel \ + --params_file model/smoke.pdiparams \ + --image data/image_2/000008.png +``` +![](img/mono.png) +### For lidar-points +``` +cd demo/ +python pcd_demo_deploy.py \ + --model_file model/pointpillars.pdmodel \ + --params_file model/pointpillars.pdiparams \ + --lidar_file data/velodyne/000008.bin \ + --calib_file data/calib/000008.txt \ + --point_cloud_range 0 -39.68 -3 69.12 39.68 1 \ + --voxel_size .16 .16 4 \ + --max_points_in_voxel 32 \ + --max_voxel_num 40000 +``` +![](img/pc.png) +### For bev-view +``` +cd demo/ +python bev_demo_deploy.py \ + --model_file model/pointpillars.pdmodel \ + --params_file model/pointpillars.pdiparams \ + --lidar_file data/velodyne/000008.bin \ + --calib_file data/calib/000008.txt \ + --point_cloud_range 0 -39.68 -3 69.12 39.68 1 \ + --voxel_size .16 .16 4 \ + --max_points_in_voxel 32 \ + --max_voxel_num 40000 +``` +![](img/bev.png) +## Multi-Frame Visualizaion +### For mono-image +``` +python demo/mono_demo.py \ + --config configs/smoke/smoke_dla34_no_dcn_kitti.yml \ + --model demo/smoke.pdparams \ + --batch_size 1 +``` +### For lidar-points +``` +python demo/pcd_demo.py \ + --config configs/pointpillars/pointpillars_xyres16_kitti_car.yml \ + --model demo/pointpillars.pdparams \ + --batch_size 1 +``` +### For bev-view +``` +python demo/bev_demo.py \ + --config configs/pointpillars/pointpillars_xyres16_kitti_car.yml \ + --model demo/pointpillars.pdparams \ + --batch_size 1 +``` + +--- +if u encounter the following problems: + +`qt.qpa.plugin: Could not load the Qt Platform plugin 'xcb' in ..` + +[ref1](https://blog.csdn.net/qq_39938666/article/details/120452028?spm=1001.2101.3001.6650.2&utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7ERate-2-120452028-blog-112303826.pc_relevant_3mothn_strategy_recovery&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7ERate-2-120452028-blog-112303826.pc_relevant_3mothn_strategy_recovery&utm_relevant_index=3) + +[ref2](https://blog.csdn.net/weixin_41794514/article/details/128578166?spm=1001.2101.3001.6650.3&utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7EYuanLiJiHua%7EPosition-3-128578166-blog-119480436.pc_relevant_landingrelevant&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7EYuanLiJiHua%7EPosition-3-128578166-blog-119480436.pc_relevant_landingrelevant) \ No newline at end of file diff --git a/demo/utils.py b/demo/utils.py new file mode 100644 index 00000000..8b7739fc --- /dev/null +++ b/demo/utils.py @@ -0,0 +1,628 @@ +import cv2 +import numba +import numpy as np +import mayavi.mlab as mlab +# import open3d as o3d +# import matplotlib.pyplot as plt + + +from paddle3d.transforms.target_generator import encode_label + + +class Calibration(object): + ''' Calibration matrices and utils + 3d XYZ in