Skip to content
This repository has been archived by the owner on Jun 3, 2020. It is now read-only.

Commit

Permalink
Merge pull request #140 from Oslandia/aerial_gt_labels
Browse files Browse the repository at this point in the history 
Aerial ground truth labels
  • Loading branch information
@delhomer
delhomer authored Jul 1, 2019
2 parents 074fa37 + 07c1913 commit d97f890
Show file tree
Hide file tree
Showing 109 changed files with 743 additions and 1,478 deletions.
5 changes: 3 additions & 2 deletions deeposlandia/datasets/aerial.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,10 @@ def _preprocess_for_training(self, image_filename, output_dir, nb_images):
image_filename.split("/")[-1], raw_img_width, raw_img_height
)

label_filename = image_filename.replace("images", "gt")
label_filename = image_filename.replace("images", "labels")
label_raster = gdal.Open(label_filename)
labels = label_raster.ReadAsArray()
labels = np.swapaxes(labels, 0, 1)

nb_attempts = 0
image_counter = 0
Expand All @@ -101,7 +102,7 @@ def _preprocess_for_training(self, image_filename, output_dir, nb_images):
label_dict = utils.build_labels(
mask, range(self.get_nb_labels()), "aerial"
)
labelled_image = utils.build_image_from_config(mask, self.labels)
labelled_image = Image.fromarray(mask)
if np.unique(mask).shape[0] > 1:
tiled_results = self._serialize(
tile_image,
Expand Down
195 changes: 99 additions & 96 deletions examples/1a_mapillary-dataset-presentation.ipynb
View file

Large diffs are not rendered by default.

1,643 changes: 403 additions & 1,240 deletions examples/1b_mapillary-label-analysis.ipynb
View file

Large diffs are not rendered by default.

103 changes: 66 additions & 37 deletions examples/1c_shape-dataset-presentation.ipynb
View file

Large diffs are not rendered by default.

153 changes: 103 additions & 50 deletions examples/1d_aerial-dataset-presentation.ipynb
View file

Large diffs are not rendered by default.

118 changes: 67 additions & 51 deletions examples/1e_tanzania-dataset-presentation.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -45,6 +45,23 @@
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"env: DEEPOSL_CONFIG=../config.ini\n"
]
}
],
"source": [
"env DEEPOSL_CONFIG=../config.ini"
]
},
{
"cell_type": "code",
"execution_count": 3,
Expand Down Expand Up @@ -84,21 +101,21 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"In order to do so, 13 annotated high-resolution images was released by during the Tanzania challenge, along with 9 images without annotations release for evaluation purpose. The image sizes come from 17k x 42k to 51k x 51k pixels, and the resolution is between 6 and 8 centimeters per pixel."
"In order to do so, 17 annotated high-resolution images was released by during the Tanzania challenge, along with 13 images without annotations release for evaluation purpose. The image sizes come from 17k x 42k to 51k x 51k pixels, and the resolution is between 6 and 8 centimeters per pixel."
]
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of training images: 11\n",
"Number of validation images: 2\n",
"Number of testing images: 9\n"
"Number of training images: 10\n",
"Number of validation images: 3\n",
"Number of testing images: 4\n"
]
}
],
Expand All @@ -117,23 +134,23 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of training images: 570\n",
"Number of validation images: 378\n",
"Number of testing images: 0\n"
"Number of training images: 10728\n",
"Number of validation images: 3296\n",
"Number of testing images: 172\n"
]
}
],
"source": [
"print(\"Number of training images: {}\".format(len(os.listdir(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE)+\"_full\", \"training\", \"images\")))))\n",
"print(\"Number of validation images: {}\".format(len(os.listdir(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE)+\"_full\", \"validation\", \"images\")))))\n",
"print(\"Number of testing images: {}\".format(len(os.listdir(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE)+\"_full\", \"testing\", \"images\")))))"
"print(\"Number of training images: {}\".format(len(os.listdir(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE), \"training\", \"images\")))))\n",
"print(\"Number of validation images: {}\".format(len(os.listdir(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE), \"validation\", \"images\")))))\n",
"print(\"Number of testing images: {}\".format(len(os.listdir(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE), \"testing\", \"images\")))))"
]
},
{
Expand All @@ -145,21 +162,21 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"def show_example_image(image_filename):\n",
" im = Image.open(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE)+\"_full\", \"training\", \"images\", image_filename))\n",
" lab = Image.open(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE)+\"_full\", \"training\", \"labels\", image_filename))\n",
" im = Image.open(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE), \"training\", \"images\", image_filename))\n",
" lab = Image.open(os.path.join(DATAPATH, DATASET, \"preprocessed\", str(IMG_SIZE), \"training\", \"labels\", image_filename))\n",
" f, ax = plt.subplots(1, 2, figsize=(12, 6))\n",
" ax[0].imshow(im)\n",
" ax[1].imshow(lab)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 10,
"metadata": {},
"outputs": [
{
Expand All @@ -181,7 +198,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 11,
"metadata": {},
"outputs": [
{
Expand All @@ -203,7 +220,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"outputs": [
{
Expand All @@ -225,7 +242,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 13,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -261,12 +278,12 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"input_folder = utils.prepare_input_folder(DATAPATH, DATASET)\n",
"preprocess_folder = utils.prepare_preprocessed_folder(DATAPATH, DATASET, IMG_SIZE, \"full\")"
"preprocess_folder = utils.prepare_preprocessed_folder(DATAPATH, DATASET, IMG_SIZE)"
]
},
{
Expand Down Expand Up @@ -313,35 +330,34 @@
"name": "stderr",
"output_type": "stream",
"text": [
"2019-01-10 18:50:54,869 :: INFO :: tanzania :: populate : Getting 10 images to preprocess...\n",
"2019-01-10 18:50:54,872 :: INFO :: tanzania :: populate : ['../data/tanzania/input/training/images/grid_001.tif', '../data/tanzania/input/training/images/grid_022.tif', '../data/tanzania/input/training/images/grid_023.tif', '../data/tanzania/input/training/images/grid_028.tif', '../data/tanzania/input/training/images/grid_029.tif', '../data/tanzania/input/training/images/grid_035.tif', '../data/tanzania/input/training/images/grid_036.tif', '../data/tanzania/input/training/images/grid_042.tif', '../data/tanzania/input/training/images/grid_043.tif', '../data/tanzania/input/training/images/grid_049.tif']\n",
"2019-01-10 18:50:54,911 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:50:54,914 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:50:54,922 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_023.tif\n",
"2019-01-10 18:50:54,924 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_022.tif\n",
"2019-01-10 18:50:54,914 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:50:54,919 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:50:54,942 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_028.tif\n",
"2019-01-10 18:50:54,943 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_001.tif\n",
"2019-01-10 18:51:33,466 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:51:33,478 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_029.tif\n",
"2019-01-10 18:51:37,314 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:51:37,325 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_035.tif\n",
"2019-01-10 18:51:39,895 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:51:39,905 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_036.tif\n",
"2019-01-10 18:52:20,712 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:52:20,723 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_042.tif\n",
"2019-01-10 18:52:25,844 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:52:25,846 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_043.tif\n",
"2019-01-10 18:52:33,777 :: INFO :: tanzania :: _preprocess : Raw image size: 3840, 3840\n",
"2019-01-10 18:52:33,785 :: INFO :: tanzania :: _preprocess : Image filename: ../data/tanzania/input/training/images/grid_049.tif\n",
"2019-01-10 18:53:11,541 :: INFO :: tanzania :: populate : Saved 1000 images in the preprocessed dataset.\n"
"2019-05-28 11:17:13,734 :: INFO :: __init__ :: populate : Getting 10 images to preprocess...\n",
"2019-05-28 11:17:14,683 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_001.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:21,234 :: INFO :: tanzania :: _preprocess_for_training : Generate 13 images after 4 attempts.\n",
"2019-05-28 11:17:21,924 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_022.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:25,670 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:26,315 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_023.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:30,313 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:30,969 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_028.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:33,569 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:34,200 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_029.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:43,421 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:44,068 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_035.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:49,314 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:49,966 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_036.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:55,230 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:55,884 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_042.tif, size: (3840, 3840)\n",
"2019-05-28 11:17:58,193 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:17:58,843 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_043.tif, size: (3840, 3840)\n",
"2019-05-28 11:18:02,958 :: INFO :: tanzania :: _preprocess_for_training : Generate 13 images after 11 attempts.\n",
"2019-05-28 11:18:03,620 :: INFO :: tanzania :: _preprocess_for_training : Image filename: grid_049.tif, size: (3840, 3840)\n",
"2019-05-28 11:18:04,938 :: INFO :: tanzania :: _preprocess_for_training : Generate 1 images after 20 attempts.\n",
"2019-05-28 11:18:04,943 :: INFO :: __init__ :: populate : Saved 34 images in the preprocessed dataset.\n"
]
},
{
"data": {
"text/plain": [
"1000"
"34"
]
},
"execution_count": 16,
Expand All @@ -365,14 +381,14 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2019-01-10 18:53:42,717 :: INFO :: __init__ :: load : The dataset has been loaded from ../data/tanzania/preprocessed/384_full/training.json\n"
"2019-05-28 11:18:43,942 :: INFO :: __init__ :: load : The dataset has been loaded from ../data/tanzania/preprocessed/384/training.json\n"
]
},
{
Expand All @@ -381,7 +397,7 @@
"50"
]
},
"execution_count": 18,
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -400,7 +416,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 18,
"metadata": {},
"outputs": [
{
Expand All @@ -409,7 +425,7 @@
"array([1. , 0.64, 0.16, 0.14])"
]
},
"execution_count": 19,
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
Expand Down
Binary file removed tests/data/aerial/input/training/gt/aerial_sample.tif
View file
Binary file not shown.
Binary file modified tests/data/aerial/input/training/images/aerial_sample.tif
View file
Binary file not shown.
Binary file added tests/data/aerial/input/training/labels/aerial_sample.tif
View file
Binary file not shown.
Loading

0 comments on commit d97f890

Please sign in to comment.