Dropout layer

CMakeLists.txt

@@ -51,6 +51,8 @@ add_library(neural-fortran
   src/nf/nf_reshape_layer_submodule.f90
   src/nf/io/nf_io_binary.f90
   src/nf/io/nf_io_binary_submodule.f90
+  src/nf/nf_dropout_layer.f90
+  src/nf/nf_dropout_layer_submodule.f90
 )
 
 target_link_libraries(neural-fortran PRIVATE)

README.md

@@ -31,6 +31,7 @@ Read the paper [here](https://arxiv.org/abs/1902.06714).
 |------------|------------------|------------------------|----------------------|--------------|---------------|
 | Input | `input` | n/a | 1, 3 | n/a | n/a |
 | Dense (fully-connected) | `dense` | `input1d`, `flatten` | 1 | ✅ | ✅ |
+| Dropout | `dropout` | Any | 1 | ✅ | ✅ |
 | Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅(*) |
 | Max-pooling (2-d) | `maxpool2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅ |
 | Flatten | `flatten` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 1 | ✅ | ✅ |

fpm.toml

@@ -1,6 +1,9 @@
 name = "neural-fortran"
-version = "0.18.0"
+version = "0.19.0"
 license = "MIT"
 author = "Milan Curcic"
 maintainer = "[email protected]"
-copyright = "Copyright 2018-2024, neural-fortran contributors"
+copyright = "Copyright 2018-2025, neural-fortran contributors"
+
+[preprocess]
+[preprocess.cpp]

src/nf.f90

@@ -3,7 +3,7 @@ module nf
   use nf_datasets_mnist, only: label_digits, load_mnist
   use nf_layer, only: layer
   use nf_layer_constructors, only: &
-    conv2d, dense, flatten, input, maxpool2d, reshape
+    conv2d, dense, dropout, flatten, input, maxpool2d, reshape
   use nf_loss, only: mse, quadratic
   use nf_metrics, only: corr, maxabs
   use nf_network, only: network

src/nf/nf_dropout_layer.f90

@@ -0,0 +1,82 @@
+module nf_dropout_layer
+
+  !! This module provides the concrete dropout layer type.
+  !! It is used internally by the layer type.
+  !! It is not intended to be used directly by the user.
+
+  use nf_base_layer, only: base_layer
+
+  implicit none
+
+  private
+  public :: dropout_layer
+
+  type, extends(base_layer) :: dropout_layer
+    !! Concrete implementation of a dropout layer type
+
+    integer :: input_size = 0
+
+    real, allocatable :: output(:)
+    real, allocatable :: gradient(:)
+    real, allocatable :: mask(:) ! binary mask for dropout
+
+    real :: dropout_rate ! probability of dropping a neuron
+    real :: scale ! scale factor to preserve the input sum
+    logical :: training = .true.
+
+  contains
+
+    procedure :: backward
+    procedure :: forward
+    procedure :: init
+
+  end type dropout_layer
+
+  interface dropout_layer
+    module function dropout_layer_cons(rate) &
+      result(res)
+      !! This function returns the `dropout_layer` instance.
+      real, intent(in) :: rate
+        !! Dropout rate
+      type(dropout_layer) :: res
+        !! dropout_layer instance
+    end function dropout_layer_cons
+  end interface dropout_layer
+
+  interface
+
+    pure module subroutine backward(self, input, gradient)
+      !! Apply the backward gradient descent pass.
+      !! Only weight and bias gradients are updated in this subroutine,
+      !! while the weights and biases themselves are untouched.
+      class(dropout_layer), intent(in out) :: self
+        !! Dropout layer instance
+      real, intent(in) :: input(:)
+        !! Input from the previous layer
+      real, intent(in) :: gradient(:)
+        !! Gradient from the next layer
+    end subroutine backward
+
+    module subroutine forward(self, input)
+      !! Propagate forward the layer.
+      !! Calling this subroutine updates the values of a few data components
+      !! of `dropout_layer` that are needed for the backward pass.
+      class(dropout_layer), intent(in out) :: self
+        !! Dense layer instance
+      real, intent(in) :: input(:)
+        !! Input from the previous layer
+    end subroutine forward
+
+    module subroutine init(self, input_shape)
+      !! Initialize the layer data structures.
+      !!
+      !! This is a deferred procedure from the `base_layer` abstract type.
+      class(dropout_layer), intent(in out) :: self
+        !! Dropout layer instance
+      integer, intent(in) :: input_shape(:)
+        !! Shape of the input layer
+    end subroutine init
+
+  end interface
+
+end module nf_dropout_layer

src/nf/nf_dropout_layer_submodule.f90

@@ -0,0 +1,65 @@
+submodule (nf_dropout_layer) nf_dropout_layer_submodule
+  !! This submodule implements the procedures defined in the
+  !! nf_dropout_layer module.
+
+contains
+
+  module function dropout_layer_cons(rate) result(res)
+    real, intent(in) :: rate
+    type(dropout_layer) :: res
+
+    ! Initialize dropout rate
+    res % dropout_rate = rate
+  end function dropout_layer_cons
+
+
+  module subroutine init(self, input_shape)
+    class(dropout_layer), intent(in out) :: self
+    integer, intent(in) :: input_shape(:)
+
+    self % input_size = input_shape(1)
+
+    ! Allocate arrays
+    allocate(self % output(self % input_size))
+    allocate(self % gradient(self % input_size))
+    allocate(self % mask(self % input_size))
+
+    ! Initialize arrays
+    self % output = 0
+    self % gradient = 0
+    self % mask = 1  ! Default mask is all ones (no dropout)
+
+  end subroutine init
+
+
+  module subroutine forward(self, input)
+    class(dropout_layer), intent(in out) :: self
+    real, intent(in) :: input(:)
+
+    ! Generate random mask for dropout
+    call random_number(self % mask)
+    where (self % mask < self % dropout_rate)
+      self % mask = 0
+    elsewhere
+      self % mask = 1
+    end where
+
+    ! Scale factor to preserve the input sum
+    self % scale = sum(input) / sum(self % output)  ! scale == 1/P(keep)
+
+    ! Apply dropout mask
+    self % output = input * self % mask * self % scale
+
+  end subroutine forward
+
+
+  pure module subroutine backward(self, input, gradient)
+    class(dropout_layer), intent(in out) :: self
+    real, intent(in) :: input(:)
+    real, intent(in) :: gradient(:)
+
+    ! Backpropagate gradient through dropout mask
+    self % gradient = gradient * self % mask * self % scale
+  end subroutine backward
+
+end submodule nf_dropout_layer_submodule 

src/nf/nf_layer.f90

@@ -76,7 +76,7 @@ end subroutine backward_3d
 
   interface
 
-    pure module subroutine forward(self, input)
+    module subroutine forward(self, input)
       !! Apply a forward pass on the layer.
       !! This changes the internal state of the layer.
       !! This is normally called internally by the `network % forward`

src/nf/nf_layer_constructors.f90

@@ -8,7 +8,7 @@ module nf_layer_constructors
   implicit none
 
   private
-  public :: conv2d, dense, flatten, input, maxpool2d, reshape
+  public :: conv2d, dense, flatten, input, maxpool2d, reshape, dropout
 
   interface input
 
@@ -85,6 +85,24 @@ module function dense(layer_size, activation) result(res)
         !! Resulting layer instance
     end function dense
 
+    module function dropout(rate) result(res)
+      !! Create a dropout layer with a given dropout rate.
+      !!
+      !! This layer is for randomly disabling neurons during training.
+      !!
+      !! Example:
+      !!
+      !! ```
+      !! use nf, only :: dropout, layer
+      !! type(layer) :: dropout_layer
+      !! dropout_layer = dropout(rate=0.5)
+      !! ```
+      real, intent(in) :: rate
+        !! Dropout rate - fraction of neurons to randomly disable during training
+      type(layer) :: res
+        !! Resulting layer instance
+    end function dropout
+
     module function flatten() result(res)
       !! Flatten (3-d -> 1-d) layer constructor.
       !!
@@ -166,6 +184,7 @@ module function reshape(output_shape) result(res)
         !! Resulting layer instance
     end function reshape
 
+
   end interface
 
 end module nf_layer_constructors

src/nf/nf_layer_constructors_submodule.f90

@@ -3,6 +3,7 @@
   use nf_layer, only: layer
   use nf_conv2d_layer, only: conv2d_layer
   use nf_dense_layer, only: dense_layer
+  use nf_dropout_layer, only: dropout_layer
   use nf_flatten_layer, only: flatten_layer
   use nf_input1d_layer, only: input1d_layer
   use nf_input3d_layer, only: input3d_layer
@@ -63,6 +64,14 @@ module function dense(layer_size, activation) result(res)
   end function dense
 
 
+  module function dropout(rate) result(res)
+    real, intent(in) :: rate
+    type(layer) :: res
+    res % name = 'dropout'
+    allocate(res % p, source=dropout_layer(rate))
+  end function dropout
+
+
   module function flatten() result(res)
     type(layer) :: res
     res % name = 'flatten'
@@ -91,6 +100,7 @@ module function input3d(layer_shape) result(res)
     res % initialized = .true.
   end function input3d
 
+
   module function maxpool2d(pool_size, stride) result(res)
     integer, intent(in) :: pool_size
     integer, intent(in), optional :: stride
@@ -119,6 +129,7 @@ module function maxpool2d(pool_size, stride) result(res)
 
   end function maxpool2d
 
+
   module function reshape(output_shape) result(res)
     integer, intent(in) :: output_shape(:)
     type(layer) :: res

src/nf/nf_layer_submodule.f90

@@ -3,6 +3,7 @@
   use iso_fortran_env, only: stderr => error_unit
   use nf_conv2d_layer, only: conv2d_layer
   use nf_dense_layer, only: dense_layer
+  use nf_dropout_layer, only: dropout_layer
   use nf_flatten_layer, only: flatten_layer
   use nf_input1d_layer, only: input1d_layer
   use nf_input3d_layer, only: input3d_layer
@@ -24,12 +25,14 @@ pure module subroutine backward_1d(self, previous, gradient)
 
       type is(dense_layer)
 
-        ! Upstream layers permitted: input1d, dense, flatten
+        ! Upstream layers permitted: input1d, dense, dropout, flatten
         select type(prev_layer => previous % p)
           type is(input1d_layer)
             call this_layer % backward(prev_layer % output, gradient)
           type is(dense_layer)
             call this_layer % backward(prev_layer % output, gradient)
+          type is(dropout_layer)
+            call this_layer % backward(prev_layer % output, gradient)
           type is(flatten_layer)
             call this_layer % backward(prev_layer % output, gradient)
         end select
@@ -106,7 +109,7 @@ pure module subroutine backward_3d(self, previous, gradient)
   end subroutine backward_3d
 
 
-  pure module subroutine forward(self, input)
+  module subroutine forward(self, input)
     implicit none
     class(layer), intent(in out) :: self
     class(layer), intent(in) :: input
@@ -115,6 +118,20 @@ pure module subroutine forward(self, input)
 
       type is(dense_layer)
 
+        ! Upstream layers permitted: input1d, dense, dropout, flatten
+        select type(prev_layer => input % p)
+          type is(input1d_layer)
+            call this_layer % forward(prev_layer % output)
+          type is(dense_layer)
+            call this_layer % forward(prev_layer % output)
+          type is(dropout_layer)
+            call this_layer % forward(prev_layer % output)
+          type is(flatten_layer)
+            call this_layer % forward(prev_layer % output)
+        end select
+
+      type is(dropout_layer)
+
         ! Upstream layers permitted: input1d, dense, flatten
         select type(prev_layer => input % p)
           type is(input1d_layer)
@@ -240,15 +257,17 @@ impure elemental module subroutine init(self, input)
       call this_layer % init(input % layer_shape)
     end select
 
-    ! The shape of conv2d, maxpool2d, or flatten layers is not known
+    ! The shape of conv2d, dropout, flatten, or maxpool2d layers is not known
     ! until we receive an input layer.
     select type(this_layer => self % p)
       type is(conv2d_layer)
         self % layer_shape = shape(this_layer % output)
-      type is(maxpool2d_layer)
+      type is(dropout_layer)
         self % layer_shape = shape(this_layer % output)
       type is(flatten_layer)
         self % layer_shape = shape(this_layer % output)
+      type is(maxpool2d_layer)
+        self % layer_shape = shape(this_layer % output)
     end select
 
     self % input_layer_shape = input % layer_shape 
@@ -284,6 +303,8 @@ elemental module function get_num_params(self) result(num_params)
         num_params = 0
       type is (dense_layer)
         num_params = this_layer % get_num_params()
+      type is (dropout_layer)
+        num_params = size(this_layer % mask)
       type is (conv2d_layer)
         num_params = this_layer % get_num_params()
       type is (maxpool2d_layer)
@@ -309,6 +330,8 @@ module function get_params(self) result(params)
          ! No parameters to get.
       type is (dense_layer)
         params = this_layer % get_params()
+      type is (dropout_layer)
+        ! No parameters to get.
       type is (conv2d_layer)
         params = this_layer % get_params()
       type is (maxpool2d_layer)
@@ -334,6 +357,8 @@ module function get_gradients(self) result(gradients)
         ! No gradients to get.
       type is (dense_layer)
         gradients = this_layer % get_gradients()
+      type is (dropout_layer)
+        ! No gradients to get.
       type is (conv2d_layer)
         gradients = this_layer % get_gradients()
       type is (maxpool2d_layer)
@@ -381,6 +406,11 @@ module subroutine set_params(self, params)
       type is (dense_layer)
         call this_layer % set_params(params)
 
+      type is (dropout_layer)
+        ! No parameters to set.
+        write(stderr, '(a)') 'Warning: calling set_params() ' &
+          // 'on a zero-parameter layer; nothing to do.'
+
       type is (conv2d_layer)
         call this_layer % set_params(params)