diff --git a/src/Grenade/Core/Layer.hs b/src/Grenade/Core/Layer.hs index db62808c..9911dae6 100644 --- a/src/Grenade/Core/Layer.hs +++ b/src/Grenade/Core/Layer.hs @@ -48,6 +48,7 @@ import Data.Kind (Type) import Grenade.Core.Shape import Grenade.Core.LearningParameters +import Control.DeepSeq (NFData) -- | Class for updating a layer. All layers implement this, as it -- describes how to create and update the layer. @@ -73,7 +74,7 @@ class UpdateLayer x where -- need to implement it for all shapes, only ones which are -- appropriate. -- -class UpdateLayer x => Layer x (i :: Shape) (o :: Shape) where +class (UpdateLayer x, NFData x) => Layer x (i :: Shape) (o :: Shape) where -- | The Wengert tape for this layer. Includes all that is required -- to generate the back propagated gradients efficiently. As a -- default, `S i` is fine. diff --git a/src/Grenade/Core/Network.hs b/src/Grenade/Core/Network.hs index f7b1f002..6690355c 100644 --- a/src/Grenade/Core/Network.hs +++ b/src/Grenade/Core/Network.hs @@ -44,6 +44,7 @@ import Data.Kind (Type) import Grenade.Core.Layer import Grenade.Core.LearningParameters import Grenade.Core.Shape +import Control.DeepSeq -- | Type of a network. -- @@ -68,6 +69,12 @@ instance Show (Network '[] '[i]) where instance (Show x, Show (Network xs rs)) => Show (Network (x ': xs) (i ': rs)) where show (x :~> xs) = show x ++ "\n~>\n" ++ show xs +instance NFData (Network '[] '[i]) where + rnf NNil = () + +instance NFData (Network xs rs) => NFData (Network (x ': xs) (i ': rs)) where + rnf (x :~> xs) = x `deepseq` xs `deepseq` () + -- | Gradient of a network. -- -- Parameterised on the layers of the network. @@ -191,7 +198,7 @@ instance CreatableNetwork sublayers subshapes => UpdateLayer (Network sublayers -- | Ultimate composition. -- -- This allows a complete network to be treated as a layer in a larger network. -instance (CreatableNetwork sublayers subshapes, i ~ (Head subshapes), o ~ (Last subshapes)) => Layer (Network sublayers subshapes) i o where +instance (CreatableNetwork sublayers subshapes, i ~ (Head subshapes), o ~ (Last subshapes), NFData (Network sublayers subshapes)) => Layer (Network sublayers subshapes) i o where type Tape (Network sublayers subshapes) i o = Tapes sublayers subshapes runForwards = runNetwork runBackwards = runGradient diff --git a/src/Grenade/Layers/Concat.hs b/src/Grenade/Layers/Concat.hs index ca3eba69..74de3e3d 100644 --- a/src/Grenade/Layers/Concat.hs +++ b/src/Grenade/Layers/Concat.hs @@ -35,6 +35,7 @@ import Data.Kind (Type) import Grenade.Core import Numeric.LinearAlgebra.Static ( row, (===), splitRows, unrow, (#), split, R ) +import Control.DeepSeq -- | A Concatentating Layer. -- @@ -52,6 +53,9 @@ import Numeric.LinearAlgebra.Static ( row, (===), splitRows, unrow, (# data Concat :: Shape -> Type -> Shape -> Type -> Type where Concat :: x -> y -> Concat m x n y +instance (NFData x, NFData y) => NFData (Concat m x n y) where + rnf (Concat x y) = rnf x `deepseq` rnf y `deepseq` () + instance (Show x, Show y) => Show (Concat m x n y) where show (Concat x y) = "Concat\n" ++ show x ++ "\n" ++ show y diff --git a/src/Grenade/Layers/Convolution.hs b/src/Grenade/Layers/Convolution.hs index aa0048b0..3d96ddbe 100644 --- a/src/Grenade/Layers/Convolution.hs +++ b/src/Grenade/Layers/Convolution.hs @@ -46,6 +46,7 @@ import Numeric.LinearAlgebra.Static hiding ((|||), build, toRows) import Grenade.Core import Grenade.Layers.Internal.Convolution import Grenade.Layers.Internal.Update +import Control.DeepSeq -- | A convolution layer for a neural network. -- This uses the im2col convolution trick popularised by Caffe, which essentially turns the @@ -113,6 +114,9 @@ instance Show (Convolution c f k k' s s') where px = (fmap . fmap . fmap) render ms in unlines $ foldl1 (zipWith (\a' b' -> a' ++ " | " ++ b')) $ px +instance NFData (Convolution c f k k' s s') where + rnf (Convolution a b) = rnf a `deepseq` rnf b `deepseq` () + randomConvolution :: ( MonadRandom m , KnownNat channels , KnownNat filters diff --git a/src/Grenade/Layers/Crop.hs b/src/Grenade/Layers/Crop.hs index 0634bca6..bcf9b693 100644 --- a/src/Grenade/Layers/Crop.hs +++ b/src/Grenade/Layers/Crop.hs @@ -37,6 +37,7 @@ import Grenade.Layers.Internal.Pad import Numeric.LinearAlgebra (konst, subMatrix, diagBlock) import Numeric.LinearAlgebra.Static (extract, create) +import Control.DeepSeq -- | A cropping layer for a neural network. data Crop :: Nat @@ -45,6 +46,9 @@ data Crop :: Nat -> Nat -> Type where Crop :: Crop cropLeft cropTop cropRight cropBottom +instance NFData (Crop cropLeft cropTop cropRight cropBottom) where + rnf Crop = () + instance Show (Crop cropLeft cropTop cropRight cropBottom) where show Crop = "Crop" diff --git a/src/Grenade/Layers/Deconvolution.hs b/src/Grenade/Layers/Deconvolution.hs index f46d12b2..f4ab8827 100644 --- a/src/Grenade/Layers/Deconvolution.hs +++ b/src/Grenade/Layers/Deconvolution.hs @@ -50,6 +50,7 @@ import Numeric.LinearAlgebra.Static hiding ((|||), build, toRows) import Grenade.Core import Grenade.Layers.Internal.Convolution import Grenade.Layers.Internal.Update +import Control.DeepSeq -- | A Deconvolution layer for a neural network. -- This uses the im2col Convolution trick popularised by Caffe. @@ -94,6 +95,9 @@ data Deconvolution' :: Nat -- Number of channels, for the first layer this could => !(L kernelFlattened channels) -- The kernel filter gradient -> Deconvolution' channels filters kernelRows kernelColumns strideRows strideColumns +instance NFData (Deconvolution c f k k' s s') where + rnf (Deconvolution a b) = rnf a `deepseq` rnf b `deepseq` () + instance Show (Deconvolution c f k k' s s') where show (Deconvolution a _) = renderConv a where diff --git a/src/Grenade/Layers/Dropout.hs b/src/Grenade/Layers/Dropout.hs index 530947d1..6354b6a4 100644 --- a/src/Grenade/Layers/Dropout.hs +++ b/src/Grenade/Layers/Dropout.hs @@ -11,6 +11,7 @@ import Control.Monad.Random hiding (fromList) import GHC.TypeLits import Grenade.Core +import Control.DeepSeq -- Dropout layer help to reduce overfitting. -- Idea here is that the vector is a shape of 1s and 0s, which we multiply the input by. @@ -20,7 +21,10 @@ import Grenade.Core data Dropout = Dropout { dropoutRate :: Double , dropoutSeed :: Int - } deriving Show + } deriving (Show) + +instance NFData Dropout where + rnf (Dropout r s) = rnf r `deepseq` rnf s `deepseq` () instance UpdateLayer Dropout where type Gradient Dropout = () diff --git a/src/Grenade/Layers/Elu.hs b/src/Grenade/Layers/Elu.hs index a1090860..7524c768 100644 --- a/src/Grenade/Layers/Elu.hs +++ b/src/Grenade/Layers/Elu.hs @@ -19,6 +19,7 @@ import GHC.TypeLits import Grenade.Core import qualified Numeric.LinearAlgebra.Static as LAS +import Control.DeepSeq -- | An exponential linear unit. -- A layer which can act between any shape of the same dimension, acting as a @@ -26,6 +27,9 @@ import qualified Numeric.LinearAlgebra.Static as LAS data Elu = Elu deriving Show +instance NFData Elu where + rnf Elu = () + instance UpdateLayer Elu where type Gradient Elu = () runUpdate _ _ _ = Elu diff --git a/src/Grenade/Layers/FullyConnected.hs b/src/Grenade/Layers/FullyConnected.hs index 70416586..a97c4ae4 100644 --- a/src/Grenade/Layers/FullyConnected.hs +++ b/src/Grenade/Layers/FullyConnected.hs @@ -21,6 +21,7 @@ import Numeric.LinearAlgebra.Static import Grenade.Core import Grenade.Layers.Internal.Update +import Control.DeepSeq -- | A basic fully connected (or inner product) neural network layer. data FullyConnected i o = FullyConnected @@ -31,6 +32,12 @@ data FullyConnected' i o = FullyConnected' !(R o) -- Bias !(L o i) -- Activations +instance NFData (FullyConnected' i o) where + rnf (FullyConnected' a b) = a `deepseq` b `deepseq` () + +instance NFData (FullyConnected i o) where + rnf (FullyConnected a b) = a `deepseq` b `deepseq` () + instance Show (FullyConnected i o) where show FullyConnected {} = "FullyConnected" diff --git a/src/Grenade/Layers/Logit.hs b/src/Grenade/Layers/Logit.hs index cc584825..fb7343d2 100644 --- a/src/Grenade/Layers/Logit.hs +++ b/src/Grenade/Layers/Logit.hs @@ -19,6 +19,7 @@ import Data.Serialize import Data.Singletons import Grenade.Core +import Control.DeepSeq -- | A Logit layer. -- @@ -28,6 +29,9 @@ import Grenade.Core data Logit = Logit deriving Show +instance NFData Logit where + rnf Logit = () + instance UpdateLayer Logit where type Gradient Logit = () runUpdate _ _ _ = Logit diff --git a/src/Grenade/Layers/Merge.hs b/src/Grenade/Layers/Merge.hs index f5333f21..34b7b594 100644 --- a/src/Grenade/Layers/Merge.hs +++ b/src/Grenade/Layers/Merge.hs @@ -29,6 +29,7 @@ import Data.Kind (Type) #endif import Grenade.Core +import Control.DeepSeq -- | A Merging layer. -- @@ -37,6 +38,9 @@ import Grenade.Core data Merge :: Type -> Type -> Type where Merge :: x -> y -> Merge x y +instance (NFData x, NFData y) => NFData (Merge x y) where + rnf (Merge x y) = rnf x `deepseq` rnf y `deepseq` () + instance (Show x, Show y) => Show (Merge x y) where show (Merge x y) = "Merge\n" ++ show x ++ "\n" ++ show y diff --git a/src/Grenade/Layers/Pad.hs b/src/Grenade/Layers/Pad.hs index 652a778c..ae1cc999 100644 --- a/src/Grenade/Layers/Pad.hs +++ b/src/Grenade/Layers/Pad.hs @@ -37,6 +37,7 @@ import Grenade.Layers.Internal.Pad import Numeric.LinearAlgebra (konst, subMatrix, diagBlock) import Numeric.LinearAlgebra.Static (extract, create) +import Control.DeepSeq -- | A padding layer for a neural network. -- @@ -47,6 +48,9 @@ data Pad :: Nat -> Nat -> Type where Pad :: Pad padLeft padTop padRight padBottom +instance NFData (Pad padLeft padTop padRight padBottom) where + rnf Pad = () + instance Show (Pad padLeft padTop padRight padBottom) where show Pad = "Pad" diff --git a/src/Grenade/Layers/Pooling.hs b/src/Grenade/Layers/Pooling.hs index ecfb9ee1..53fbe57e 100644 --- a/src/Grenade/Layers/Pooling.hs +++ b/src/Grenade/Layers/Pooling.hs @@ -37,6 +37,7 @@ import Grenade.Core import Grenade.Layers.Internal.Pooling import Numeric.LinearAlgebra.Static as LAS hiding ((|||), build, toRows) +import Control.DeepSeq -- | A pooling layer for a neural network. -- @@ -49,6 +50,9 @@ import Numeric.LinearAlgebra.Static as LAS hiding ((|||), build, toRow data Pooling :: Nat -> Nat -> Nat -> Nat -> Type where Pooling :: Pooling kernelRows kernelColumns strideRows strideColumns +instance NFData (Pooling k k' s s') where + rnf Pooling = () + instance Show (Pooling k k' s s') where show Pooling = "Pooling" diff --git a/src/Grenade/Layers/Relu.hs b/src/Grenade/Layers/Relu.hs index 3db4670f..6e413a93 100644 --- a/src/Grenade/Layers/Relu.hs +++ b/src/Grenade/Layers/Relu.hs @@ -19,6 +19,7 @@ import GHC.TypeLits import Grenade.Core import qualified Numeric.LinearAlgebra.Static as LAS +import Control.DeepSeq -- | A rectifying linear unit. -- A layer which can act between any shape of the same dimension, acting as a @@ -26,6 +27,9 @@ import qualified Numeric.LinearAlgebra.Static as LAS data Relu = Relu deriving Show +instance NFData Relu where + rnf Relu = () + instance UpdateLayer Relu where type Gradient Relu = () runUpdate _ _ _ = Relu diff --git a/src/Grenade/Layers/Reshape.hs b/src/Grenade/Layers/Reshape.hs index 8a657bc3..25700a8c 100644 --- a/src/Grenade/Layers/Reshape.hs +++ b/src/Grenade/Layers/Reshape.hs @@ -24,6 +24,7 @@ import Numeric.LinearAlgebra.Static import Numeric.LinearAlgebra.Data as LA ( flatten ) import Grenade.Core +import Control.DeepSeq -- | Reshape Layer -- @@ -36,6 +37,9 @@ import Grenade.Core data Reshape = Reshape deriving Show +instance NFData Reshape where + rnf Reshape = () + instance UpdateLayer Reshape where type Gradient Reshape = () runUpdate _ _ _ = Reshape diff --git a/src/Grenade/Layers/Sinusoid.hs b/src/Grenade/Layers/Sinusoid.hs index cc68dd01..91adcdd5 100644 --- a/src/Grenade/Layers/Sinusoid.hs +++ b/src/Grenade/Layers/Sinusoid.hs @@ -18,12 +18,16 @@ import Data.Serialize import Data.Singletons import Grenade.Core +import Control.DeepSeq -- | A Sinusoid layer. -- A layer which can act between any shape of the same dimension, performing a sin function. data Sinusoid = Sinusoid deriving Show +instance NFData Sinusoid where + rnf Sinusoid = () + instance UpdateLayer Sinusoid where type Gradient Sinusoid = () runUpdate _ _ _ = Sinusoid diff --git a/src/Grenade/Layers/Softmax.hs b/src/Grenade/Layers/Softmax.hs index 27ca4b68..e850509c 100644 --- a/src/Grenade/Layers/Softmax.hs +++ b/src/Grenade/Layers/Softmax.hs @@ -22,6 +22,7 @@ import GHC.TypeLits import Grenade.Core import Numeric.LinearAlgebra.Static as LAS +import Control.DeepSeq -- | A Softmax layer -- @@ -33,6 +34,9 @@ import Numeric.LinearAlgebra.Static as LAS data Softmax = Softmax deriving Show +instance NFData Softmax where + rnf Softmax = () + instance UpdateLayer Softmax where type Gradient Softmax = () runUpdate _ _ _ = Softmax diff --git a/src/Grenade/Layers/Tanh.hs b/src/Grenade/Layers/Tanh.hs index fafbe3d3..44482702 100644 --- a/src/Grenade/Layers/Tanh.hs +++ b/src/Grenade/Layers/Tanh.hs @@ -18,12 +18,16 @@ import Data.Serialize import Data.Singletons import Grenade.Core +import Control.DeepSeq -- | A Tanh layer. -- A layer which can act between any shape of the same dimension, performing a tanh function. data Tanh = Tanh deriving Show +instance NFData Tanh where + rnf Tanh = () + instance UpdateLayer Tanh where type Gradient Tanh = () runUpdate _ _ _ = Tanh diff --git a/src/Grenade/Layers/Trivial.hs b/src/Grenade/Layers/Trivial.hs index 0d756ca7..61bbe9f1 100644 --- a/src/Grenade/Layers/Trivial.hs +++ b/src/Grenade/Layers/Trivial.hs @@ -17,6 +17,7 @@ module Grenade.Layers.Trivial ( import Data.Serialize import Grenade.Core +import Control.DeepSeq -- | A Trivial layer. -- @@ -25,6 +26,9 @@ import Grenade.Core data Trivial = Trivial deriving Show +instance NFData Trivial where + rnf Trivial = () + instance Serialize Trivial where put _ = return () get = return Trivial diff --git a/src/Grenade/Recurrent/Core/Layer.hs b/src/Grenade/Recurrent/Core/Layer.hs index 91243712..a7660018 100644 --- a/src/Grenade/Recurrent/Core/Layer.hs +++ b/src/Grenade/Recurrent/Core/Layer.hs @@ -14,6 +14,7 @@ import Data.Kind (Type) #endif import Grenade.Core +import Control.DeepSeq (NFData) -- | Class for a recurrent layer. -- It's quite similar to a normal layer but for the input and output @@ -22,7 +23,7 @@ class UpdateLayer x => RecurrentUpdateLayer x where -- | Shape of data that is passed between each subsequent run of the layer type RecurrentShape x :: Type -class (RecurrentUpdateLayer x, Num (RecurrentShape x)) => RecurrentLayer x (i :: Shape) (o :: Shape) where +class (RecurrentUpdateLayer x, Num (RecurrentShape x), NFData x) => RecurrentLayer x (i :: Shape) (o :: Shape) where -- | Wengert Tape type RecTape x i o :: Type -- | Used in training and scoring. Take the input from the previous diff --git a/src/Grenade/Recurrent/Core/Network.hs b/src/Grenade/Recurrent/Core/Network.hs index 62df741e..414c969d 100644 --- a/src/Grenade/Recurrent/Core/Network.hs +++ b/src/Grenade/Recurrent/Core/Network.hs @@ -39,6 +39,7 @@ import Data.Kind (Type) import Grenade.Core import Grenade.Recurrent.Core.Layer +import Control.DeepSeq -- | Witness type to say indicate we're building up with a normal feed -- forward layer. @@ -198,6 +199,12 @@ applyRecurrentUpdate rate (layer :~@> rest) (gradient ://> grest) applyRecurrentUpdate _ RNil RGNil = RNil +instance NFData (RecurrentNetwork '[] '[i]) where + rnf RNil = () + +instance NFData (RecurrentNetwork xs rs) => NFData (RecurrentNetwork (x ': xs) (i ': rs)) where + rnf (layer :~~> rest) = rnf layer `deepseq` rnf rest `deepseq` () + rnf (layer :~@> rest) = rnf layer `deepseq` rnf rest `deepseq` () instance Show (RecurrentNetwork '[] '[i]) where show RNil = "NNil" @@ -325,6 +332,7 @@ instance CreatableRecurrent sublayers subshapes => RecurrentUpdateLayer (Recurre instance ( CreatableRecurrent sublayers subshapes , i ~ (Head subshapes), o ~ (Last subshapes) , Num (RecurrentShape (RecurrentNetwork sublayers subshapes)) + , NFData (RecurrentNetwork sublayers subshapes) ) => RecurrentLayer (RecurrentNetwork sublayers subshapes) i o where type RecTape (RecurrentNetwork sublayers subshapes) i o = RecurrentTape sublayers subshapes runRecurrentForwards = runRecurrent diff --git a/src/Grenade/Recurrent/Layers/BasicRecurrent.hs b/src/Grenade/Recurrent/Layers/BasicRecurrent.hs index 53b0133a..009022c9 100644 --- a/src/Grenade/Recurrent/Layers/BasicRecurrent.hs +++ b/src/Grenade/Recurrent/Layers/BasicRecurrent.hs @@ -28,6 +28,7 @@ import GHC.TypeLits import Grenade.Core import Grenade.Recurrent.Core +import Control.DeepSeq data BasicRecurrent :: Nat -- Input layer size -> Nat -- Output layer size @@ -42,6 +43,9 @@ data BasicRecurrent :: Nat -- Input layer size -> !(L output matrixCols) -- Momentum -> BasicRecurrent input output +instance NFData (BasicRecurrent input output) where + rnf (BasicRecurrent b bm a aM) = rnf b `deepseq` rnf bm `deepseq` rnf a `deepseq` rnf aM `deepseq` () + data BasicRecurrent' :: Nat -- Input layer size -> Nat -- Output layer size -> Type where diff --git a/src/Grenade/Recurrent/Layers/ConcatRecurrent.hs b/src/Grenade/Recurrent/Layers/ConcatRecurrent.hs index 65aa197e..9477347e 100644 --- a/src/Grenade/Recurrent/Layers/ConcatRecurrent.hs +++ b/src/Grenade/Recurrent/Layers/ConcatRecurrent.hs @@ -36,6 +36,7 @@ import Grenade.Core import Grenade.Recurrent.Core import Numeric.LinearAlgebra.Static ( (#), split, R ) +import Control.DeepSeq -- | A Concatentating Layer. -- @@ -55,6 +56,15 @@ data ConcatRecurrent :: Shape -> Type -> Shape -> Type -> Type where ConcatRecRight :: x -> y -> ConcatRecurrent m (FeedForward x) n (Recurrent y) ConcatRecBoth :: x -> y -> ConcatRecurrent m (Recurrent x) n (Recurrent y) +instance (NFData x, NFData y) => NFData (ConcatRecurrent m (Recurrent x) n (Recurrent y)) where + rnf (ConcatRecBoth x y) = rnf x `deepseq` rnf y `deepseq` () + +instance (NFData x, NFData y) => NFData (ConcatRecurrent m (Recurrent x) n (FeedForward y)) where + rnf (ConcatRecLeft x y) = rnf x `deepseq` rnf y `deepseq` () + +instance (NFData x, NFData y) => NFData (ConcatRecurrent m (FeedForward x) n (Recurrent y)) where + rnf (ConcatRecRight x y) = rnf x `deepseq` rnf y `deepseq` () + instance (Show x, Show y) => Show (ConcatRecurrent m (p x) n (q y)) where show (ConcatRecLeft x y) = "ConcatRecLeft\n" ++ show x ++ "\n" ++ show y show (ConcatRecRight x y) = "ConcatRecRight\n" ++ show x ++ "\n" ++ show y diff --git a/src/Grenade/Recurrent/Layers/LSTM.hs b/src/Grenade/Recurrent/Layers/LSTM.hs index 24371489..0c07d48e 100644 --- a/src/Grenade/Recurrent/Layers/LSTM.hs +++ b/src/Grenade/Recurrent/Layers/LSTM.hs @@ -33,6 +33,7 @@ import Numeric.LinearAlgebra.Static import Grenade.Core import Grenade.Recurrent.Core import Grenade.Layers.Internal.Update +import Control.DeepSeq -- | Long Short Term Memory Recurrent unit @@ -64,6 +65,16 @@ data LSTMWeights :: Nat -> Nat -> Type where , lstmBc :: !(R output) -- Bias Cell (b_c) } -> LSTMWeights input output +instance NFData (LSTMWeights input output) where + rnf (LSTMWeights wf uf bf wi ui bi wo uo bo wc bc) = + rnf wf `deepseq` rnf uf `deepseq` rnf bf `deepseq` + rnf wi `deepseq` rnf ui `deepseq` rnf bi `deepseq` + rnf wo `deepseq` rnf uo `deepseq` rnf bo `deepseq` + rnf wc `deepseq` rnf bc `deepseq` () + +instance NFData (LSTM input output) where + rnf (LSTM w m) = rnf w `deepseq` rnf m `deepseq` () + instance Show (LSTM i o) where show LSTM {} = "LSTM"