@@ -1,3 +1,4 @@
1{-# LANGUAGE DataKinds #-}
2{-# LANGUAGE DataKinds #-}
2{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE TypeOperators #-}
4{-# LANGUAGE TypeOperators #-}
@@ -17,37 +18,34 @@ module Grenade.Recurrent.Core.Network (
18 , RecurrentNetwork (..)
19 , RecurrentNetwork (..)
19 , RecurrentInputs (..)
20 , RecurrentInputs (..)
20− , RecurrentTapes (..)
21+ , RecurrentTape (..)
21− , RecurrentGradients (..)
22+ , RecurrentGradient (..)
23 , randomRecurrent
24 , randomRecurrent
24− , runRecurrentNetwork
25+ , runRecurrent
25− , runRecurrentGradient
26+ , runRecurrent'
26 , applyRecurrentUpdate
27 , applyRecurrentUpdate
30import Control.Monad.Random ( MonadRandom )
31import Control.Monad.Random ( MonadRandom )
31−import Data.Singletons ( SingI )
32−import Data.Singletons.Prelude ( Head, Last )
33import Data.Serialize
32import Data.Serialize
34−import qualified Data.Vector.Storable as V
33+
34+import Data.Kind (Type)
36import Grenade.Core
36import Grenade.Core
37import Grenade.Recurrent.Core.Layer
37import Grenade.Recurrent.Core.Layer
38−
38+import Prelude.Singletons
39−import qualified Numeric.LinearAlgebra as LA
40−import qualified Numeric.LinearAlgebra.Static as LAS
42-- | Witness type to say indicate we're building up with a normal feed
40-- | Witness type to say indicate we're building up with a normal feed
43-- forward layer.
41-- forward layer.
44−data FeedForward :: * -> *
42+data FeedForward :: Type -> Type
45-- | Witness type to say indicate we're building up with a recurrent layer.
43-- | Witness type to say indicate we're building up with a recurrent layer.
46−data Recurrent :: * -> *
44+data Recurrent :: Type -> Type
48-- | Type of a recurrent neural network.
46-- | Type of a recurrent neural network.
50−-- The [*] type specifies the types of the layers.
48+-- The [Type] type specifies the types of the layers.
52-- The [Shape] type specifies the shapes of data passed between the layers.
50-- The [Shape] type specifies the shapes of data passed between the layers.
@@ -56,7 +54,7 @@ data Recurrent :: * -> *
57-- Often, to make the definitions more concise, one will use a type alias
55-- Often, to make the definitions more concise, one will use a type alias
58-- for these empty data types.
56-- for these empty data types.
59−data RecurrentNetwork :: [*] -> [Shape] -> * where
57+data RecurrentNetwork :: [Type] -> [Shape] -> Type where
60 RNil :: SingI i
58 RNil :: SingI i
61 => RecurrentNetwork '[] '[i]
59 => RecurrentNetwork '[] '[i]
@@ -75,96 +73,82 @@ infixr 5 :~@>
75-- | Gradient of a network.
73-- | Gradient of a network.
77-- Parameterised on the layers of the network.
75-- Parameterised on the layers of the network.
78−data RecurrentGradients :: [*] -> * where
76+data RecurrentGradient :: [Type] -> Type where
79− RGNil :: RecurrentGradients '[]
77+ RGNil :: RecurrentGradient '[]
81 (://>) :: UpdateLayer x
79 (://>) :: UpdateLayer x
82− => [Gradient x]
80+ => Gradient x
83− -> RecurrentGradients xs
81+ -> RecurrentGradient xs
84− -> RecurrentGradients (phantom x ': xs)
82+ -> RecurrentGradient (phantom x ': xs)
86-- | Recurrent inputs (sideways shapes on an imaginary unrolled graph)
84-- | Recurrent inputs (sideways shapes on an imaginary unrolled graph)
87-- Parameterised on the layers of a Network.
85-- Parameterised on the layers of a Network.
88−data RecurrentInputs :: [*] -> * where
86+data RecurrentInputs :: [Type] -> Type where
89 RINil :: RecurrentInputs '[]
87 RINil :: RecurrentInputs '[]
91− (:~~+>) :: UpdateLayer x
89+ (:~~+>) :: (UpdateLayer x, Fractional (RecurrentInputs xs))
92 => () -> !(RecurrentInputs xs) -> RecurrentInputs (FeedForward x ': xs)
90 => () -> !(RecurrentInputs xs) -> RecurrentInputs (FeedForward x ': xs)
94− (:~@+>) :: (SingI (RecurrentShape x), RecurrentUpdateLayer x)
92+ (:~@+>) :: (Fractional (RecurrentShape x), Fractional (RecurrentInputs xs), RecurrentUpdateLayer x)
95− => !(S (RecurrentShape x)) -> !(RecurrentInputs xs) -> RecurrentInputs (Recurrent x ': xs)
93+ => !(RecurrentShape x) -> !(RecurrentInputs xs) -> RecurrentInputs (Recurrent x ': xs)
97-- | All the information required to backpropogate
95-- | All the information required to backpropogate
98-- through time safely.
96-- through time safely.
100-- We index on the time step length as well, to ensure
98-- We index on the time step length as well, to ensure
101-- that that all Tape lengths are the same.
99-- that that all Tape lengths are the same.
102−data RecurrentTapes :: [*] -> [Shape] -> * where
100+data RecurrentTape :: [Type] -> [Shape] -> Type where
103 TRNil :: SingI i
101 TRNil :: SingI i
104− => RecurrentTapes '[] '[i]
102+ => RecurrentTape '[] '[i]
106− (:\~>) :: [Tape x i h]
107− -> !(RecurrentTapes xs (h ': hs))
108− -> RecurrentTapes (FeedForward x ': xs) (i ': h ': hs)
104+ (:\~>) :: Tape x i h
105+ -> !(RecurrentTape xs (h ': hs))
106+ -> RecurrentTape (FeedForward x ': xs) (i ': h ': hs)
111− (:\@>) :: [RecTape x i h]
108+ (:\@>) :: RecTape x i h
112− -> !(RecurrentTapes xs (h ': hs))
109+ -> !(RecurrentTape xs (h ': hs))
113− -> RecurrentTapes (Recurrent x ': xs) (i ': h ': hs)
110+ -> RecurrentTape (Recurrent x ': xs) (i ': h ': hs)
116−runRecurrentNetwork :: forall shapes layers.
113+runRecurrent :: forall shapes layers.
117− RecurrentNetwork layers shapes
114+ RecurrentNetwork layers shapes
118− -> RecurrentInputs layers
115+ -> RecurrentInputs layers
119− -> [S (Head shapes)]
116+ -> S (Head shapes)
120− -> (RecurrentTapes layers shapes, RecurrentInputs layers, [S (Last shapes)])
117+ -> (RecurrentTape layers shapes, RecurrentInputs layers, S (Last shapes))
121−runRecurrentNetwork =
118+runRecurrent =
124 go :: forall js sublayers. (Last js ~ Last shapes)
121 go :: forall js sublayers. (Last js ~ Last shapes)
125 => RecurrentNetwork sublayers js
122 => RecurrentNetwork sublayers js
126 -> RecurrentInputs sublayers
123 -> RecurrentInputs sublayers
127− -> [S (Head js)]
124+ -> S (Head js)
128− -> (RecurrentTapes sublayers js, RecurrentInputs sublayers, [S (Last js)])
125+ -> (RecurrentTape sublayers js, RecurrentInputs sublayers, S (Last js))
129− -- This is a simple non-recurrent layer, just map it forwards
130− go (layer :~~> n) (() :~~+> nIn) !xs
126+ go (!layer :~~> n) (() :~~+> nIn) !x
131− = let tys = runForwards layer <$> xs
127+ = let (!tape, !forwards) = runForwards layer x
132− feedForwardTapes = fst <$> tys
128+
133− forwards = snd <$> tys
134 -- recursively run the rest of the network, and get the gradients from above.
129 -- recursively run the rest of the network, and get the gradients from above.
135− (newFN, ig, answer) = go n nIn forwards
130+ (!newFN, !ig, !answer) = go n nIn forwards
136− in (feedForwardTapes :\~> newFN, () :~~+> ig, answer)
131+ in (tape :\~> newFN, () :~~+> ig, answer)
138 -- This is a recurrent layer, so we need to do a scan, first input to last, providing
133 -- This is a recurrent layer, so we need to do a scan, first input to last, providing
139 -- the recurrent shape output to the next layer.
134 -- the recurrent shape output to the next layer.
140− go (layer :~@> n) (recIn :~@+> nIn) !xs
135+ go (layer :~@> n) (recIn :~@+> nIn) !x
141− = let (recOut, tys) = goR layer recIn xs
136+ = let (tape, shape, forwards) = runRecurrentForwards layer recIn x
142− recurrentTapes = fst <$> tys
143− forwards = snd <$> tys
145 (newFN, ig, answer) = go n nIn forwards
137 (newFN, ig, answer) = go n nIn forwards
146− in (recurrentTapes :\@> newFN, recOut :~@+> ig, answer)
138+ in (tape :\@> newFN, shape :~@+> ig, answer)
148 -- Handle the output layer, bouncing the derivatives back down.
140 -- Handle the output layer, bouncing the derivatives back down.
149 -- We may not have a target for each example, so when we don't use 0 gradient.
141 -- We may not have a target for each example, so when we don't use 0 gradient.
150 go RNil RINil !x
142 go RNil RINil !x
151 = (TRNil, RINil, x)
143 = (TRNil, RINil, x)
153− -- Helper function for recurrent layers
154− -- Scans over the recurrent direction of the graph.
155− goR !layer !recShape (x:xs) =
156− let (tape, lerec, lepush) = runRecurrentForwards layer recShape x
157− (rems, push) = goR layer lerec xs
158− in (rems, (tape, lepush) : push)
159− goR _ rin [] = (rin, [])
161−runRecurrentGradient :: forall layers shapes.
145+runRecurrent' :: forall layers shapes.
162− RecurrentNetwork layers shapes
146+ RecurrentNetwork layers shapes
163− -> RecurrentTapes layers shapes
147+ -> RecurrentTape layers shapes
164− -> RecurrentInputs layers
148+ -> RecurrentInputs layers
165− -> [S (Last shapes)]
149+ -> S (Last shapes)
166− -> (RecurrentGradients layers, RecurrentInputs layers, [S (Head shapes)])
150+ -> (RecurrentGradient layers, RecurrentInputs layers, S (Head shapes))
167−runRecurrentGradient net tapes r o =
151+runRecurrent' net tapes r o =
168 go net tapes r
152 go net tapes r
170 -- We have to be careful regarding the direction of the lists
154 -- We have to be careful regarding the direction of the lists
@@ -172,51 +156,41 @@ runRecurrentGradient net tapes r o =
172 -- through time.
156 -- through time.
173 go :: forall js ss. (Last js ~ Last shapes)
157 go :: forall js ss. (Last js ~ Last shapes)
174 => RecurrentNetwork ss js
158 => RecurrentNetwork ss js
175− -> RecurrentTapes ss js
159+ -> RecurrentTape ss js
176 -> RecurrentInputs ss
160 -> RecurrentInputs ss
177− -> (RecurrentGradients ss, RecurrentInputs ss, [S (Head js)])
161+ -> (RecurrentGradient ss, RecurrentInputs ss, S (Head js))
178 -- This is a simple non-recurrent layer
162 -- This is a simple non-recurrent layer
179− -- Run the rest of the network, then fmap the tapes and gradients
163+ -- Run the rest of the network, update with the tapes and gradients
180− go (layer :~~> n) (feedForwardTapes :\~> nTapes) (() :~~+> nRecs) =
164+ go (!layer :~~> n) (!tape :\~> nTapes) (() :~~+> nRecs) =
181− let (gradients, rins, feed) = go n nTapes nRecs
165+ let (!gradients, !rins, !feed) = go n nTapes nRecs
182− backs = uncurry (runBackwards layer) <$> zip (reverse feedForwardTapes) feed
166+ (!grad, !back) = runBackwards layer tape feed
183− in ((fst <$> backs) ://> gradients, () :~~+> rins, snd <$> backs)
167+ in (grad ://> gradients, () :~~+> rins, back)
185 -- This is a recurrent layer
169 -- This is a recurrent layer
186 -- Run the rest of the network, scan over the tapes in reverse
170 -- Run the rest of the network, scan over the tapes in reverse
187− go (layer :~@> n) (recurrentTapes :\@> nTapes) (recGrad :~@+> nRecs) =
171+ go (!layer :~@> n) (!tape :\@> nTapes) (!recGrad :~@+> nRecs) =
188− let (gradients, rins, feed) = go n nTapes nRecs
172+ let (!gradients, !rins, !feed) = go n nTapes nRecs
189− backExamples = zip (reverse recurrentTapes) feed
173+ (!grad, !sidegrad, !back) = runRecurrentBackwards layer tape recGrad feed
190− (rg, backs) = goX layer recGrad backExamples
191− in ((fst <$> backs) ://> gradients, rg :~@+> rins, snd <$> backs)
174+ in (grad ://> gradients, sidegrad :~@+> rins, back)
193 -- End of the road, so we reflect the given gradients backwards.
176 -- End of the road, so we reflect the given gradients backwards.
194 -- Crucially, we reverse the list, so it's backwards in time as
177 -- Crucially, we reverse the list, so it's backwards in time as
196− go RNil TRNil RINil
179+ go !RNil !TRNil !RINil
197− = (RGNil, RINil, reverse o)
180+ = (RGNil, RINil, o)
199− -- Helper function for recurrent layers
200− -- Scans over the recurrent direction of the graph.
201− goX :: RecurrentLayer x i o => x -> S (RecurrentShape x) -> [(RecTape x i o, S o)] -> (S (RecurrentShape x), [(Gradient x, S i)])
202− goX layer !lastback ((recTape, backgrad):xs) =
203− let (layergrad, recgrad, ingrad) = runRecurrentBackwards layer recTape lastback backgrad
204− (pushedback, ll) = goX layer recgrad xs
205− in (pushedback, (layergrad, ingrad) : ll)
206− goX _ !lastback [] = (lastback, [])
208-- | Apply a batch of gradients to the network
182-- | Apply a batch of gradients to the network
209-- Uses runUpdates which can be specialised for
183-- Uses runUpdates which can be specialised for
210-- a layer.
184-- a layer.
211applyRecurrentUpdate :: LearningParameters
185applyRecurrentUpdate :: LearningParameters
212 -> RecurrentNetwork layers shapes
186 -> RecurrentNetwork layers shapes
213− -> RecurrentGradients layers
187+ -> RecurrentGradient layers
214 -> RecurrentNetwork layers shapes
188 -> RecurrentNetwork layers shapes
215applyRecurrentUpdate rate (layer :~~> rest) (gradient ://> grest)
189applyRecurrentUpdate rate (layer :~~> rest) (gradient ://> grest)
216− = runUpdates rate layer gradient :~~> applyRecurrentUpdate rate rest grest
190+ = runUpdate rate layer gradient :~~> applyRecurrentUpdate rate rest grest
218applyRecurrentUpdate rate (layer :~@> rest) (gradient ://> grest)
192applyRecurrentUpdate rate (layer :~@> rest) (gradient ://> grest)
219− = runUpdates rate layer gradient :~@> applyRecurrentUpdate rate rest grest
193+ = runUpdate rate layer gradient :~@> applyRecurrentUpdate rate rest grest
221applyRecurrentUpdate _ RNil RGNil
195applyRecurrentUpdate _ RNil RGNil
@@ -232,26 +206,25 @@ instance (Show x, Show (RecurrentNetwork xs rs)) => Show (RecurrentNetwork (Recu
233-- | A network can easily be created by hand with (:~~>) and (:~@>), but an easy way to initialise a random
207-- | A network can easily be created by hand with (:~~>) and (:~@>), but an easy way to initialise a random
234-- recurrent network and a set of random inputs for it is with the randomRecurrent.
208-- recurrent network and a set of random inputs for it is with the randomRecurrent.
235−class CreatableRecurrent (xs :: [*]) (ss :: [Shape]) where
209+class CreatableRecurrent (xs :: [Type]) (ss :: [Shape]) where
236 -- | Create a network of the types requested
210 -- | Create a network of the types requested
237− randomRecurrent :: MonadRandom m => m (RecurrentNetwork xs ss, RecurrentInputs xs)
211+ randomRecurrent :: MonadRandom m => m (RecurrentNetwork xs ss)
239instance SingI i => CreatableRecurrent '[] '[i] where
213instance SingI i => CreatableRecurrent '[] '[i] where
240 randomRecurrent =
214 randomRecurrent =
241− return (RNil, RINil)
215+ return RNil
243instance (SingI i, Layer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (FeedForward x ': xs) (i ': o ': rs) where
217instance (SingI i, Layer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (FeedForward x ': xs) (i ': o ': rs) where
244 randomRecurrent = do
218 randomRecurrent = do
245 thisLayer <- createRandom
219 thisLayer <- createRandom
246− (rest, resti) <- randomRecurrent
220+ rest <- randomRecurrent
247− return (thisLayer :~~> rest, () :~~+> resti)
221+ return (thisLayer :~~> rest)
249instance (SingI i, RecurrentLayer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (Recurrent x ': xs) (i ': o ': rs) where
223instance (SingI i, RecurrentLayer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (Recurrent x ': xs) (i ': o ': rs) where
250 randomRecurrent = do
224 randomRecurrent = do
251 thisLayer <- createRandom
225 thisLayer <- createRandom
252− thisShape <- randomOfShape
253− (rest, resti) <- randomRecurrent
226+ rest <- randomRecurrent
254− return (thisLayer :~@> rest, thisShape :~@+> resti)
227+ return (thisLayer :~@> rest)
256-- | Add very simple serialisation to the recurrent network
229-- | Add very simple serialisation to the recurrent network
257instance SingI i => Serialize (RecurrentNetwork '[] '[i]) where
230instance SingI i => Serialize (RecurrentNetwork '[] '[i]) where
@@ -270,23 +243,13 @@ instance (Serialize (RecurrentInputs '[])) where
270 put _ = return ()
243 put _ = return ()
271 get = return RINil
244 get = return RINil
273−instance (UpdateLayer x, Serialize (RecurrentInputs ys)) => (Serialize (RecurrentInputs (FeedForward x ': ys))) where
246+instance (UpdateLayer x, Serialize (RecurrentInputs ys), Fractional (RecurrentInputs ys)) => (Serialize (RecurrentInputs (FeedForward x ': ys))) where
274 put ( () :~~+> rest) = put rest
247 put ( () :~~+> rest) = put rest
275 get = ( () :~~+> ) <$> get
248 get = ( () :~~+> ) <$> get
277−instance (SingI (RecurrentShape x), RecurrentUpdateLayer x, Serialize (RecurrentInputs ys)) => (Serialize (RecurrentInputs (Recurrent x ': ys))) where
250+instance (Serialize (RecurrentShape x), Fractional (RecurrentShape x), RecurrentUpdateLayer x, Serialize (RecurrentInputs ys), Fractional (RecurrentInputs ys)) => (Serialize (RecurrentInputs (Recurrent x ': ys))) where
278− put ( i :~@+> rest ) = do
251+ put ( i :~@+> rest ) = put i >> put rest
280− (S1D x) -> putListOf put . LA.toList . LAS.extract $ x
281− (S2D x) -> putListOf put . LA.toList . LA.flatten . LAS.extract $ x
282− (S3D x) -> putListOf put . LA.toList . LA.flatten . LAS.extract $ x
286− get = do
252+ get = (:~@+>) <$> get <*> get
287− Just i <- fromStorable . V.fromList <$> getListOf get
289− return ( i :~@+> rest)
292-- Num instance for `RecurrentInputs layers`
255-- Num instance for `RecurrentInputs layers`
@@ -307,7 +270,7 @@ instance (Num (RecurrentInputs '[])) where
307 signum _ = RINil
270 signum _ = RINil
308 fromInteger _ = RINil
271 fromInteger _ = RINil
310−instance (UpdateLayer x, Num (RecurrentInputs ys)) => (Num (RecurrentInputs (FeedForward x ': ys))) where
273+instance (UpdateLayer x, Fractional (RecurrentInputs ys)) => (Num (RecurrentInputs (FeedForward x ': ys))) where
311 (+) (() :~~+> x) (() :~~+> y) = () :~~+> (x + y)
274 (+) (() :~~+> x) (() :~~+> y) = () :~~+> (x + y)
312 (-) (() :~~+> x) (() :~~+> y) = () :~~+> (x - y)
275 (-) (() :~~+> x) (() :~~+> y) = () :~~+> (x - y)
313 (*) (() :~~+> x) (() :~~+> y) = () :~~+> (x * y)
276 (*) (() :~~+> x) (() :~~+> y) = () :~~+> (x * y)
@@ -315,10 +278,51 @@ instance (UpdateLayer x, Num (RecurrentInputs ys)) => (Num (RecurrentInputs (Fee
315 signum (() :~~+> x) = () :~~+> signum x
278 signum (() :~~+> x) = () :~~+> signum x
316 fromInteger x = () :~~+> fromInteger x
279 fromInteger x = () :~~+> fromInteger x
318−instance (SingI (RecurrentShape x), RecurrentUpdateLayer x, Num (RecurrentInputs ys)) => (Num (RecurrentInputs (Recurrent x ': ys))) where
281+instance (Fractional (RecurrentShape x), RecurrentUpdateLayer x, Fractional (RecurrentInputs ys)) => (Num (RecurrentInputs (Recurrent x ': ys))) where
319 (+) (x :~@+> x') (y :~@+> y') = (x + y) :~@+> (x' + y')
282 (+) (x :~@+> x') (y :~@+> y') = (x + y) :~@+> (x' + y')
320 (-) (x :~@+> x') (y :~@+> y') = (x - y) :~@+> (x' - y')
283 (-) (x :~@+> x') (y :~@+> y') = (x - y) :~@+> (x' - y')
321 (*) (x :~@+> x') (y :~@+> y') = (x * y) :~@+> (x' * y')
284 (*) (x :~@+> x') (y :~@+> y') = (x * y) :~@+> (x' * y')
322 abs (x :~@+> x') = abs x :~@+> abs x'
285 abs (x :~@+> x') = abs x :~@+> abs x'
323 signum (x :~@+> x') = signum x :~@+> signum x'
286 signum (x :~@+> x') = signum x :~@+> signum x'
324 fromInteger x = fromInteger x :~@+> fromInteger x
287 fromInteger x = fromInteger x :~@+> fromInteger x
289+instance (Fractional (RecurrentInputs '[])) where
292+ fromRational _ = RINil
294+instance (UpdateLayer x, Fractional (RecurrentInputs ys)) => (Fractional (RecurrentInputs (FeedForward x ': ys))) where
295+ (/) (() :~~+> x) (() :~~+> y) = () :~~+> (x / y)
296+ recip (() :~~+> x) = () :~~+> recip x
297+ fromRational x = () :~~+> fromRational x
299+instance (Fractional (RecurrentShape x), RecurrentUpdateLayer x, Fractional (RecurrentInputs ys)) => (Fractional (RecurrentInputs (Recurrent x ': ys))) where
300+ (/) (x :~@+> x') (y :~@+> y') = (x / y) :~@+> (x' / y')
301+ recip (x :~@+> x') = recip x :~@+> recip x'
302+ fromRational x = fromRational x :~@+> fromRational x
305+-- | Ultimate composition.
307+-- This allows a complete network to be treated as a layer in a larger network.
308+instance CreatableRecurrent sublayers subshapes => UpdateLayer (RecurrentNetwork sublayers subshapes) where
309+ type Gradient (RecurrentNetwork sublayers subshapes) = RecurrentGradient sublayers
310+ runUpdate = applyRecurrentUpdate
311+ createRandom = randomRecurrent
313+-- | Ultimate composition.
315+-- This allows a complete network to be treated as a layer in a larger network.
316+instance CreatableRecurrent sublayers subshapes => RecurrentUpdateLayer (RecurrentNetwork sublayers subshapes) where
317+ type RecurrentShape (RecurrentNetwork sublayers subshapes) = RecurrentInputs sublayers
319+-- | Ultimate composition.
321+-- This allows a complete network to be treated as a layer in a larger network.
322+instance ( CreatableRecurrent sublayers subshapes
323+ , i ~ (Head subshapes), o ~ (Last subshapes)
324+ , Num (RecurrentShape (RecurrentNetwork sublayers subshapes))
325+ ) => RecurrentLayer (RecurrentNetwork sublayers subshapes) i o where
326+ type RecTape (RecurrentNetwork sublayers subshapes) i o = RecurrentTape sublayers subshapes
327+ runRecurrentForwards = runRecurrent
328+ runRecurrentBackwards = runRecurrent'