1ec65a4 · grenade · git.iko.soy

git.iko.soy/grenade

Add stupidly fast col2im

commit1ec65a414f1f3ac5806a582ecf47dbaa94651836 authorHuw Campbell <huw.campbell@ambiata.com> dateTue 13 Dec 2016, 01:11 (10 years ago) parent3830e55

6 files changed+124−38

compare with master

unifiedsplit

bench/bench.hs+7−3 cbits/im2col.c+33−4 cbits/im2col.h+6 src/Grenade/Layers/Convolution.hs+10−10 src/Grenade/Layers/Internal/Convolution.hs+55−10 test/Test/Grenade/Layers/Convolution.hs+13−11

bench/bench.hs+7−3

@@ -11,14 +11,18 @@ main = defaultMain [

1111 , bench "im2col 28x28" $ whnf (im2colUnsafe 5 5 1 1) ((28><28) [1..])

1212 , bench "im2col 100x100" $ whnf (im2colUnsafe 10 10 1 1) ((100><100) [1..])

1313 ]

14− , bgroup "im2col_c" [ bench "im2col_c 3x4" $ whnf (im2col_c 2 2 1 1) ((3><4) [1..])

15− , bench "im2col_c 28x28" $ whnf (im2col_c 5 5 1 1) ((28><28) [1..])

16− , bench "im2col_c 100x100" $ whnf (im2col_c 10 10 1 1) ((100><100) [1..])

14+ , bgroup "im2col_c" [ bench "im2col_c 3x4" $ whnf (im2col 2 2 1 1) ((3><4) [1..])

15+ , bench "im2col_c 28x28" $ whnf (im2col 5 5 1 1) ((28><28) [1..])

16+ , bench "im2col_c 100x100" $ whnf (im2col 10 10 1 1) ((100><100) [1..])

1717 ]

1818 , bgroup "col2im" [ bench "col2im 3x4" $ whnf (col2imUnsafe 2 2 1 1 3 4) ((6><4) [1..])

1919 , bench "col2im 28x28" $ whnf (col2imUnsafe 5 5 1 1 28 28) ((576><25) [1..])

2020 , bench "col2im 100x100" $ whnf (col2imUnsafe 10 10 1 1 100 100) ((8281><100) [1..])

2121 ]

22+ , bgroup "col2im_c" [ bench "col2im_c 3x4" $ whnf (col2im 2 2 1 1 3 4) ((6><4) [1..])

23+ , bench "col2im_c 28x28" $ whnf (col2im 5 5 1 1 28 28) ((576><25) [1..])

24+ , bench "col2im_c 100x100" $ whnf (col2im 10 10 1 1 100 100) ((8281><100) [1..])

25+ ]

2226 , bgroup "poolfw" [ bench "poolforwards 3x4" $ whnf (poolForward 2 2 1 1 2 3) ((3><4) [1..])

2327 , bench "poolforwards 28x28" $ whnf (poolForward 5 5 1 1 4 24) ((28><28) [1..])

2428 , bench "poolforwards 100x100" $ whnf (poolForward 10 10 1 1 91 91) ((100><100) [1..])

cbits/im2col.c+33−4

@@ -12,16 +12,45 @@ void im2col_cpu(const double* data_im, int dataOffset, const int channels,

1212 data_im += dataOffset;

1313 const int output_h = (height - kernel_h) / stride_h + 1;

1414 const int output_w = (width - kernel_w) / stride_w + 1;

15+ const int channel_size = height * width;

1516

17+ for (int fitting_height = 0; fitting_height <= (height - kernel_h); fitting_height += stride_h) {

18+ for (int fitting_width = 0; fitting_width <= (width - kernel_w); fitting_width += stride_w) {

19+ for (int channel = 0; channel < channels; channel++) {

20+ for (int kernel_row = 0; kernel_row < kernel_h; kernel_row++) {

21+ for (int kernel_col = 0; kernel_col < kernel_w; kernel_col++) {

22+ int input_row = fitting_height + kernel_row;

23+ int input_col = fitting_width + kernel_col;

24+ *(data_col++) = data_im[input_row * width + input_col + channel_size * channel];

25+ }

26+ }

27+ }

28+ }

29+ }

30+}

31+

32+void col2im_cpu(const double* data_col, int dataOffset, const int channels,

33+ const int height, const int width, const int kernel_h, const int kernel_w,

34+ const int stride_h, const int stride_w,

35+ double* data_im) {

36+

37+ memset(data_im, 0, height * width * channels * sizeof(double));

38+ data_col += dataOffset;

39+ const int output_h = (height - kernel_h) / stride_h + 1;

40+ const int output_w = (width - kernel_w) / stride_w + 1;

1641 const int channel_size = height * width;

17− for (int channel = channels; channel--; data_im += channel_size) {

18− for (int fitting_height = 0; fitting_height <= (height - kernel_h); fitting_height += stride_h) {

19− for (int fitting_width = 0; fitting_width <= (width - kernel_w); fitting_width += stride_w) {

42+

43+ int offsetRow = 0;

44+ int offsetColumn = 0;

45+

46+ for (int fitting_height = 0; fitting_height <= (height - kernel_h); fitting_height += stride_h) {

47+ for (int fitting_width = 0; fitting_width <= (width - kernel_w); fitting_width += stride_w) {

48+ for (int channel = 0; channel < channels; channel++) {

2049 for (int kernel_row = 0; kernel_row < kernel_h; kernel_row++) {

2150 for (int kernel_col = 0; kernel_col < kernel_w; kernel_col++) {

2251 int input_row = fitting_height + kernel_row;

2352 int input_col = fitting_width + kernel_col;

24− *(data_col++) = data_im[input_row * width + input_col];

53+ data_im[input_row * width + input_col + channel_size * channel] += *(data_col++);

2554 }

2655 }

2756 }

cbits/im2col.h+6

@@ -1,7 +1,13 @@

11#include <stdio.h>

22#include <stdint.h>

3+#include <string.h>

45void im2col_cpu(const double* data_im, int dataOffset, const int channels,

56 const int height, const int width, const int kernel_h, const int kernel_w,

67 const int stride_h, const int stride_w,

78 double* data_col);

10+void col2im_cpu(const double* data_col, int dataOffset, const int channels,

11+ const int height, const int width, const int kernel_h, const int kernel_w,

12+ const int stride_h, const int stride_w,

13+ double* data_im);

src/Grenade/Layers/Convolution.hs+10−10

@@ -155,9 +155,9 @@ instance ( KnownNat kernelRows

155155 sy = fromIntegral $ natVal (Proxy :: Proxy strideCols)

156156 ox = fromIntegral $ natVal (Proxy :: Proxy outputRows)

157157 oy = fromIntegral $ natVal (Proxy :: Proxy outputCols)

158− c = im2col_c kx ky sx sy ex

158+ c = im2col kx ky sx sy ex

159159 mt = c LA.<> ek

160− r = col2vidUnsafe 1 1 1 1 ox oy mt

160+ r = col2vid 1 1 1 1 ox oy mt

161161 rs = fmap (fromJust . create) r

162162 in S3D' $ mkVector rs

163163

@@ -172,17 +172,17 @@ instance ( KnownNat kernelRows

172172 ox = fromIntegral $ natVal (Proxy :: Proxy outputRows)

173173 oy = fromIntegral $ natVal (Proxy :: Proxy outputCols)

174174

175− c = im2col_c kx ky sx sy ex

175+ c = im2col kx ky sx sy ex

176176

177177 eo = vecToList $ fmap extract dEdy

178178 ek = extract kernel

179179

180− vs = vid2colUnsafe 1 1 1 1 ox oy eo

180+ vs = vid2col 1 1 1 1 ox oy eo

181181

182182 kN = fromJust . create $ tr c LA.<> vs

183183 dW = vs LA.<> tr ek

184184

185− xW = col2imUnsafe kx ky sx sy ix iy dW

185+ xW = col2im kx ky sx sy ix iy dW

186186 in (Convolution' kN, S2D' . fromJust . create $ xW)

187187

188188

@@ -214,9 +214,9 @@ instance ( KnownNat kernelRows

214214 ox = fromIntegral $ natVal (Proxy :: Proxy outputRows)

215215 oy = fromIntegral $ natVal (Proxy :: Proxy outputCols)

216216

217− c = vid2colUnsafe kx ky sx sy ix iy ex

217+ c = vid2col kx ky sx sy ix iy ex

218218 mt = c LA.<> ek

219− r = col2vidUnsafe 1 1 1 1 ox oy mt

219+ r = col2vid 1 1 1 1 ox oy mt

220220 rs = fmap (fromJust . create) r

221221 in S3D' $ mkVector rs

222222 runBackwards (Convolution kernel _) (S3D' input) (S3D' dEdy) =

@@ -230,16 +230,16 @@ instance ( KnownNat kernelRows

230230 ox = fromIntegral $ natVal (Proxy :: Proxy outputRows)

231231 oy = fromIntegral $ natVal (Proxy :: Proxy outputCols)

232232

233− c = vid2colUnsafe kx ky sx sy ix iy ex

233+ c = vid2col kx ky sx sy ix iy ex

234234

235235 eo = vecToList $ fmap extract dEdy

236236 ek = extract kernel

237237

238− vs = vid2colUnsafe 1 1 1 1 ox oy eo

238+ vs = vid2col 1 1 1 1 ox oy eo

239239

240240 kN = fromJust . create $ tr c LA.<> vs

241241

242242 dW = vs LA.<> tr ek

243243

244− xW = col2vidUnsafe kx ky sx sy ix iy dW

244+ xW = col2vid kx ky sx sy ix iy dW

245245 in (Convolution' kN, S3D' . mkVector . fmap (fromJust . create) $ xW)

src/Grenade/Layers/Internal/Convolution.hs+55−10

@@ -1,11 +1,13 @@

1−{-# LANGUAGE BangPatterns #-}

21{-# LANGUAGE ForeignFunctionInterface #-}

32module Grenade.Layers.Internal.Convolution (

43 col2vidUnsafe

54 , col2imUnsafe

65 , vid2colUnsafe

76 , im2colUnsafe

8− , im2col_c

7+ , im2col

8+ , col2im

9+ , col2vid

10+ , vid2col

911 , fittingStarts

1012 , unsafeModifyMatrix

1113 ) where

@@ -67,6 +69,36 @@ import System.IO.Unsafe ( unsafePerformIO )

6769-- @

6870--

6971

72+col2vid :: Int -> Int -> Int -> Int -> Int -> Int -> Matrix Double -> [Matrix Double]

73+col2vid kernelRows kernelColumns strideRows strideColumns height width dataCol =

74+ let channels = cols dataCol `div` (kernelRows * kernelColumns)

75+ retMat = col2im_c channels height width kernelRows kernelColumns strideRows strideColumns dataCol

76+ in (\f -> subMatrix (f * height, 0) (height, width) retMat) <$> [0..channels -1]

77+

78+col2im :: Int -> Int -> Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double

79+col2im kernelRows kernelColumns strideRows strideColumns height width dataCol =

80+ let channels = 1

81+ in col2im_c channels height width kernelRows kernelColumns strideRows strideColumns dataCol

82+

83+col2im_c :: Int -> Int -> Int -> Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double

84+col2im_c channels height width kernelRows kernelColumns strideRows strideColumns dataCol =

85+ let vec = flatten dataCol

86+ in unsafePerformIO $ do

87+ outPtr <- mallocForeignPtrArray0 (height * width * channels)

88+ let (inPtr, inOffset, _) = U.unsafeToForeignPtr vec

89+

90+ withForeignPtr inPtr $ \inPtr' ->

91+ withForeignPtr outPtr $ \outPtr' ->

92+ col2im_cpu inPtr' inOffset channels height width kernelRows kernelColumns strideRows strideColumns outPtr'

93+

94+ let matVec = U.unsafeFromForeignPtr outPtr 0 (height * width * channels)

95+ return $ U.matrixFromVector U.RowMajor (height * channels) width matVec

96+

97+foreign import ccall safe

98+ col2im_cpu

99+ :: Ptr Double -> Int -> Int -> Int -> Int -> Int -> Int -> Int -> Int -> Ptr Double -> IO ()

100+

101+

70102-- | col2im function.

71103--

72104-- Takes a column patch, and reconstitutes it into a normal image.

@@ -127,26 +159,36 @@ col2vidUnsafe kernelRows kernelColumns strideRows strideColumns destinationRows

127159

128160 U.unsafeFreezeMatrix dataIm

129161

162+vid2col :: Int -> Int -> Int -> Int -> Int -> Int -> [Matrix Double] -> Matrix Double

163+vid2col kernelRows kernelColumns strideRows strideColumns height width dataVid =

164+ let channels = length dataVid

165+ in im2col_c channels height width kernelRows kernelColumns strideRows strideColumns (foldl1 (===) dataVid)

130166

131−im2col_c :: Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double

132−im2col_c kernelRows kernelColumns strideRows strideColumns dataIm =

133− let height = rows dataIm

167+

168+im2col :: Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double

169+im2col kernelRows kernelColumns strideRows strideColumns dataIm =

170+ let channels = 1

171+ height = rows dataIm

134172 width = cols dataIm

135− vec = flatten dataIm

173+ in im2col_c channels height width kernelRows kernelColumns strideRows strideColumns dataIm

174+

175+im2col_c :: Int -> Int -> Int -> Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double

176+im2col_c channels height width kernelRows kernelColumns strideRows strideColumns dataIm =

177+ let vec = flatten dataIm

136178 rowOut = (height - kernelRows) `div` strideRows + 1

137179 colOut = (width - kernelColumns) `div` strideColumns + 1

138180 kernelSize = kernelRows * kernelColumns

139181 numberOfPatches = rowOut * colOut

140182 in unsafePerformIO $ do

141− outPtr <- mallocForeignPtrArray0 (numberOfPatches * kernelSize)

183+ outPtr <- mallocForeignPtrArray0 (numberOfPatches * kernelSize * channels)

142184 let (inPtr, inOffset, _) = U.unsafeToForeignPtr vec

143185

144186 withForeignPtr inPtr $ \inPtr' ->

145187 withForeignPtr outPtr $ \outPtr' ->

146− im2col_cpu inPtr' inOffset 1 height width kernelRows kernelColumns strideRows strideColumns outPtr'

188+ im2col_cpu inPtr' inOffset channels height width kernelRows kernelColumns strideRows strideColumns outPtr'

147189

148− let matVec = U.unsafeFromForeignPtr outPtr 0 (numberOfPatches * kernelSize)

149− return $ U.matrixFromVector U.RowMajor numberOfPatches kernelSize matVec

190+ let matVec = U.unsafeFromForeignPtr outPtr 0 (numberOfPatches * kernelSize * channels)

191+ return $ U.matrixFromVector U.RowMajor numberOfPatches (kernelSize * channels) matVec

150192

151193foreign import ccall safe

152194 im2col_cpu

@@ -231,3 +273,6 @@ im2colUnsafe kernelRows kernelColumns striderows stridecols dataIm = U.runSTMatr

231273 modifySTRef' inputRowRef (+1)

232274

233275 return dataCol

276+

277+

278+{-# ANN module "HLint: ignore Reduce duplication" #-}

test/Test/Grenade/Layers/Convolution.hs+13−11

@@ -42,7 +42,7 @@ prop_im2col_c = once $

4242 , 5.0, 6.0, 9.0, 10.0

4343 , 6.0, 7.0, 10.0, 11.0

4444 , 7.0, 8.0, 11.0, 12.0 ]

45− out = im2col_c 2 2 1 1 input

45+ out = im2col 2 2 1 1 input

4646 in expected === out

4747

4848prop_im2col_stride = once $

@@ -68,7 +68,7 @@ prop_im2col_c_stride = once $

6868 , 3.0, 4.0, 7.0, 8.0

6969 , 5.0, 6.0, 9.0, 10.0

7070 , 7.0, 8.0, 11.0, 12.0 ]

71− out = im2col_c 2 2 1 2 input

71+ out = im2col 2 2 1 2 input

7272 in expected === out

7373

7474prop_im2col_other = once $

@@ -90,12 +90,12 @@ prop_im2col_c_other = once $

9090 expected = (2><6)

9191 [ 1.0, 2.0, 5.0, 6.0 , 9.0, 10.0

9292 , 3.0, 4.0, 7.0, 8.0 , 11.0 ,12.0 ]

93− out = im2col_c 3 2 1 2 input

93+ out = im2col 3 2 1 2 input

9494 in expected === out

9595

9696prop_im2col_bigger = once $

9797 let input = (7><7) [ 1.0 .. ]

98− in im2colUnsafe 5 5 2 2 input === im2col_c 5 5 2 2 input

98+ in im2colUnsafe 5 5 2 2 input === im2col 5 5 2 2 input

9999

100100-- If there's no overlap (stride is the same size as the kernel)

101101-- then col2im . im2col should be symmetric.

@@ -104,7 +104,7 @@ prop_im2col_sym_on_same_stride = once $

104104 [ 1.0, 2.0, 3.0, 4.0

105105 , 5.0, 6.0, 7.0, 8.0

106106 , 9.0, 10.0, 11.0, 12.0 ]

107− out = col2imUnsafe 3 2 3 2 3 4 . im2colUnsafe 3 2 3 2 $ input

107+ out = col2im 3 2 3 2 3 4 . im2colUnsafe 3 2 3 2 $ input

108108 in input === out

109109

110110-- If there's no overlap (stride is the same size as the kernel)

@@ -114,7 +114,7 @@ prop_im2colunsafe_sym_on_same_stride = once $

114114 [ 1.0, 2.0, 3.0, 4.0

115115 , 5.0, 6.0, 7.0, 8.0

116116 , 9.0, 10.0, 11.0, 12.0 ]

117− out = col2imUnsafe 3 2 3 2 3 4 . im2colUnsafe 3 2 3 2 $ input

117+ out = col2im 3 2 3 2 3 4 . im2colUnsafe 3 2 3 2 $ input

118118 in input === out

119119

120120

@@ -129,7 +129,7 @@ prop_im2col_col2im_additive = once $

129129 [ 1.0, 2.0, 2.0, 1.0

130130 , 2.0, 4.0, 4.0, 2.0

131131 , 1.0, 2.0, 2.0, 1.0 ]

132− out = col2imUnsafe 2 2 1 1 3 4 . im2colUnsafe 2 2 1 1 $ input

132+ out = col2im 2 2 1 1 3 4 . im2colUnsafe 2 2 1 1 $ input

133133 in expected === out

134134

135135prop_simple_conv_forwards = once $

@@ -207,7 +207,8 @@ prop_vid2col_no_stride = once $

207207 , 6.0, 7.0, 10.0, 11.0 , 26.0, 27.0, 30.0, 31.0

208208 , 7.0, 8.0, 11.0, 12.0 , 27.0, 28.0, 31.0, 32.0 ]

209209 out = vid2colUnsafe 2 2 1 1 3 4 input

210− in expected === out

210+ out_c = vid2col 2 2 1 1 3 4 input

211+ in expected === out .&&. expected === out_c

211212

212213prop_vid2col_stride = once $

213214 let input = [(3><4)

@@ -223,8 +224,9 @@ prop_vid2col_stride = once $

223224 , 3.0, 4.0, 7.0, 8.0 , 23.0, 24.0, 27.0, 28.0

224225 , 5.0, 6.0, 9.0, 10.0 , 25.0, 26.0, 29.0, 30.0

225226 , 7.0, 8.0, 11.0, 12.0 , 27.0, 28.0, 31.0, 32.0 ]

226− out = vid2colUnsafe 2 2 1 2 3 4 input

227− in expected === out

227+ out = vid2colUnsafe 2 2 1 2 3 4 input

228+ out_c = vid2col 2 2 1 2 3 4 input

229+ in expected === out .&&. expected === out_c

228230

229231prop_vid2col_invert = once $

230232 let input = [(3><4)

@@ -235,7 +237,7 @@ prop_vid2col_invert = once $

235237 [ 21.0, 22.0, 23.0, 24.0

236238 , 25.0, 26.0, 27.0, 28.0

237239 , 29.0, 30.0, 31.0, 32.0 ] ]

238− out = col2vidUnsafe 3 2 3 2 3 4 . vid2colUnsafe 3 2 3 2 3 4 $ input

240+ out = col2vid 3 2 3 2 3 4 . vid2colUnsafe 3 2 3 2 3 4 $ input

239241 in input === out

240242

241243