518be58 · typeable/stackage2nix

git.iko.soy/typeable/stackage2nix

Fix memory regression

commit518be58a1ea52356a2016310f296f35abd7fa1d2 authorDmitry Bushev <bushevdv@gmail.com> dateThu 18 Jan 2018, 18:08 (9 years ago) parent393cfef

11 files changed+41−105

compare with master

unifiedsplit

.travis.yml−1 CHANGELOG.md+1 src/Distribution/Nixpkgs/Haskell/Stack.hs+7−38 src/LtsHaskell.hs+10−14 src/Runner.hs+10−7 src/Runner/Cli.hs+7−9 stack-ghc-7103.yaml−26 stack-ghc-802.yaml+3−2 stack-ghc-821.yaml−4 stack-ghc-822.yaml+1−2 stackage2nix.cabal+2−2

.travis.yml−1

@@ -14,7 +14,6 @@ cache:

1414

1515env:

1616 - STACK_YAML=stack-ghc-802.yaml

17− - STACK_YAML=stack-ghc-821.yaml

1817 - STACK_YAML=stack-ghc-822.yaml

1918

2019jobs:

CHANGELOG.md+1

@@ -24,3 +24,4 @@

2424- Remove `--do-check-stackage` flag (enabled by default)

2525- Remove `--do-haddock-stackage` flag (enabled by default)

2626- Remove `--no-packages-closure` flag (use `--with-stackage-closure`)

27+- Remove `cabal2nix < 2.7.2` support #45

src/Distribution/Nixpkgs/Haskell/Stack.hs+7−38

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21module Distribution.Nixpkgs.Haskell.Stack where

43import Control.Lens

@@ -9,6 +8,7 @@ import Distribution.Nixpkgs.Fetch

98import Distribution.Nixpkgs.Haskell.Derivation

109import Distribution.Nixpkgs.Haskell.FromCabal as FromCabal

1110import Distribution.Nixpkgs.Haskell.FromStack as FromStack

11+import Distribution.Nixpkgs.Haskell.Hackage as DB

1212import Distribution.Nixpkgs.Haskell.PackageSourceSpec as PackageSourceSpec

1313import Distribution.PackageDescription as PackageDescription

1414import Distribution.System as System

@@ -16,13 +16,6 @@ import Network.URI as URI

1616import Stack.Config

1717

1818

19−newtype HackageDb = HackageDb { fromHackageDB :: T.Text }

20− deriving (Eq, Ord, Show)

21−

22−makePrisms ''HackageDb

23−

24−unHackageDb :: HackageDb -> String

25−unHackageDb = T.unpack . fromHackageDB

2619

2720data StackPackagesConfig = StackPackagesConfig

2821 { _spcHaskellResolver :: HaskellResolver

@@ -36,36 +29,12 @@ data StackPackagesConfig = StackPackagesConfig

3629

3730makeLenses ''StackPackagesConfig

3831

39−-- TODO: cabal2nix versions [2.6.0, 2.7.1, 2.7.2] cause #45 memory usage regression

40−-- Possible fix would be to load 'Distribution.Nixpkgs.Haskell.Hackage.HackageDB'

41−-- beforehand and pass it as an argument

42−getStackPackageFromDb

43− :: Maybe HackageDb

44− -> StackPackage

45− -> IO Package

46−getStackPackageFromDb optHackageDb stackPackage = do

47−#if MIN_VERSION_cabal2nix(2,7,2)

32+getStackPackageFromDb :: DB.HackageDB -> StackPackage -> IO Package

33+getStackPackageFromDb hackageDb stackPackage = do

4834 PackageSourceSpec.getPackage'

4935 False

50− (PackageSourceSpec.loadHackageDB (unHackageDb <$> optHackageDb) Nothing)

51− (stackLocationToSource (stackPackage ^. spLocation) (stackPackage ^. spDir))

52−#elif MIN_VERSION_cabal2nix(2,7,1)

53− hackageDb <- PackageSourceSpec.loadHackageDB (unHackageDb <$> optHackageDb) Nothing

54− PackageSourceSpec.getPackage'

55− False

56− hackageDb

57− (stackLocationToSource (stackPackage ^. spLocation) (stackPackage ^. spDir))

58−#elif MIN_VERSION_cabal2nix(2,6,0)

59− PackageSourceSpec.getPackage

60− False

61− (unHackageDb <$> optHackageDb)

62− Nothing

63− (stackLocationToSource (stackPackage ^. spLocation) (stackPackage ^. spDir))

64−#else

65− PackageSourceSpec.getPackage

66− (unHackageDb <$> optHackageDb)

36+ (pure hackageDb)

6737 (stackLocationToSource (stackPackage ^. spLocation) (stackPackage ^. spDir))

68−#endif

6938

7039stackLocationToSource

7140 :: PackageLocation

@@ -98,11 +67,11 @@ stackLocationToSource pl mCabalDir = case pl of

9867

9968packageDerivation

10069 :: StackPackagesConfig

101− -> Maybe HackageDb

70+ -> DB.HackageDB

10271 -> StackPackage

10372 -> IO Derivation

104−packageDerivation conf optHackageDb stackPackage = do

105− pkg <- getStackPackageFromDb optHackageDb stackPackage

73+packageDerivation conf hackageDb stackPackage = do

74+ pkg <- getStackPackageFromDb hackageDb stackPackage

10675 let

10776 drv = genericPackageDerivation conf pkg

10877 & subpath %~ flip fromMaybe (stackPackage ^. spDir)

src/LtsHaskell.hs+10−14

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21module LtsHaskell where

43import AllCabalHashes

@@ -24,6 +23,7 @@ import Stackage.BuildPlan

2423import Stackage.Types (SystemInfo(..))

2524import System.FilePath as Path

2625

26+import qualified Distribution.Nixpkgs.Haskell.Hackage as DB

2727import qualified Data.Yaml as Yaml

2828import qualified Data.Map as Map

2929import qualified Data.Set as Set

@@ -47,21 +47,17 @@ getPackageFromRepo allCabalHashesPath mSha1Hash pkgId = do

4747 (error (display pkgId ++ ": meta data has no SHA256 hash for the tarball"))

4848 (view (mHashes . at "SHA256") meta)

4949 source = DerivationSource "url" ("mirror://hackage/" ++ display pkgId ++ ".tar.gz") "" tarballSHA256

50−#if MIN_VERSION_cabal2nix(2,6,0)

5150 return $ Package source False pkgDesc

52−#else

53− return $ Package source pkgDesc

54−#endif

5551

56−getPackageFromDb :: Maybe HackageDb -> PackageIdentifier -> IO Package

57−getPackageFromDb mHackageDb pkgId =

58− getStackPackageFromDb mHackageDb

52+getPackageFromDb :: DB.HackageDB -> PackageIdentifier -> IO Package

53+getPackageFromDb hackageDb pkgId =

54+ getStackPackageFromDb hackageDb

5955 $ StackPackage (HackagePackage (T.pack $ Text.display pkgId)) True Nothing

6056

61−loadPackage :: Maybe HackageDb -> FilePath -> Maybe SHA1Hash -> PackageIdentifier -> IO Package

62−loadPackage mHackageDb allCabalHashesPath mSha1Hash pkgId =

57+loadPackage :: DB.HackageDB -> FilePath -> Maybe SHA1Hash -> PackageIdentifier -> IO Package

58+loadPackage hackageDb allCabalHashesPath mSha1Hash pkgId =

6359 getPackageFromRepo allCabalHashesPath mSha1Hash pkgId

64− `catchIOError` const (getPackageFromDb mHackageDb pkgId)

60+ `catchIOError` const (getPackageFromDb hackageDb pkgId)

6561

6662ghcCompilerInfo :: Version -> CompilerInfo

6763ghcCompilerInfo v = CompilerInfo

@@ -77,18 +73,18 @@ buildPlanContainsDependency packageVersions (Dependency depName versionRange) =

7773 maybe False (`withinRange` versionRange) $ Map.lookup depName packageVersions

7874

7975buildPackageSetConfig

80− :: Maybe HackageDb

76+ :: DB.HackageDB

8177 -> FilePath

8278 -> FilePath

8379 -> BuildPlan

8480 -> IO PackageSetConfig

85−buildPackageSetConfig mHackageDb optAllCabalHashes optNixpkgsRepository buildPlan = do

81+buildPackageSetConfig hackageDb optAllCabalHashes optNixpkgsRepository buildPlan = do

8682 nixpkgs <- readNixpkgPackageMap optNixpkgsRepository Nothing

8783 let

8884 systemInfo = bpSystemInfo buildPlan

8985 packageVersions = fmap ppVersion (bpPackages buildPlan) `Map.union` siCorePackages systemInfo

9086 return PackageSetConfig

91− { packageLoader = loadPackage mHackageDb optAllCabalHashes

87+ { packageLoader = loadPackage hackageDb optAllCabalHashes

9288 , targetPlatform = Platform (siArch systemInfo) (siOS systemInfo)

9389 , targetCompiler = ghcCompilerInfo (siGhcVersion systemInfo)

9490 , nixpkgsResolver = resolve (Map.map (Set.map (over path ("pkgs":))) nixpkgs)

src/Runner.hs+10−7

@@ -7,6 +7,8 @@ import Data.Maybe (fromMaybe)

77import Data.Semigroup ((<>))

88import Distribution.Nixpkgs.Haskell.FromStack

99import Distribution.Nixpkgs.Haskell.FromStack.Package

10+import Distribution.Nixpkgs.Haskell.Hackage as DB

11+import Distribution.Nixpkgs.Haskell.PackageSourceSpec (loadHackageDB)

1012import Distribution.Nixpkgs.Haskell.Stack

1113import Distribution.Nixpkgs.Haskell.Stack.PrettyPrinting as PP

1214import Distribution.Version (Version)

@@ -30,6 +32,7 @@ import qualified LtsHaskell as LH

3032run :: IO ()

3133run = do

3234 opts <- execParser pinfo

35+ hackageDb <- loadHackageDB (opts ^. optHackageDb) Nothing

3336 let

3437 -- With optparse-applicative it seems there's no way to make

3538 -- 'StackageOptions' optional for Stackage override and required in

@@ -46,12 +49,12 @@ run = do

4649 let

4750 stackResolver = stackConf ^. scResolver

4851 stackPackagesConfig = mkStackPackagesConfig opts

49− stackConfPackages <- traverse (fmap mkNode . packageDerivation stackPackagesConfig (opts ^. optHackageDb))

52+ stackConfPackages <- traverse (fmap mkNode . packageDerivation stackPackagesConfig hackageDb)

5053 $ stackConf ^. scPackages

5154

5255 case buildTarget of

5356 TargetStackageClosure -> do

54− (buildPlanFile, buildPlan, packageSetConfig) <- loadStackageBuildPlan stackResolver opts sopts

57+ (buildPlanFile, buildPlan, packageSetConfig) <- loadStackageBuildPlan hackageDb stackResolver opts sopts

5558 let

5659 overrideConfig = mkOverrideConfig opts (siGhcVersion $ bpSystemInfo buildPlan)

5760 stackageSetConfig = if opts ^. optTweaks . tExtraDepsRevisionLatest

@@ -84,7 +87,7 @@ run = do

8487 $ PP.overrideHaskellPackages overrideConfig (view nodeDerivation <$> stackConfPackages)

8588

8689 TargetStackagePackages -> do

87− (buildPlanFile, buildPlan, packageSetConfig) <- loadStackageBuildPlan stackResolver opts sopts

90+ (buildPlanFile, buildPlan, packageSetConfig) <- loadStackageBuildPlan hackageDb stackResolver opts sopts

8891 let

8992 overrideConfig = mkOverrideConfig opts (siGhcVersion $ bpSystemInfo buildPlan)

9093 stackageSetConfig = if opts ^. optTweaks . tExtraDepsRevisionLatest

@@ -112,7 +115,7 @@ run = do

112115

113116 -- Generate Stackage packages from resolver

114117 OriginResolver stackResolver -> do

115− (buildPlanFile, buildPlan, packageSetConfig) <- loadStackageBuildPlan stackResolver opts sopts

118+ (buildPlanFile, buildPlan, packageSetConfig) <- loadStackageBuildPlan hackageDb stackResolver opts sopts

116119 nodes <- buildStackagePackages packageSetConfig buildPlan

117120 let overrideConfig = mkOverrideConfig opts (siGhcVersion $ bpSystemInfo buildPlan)

118121 writeOutFile buildPlanFile (opts ^. optOutStackagePackages)

@@ -122,12 +125,12 @@ run = do

122125 writeOutFile buildPlanFile (opts ^. optOutDerivation)

123126 $ PP.pPrintHaskellPackages overrideConfig

124127

125−loadStackageBuildPlan :: StackResolver -> Options -> StackageOptions -> IO (FilePath, BuildPlan, PackageSetConfig)

126−loadStackageBuildPlan stackResolver opts sopts = do

128+loadStackageBuildPlan :: DB.HackageDB -> StackResolver -> Options -> StackageOptions -> IO (FilePath, BuildPlan, PackageSetConfig)

129+loadStackageBuildPlan hackageDb stackResolver opts sopts = do

127130 let buildPlanFile = LH.buildPlanFilePath (sopts ^. soptLtsHaskellRepo) stackResolver

128131 buildPlan <- LH.loadBuildPlan buildPlanFile

129132 packageSetConfig <- LH.buildPackageSetConfig

130− (opts ^. optHackageDb)

133+ hackageDb

131134 (sopts ^. soptAllCabalHashesRepo)

132135 (opts ^. optNixpkgsRepository)

133136 buildPlan

src/Runner/Cli.hs+7−9

@@ -5,7 +5,6 @@ import Data.Monoid ( (<>) )

55import Data.Text as T

66import qualified Distribution.Compat.ReadP as P

77import Distribution.Compiler as Compiler

8−import Distribution.Nixpkgs.Haskell.Stack

98import Distribution.System as System

109import qualified Distribution.Text as Text

1110import Options.Applicative as Opts

@@ -55,7 +54,7 @@ data Options = Options

5554 , _optDoCheckPackages :: Bool

5655 , _optDoHaddockPackages :: Bool

5756 , _optTweaks :: Tweaks

58− , _optHackageDb :: Maybe HackageDb

57+ , _optHackageDb :: Maybe FilePath

5958 , _optNixpkgsRepository :: FilePath

6059 , _optCompilerId :: CompilerId

6160 , _optPlatform :: Platform

@@ -83,7 +82,7 @@ options = Options

8382 <*> doCheckPackages

8483 <*> doHaddockPackages

8584 <*> tweaks

86− <*> optional hackageDb

85+ <*> optional optionHackageDb

8786 <*> nixpkgsRepository

8887 <*> compilerId

8988 <*> platform

@@ -192,12 +191,11 @@ configOrigin =

192191

193192-- required for cabal2nix

194193

195−hackageDb :: Parser HackageDb

196−hackageDb = HackageDb <$>

197− option text

198− ( long "hackage-db"

199− <> metavar "HACKAGE_DB"

200− <> help "path to the local hackage db in tar format" )

194+optionHackageDb :: Parser FilePath

195+optionHackageDb = option str

196+ ( long "hackage-db"

197+ <> metavar "HACKAGE_DB"

198+ <> help "path to the local hackage db in tar format" )

201199

202200cabalFlag :: Parser CabalFlag

203201cabalFlag = CabalFlag <$>

stack-ghc-7103.yamldeleted−26

@@ -1,26 +0,0 @@

1−resolver: lts-6.35

2−extra-deps:

3− - aeson-1.2.3.0

4− - amazonka-1.4.5

5− - amazonka-core-1.4.5

6− - amazonka-s3-1.4.5

7− - Cabal-2.0.0.2

8− - cabal2nix-2.5

9− - chunked-data-0.3.0

10− - classy-prelude-1.2.0.1

11− - classy-prelude-conduit-1.2.0

12− - conduit-combinators-1.1.1

13− - distribution-nixpkgs-1.1

14− - hopenssl-2.2.1

15− - http-client-0.5.7.1

16− - http-client-tls-0.3.5.1

17− - http-conduit-2.2.3.2

18− - inflections-0.3.0.0

19− - megaparsec-5.3.1

20− - mono-traversable-1.0.4.0

21− - mono-traversable-instances-0.1.0.0

22− - stackage-curator-0.15.1.0

23− - store-0.4.3.2

24− - store-core-0.4.1

25− - th-abstraction-0.2.6.0

26− - th-utilities-0.2.0.1

stack-ghc-802.yaml+3−2

@@ -1,7 +1,8 @@

1−resolver: lts-9.17

1+resolver: lts-9.21

22extra-deps:

33- Cabal-2.0.0.2

4−- cabal2nix-2.5

4+- cabal2nix-2.7.2

5+- hackage-db-2.0

56- stackage-curator-0.15.0.0

67nix:

78 packages: [icu openssl zlib]

stack-ghc-821.yamldeleted−4

@@ -1,4 +0,0 @@

1−resolver: nightly-2017-11-24

2−extra-deps:

3− - cabal2nix-2.5

4− - hackage-db-1.22

stack-ghc-822.yaml+1−2

@@ -1,4 +1,3 @@

11resolver: lts-10.0

22extra-deps:

3− - cabal2nix-2.5

4− - hackage-db-1.22

3+ - cabal2nix-2.7.2

stackage2nix.cabal+2−2

@@ -31,11 +31,11 @@ library

3131 , LtsHaskell

3232 , Paths_stackage2nix

3333 build-depends: base > 4.7 && < 5

34− , Cabal > 1.24

34+ , Cabal > 2

3535 , QuickCheck

3636 , aeson

3737 , bytestring

38− , cabal2nix >= 2.5

38+ , cabal2nix >= 2.7.2

3939 , containers

4040 , deepseq

4141 , distribution-nixpkgs >= 1.1