From 79f82a54ac407657360a94b8ebe40b196289d25c Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 07:51:17 -0700 Subject: [PATCH 01/11] Move to a nix-based setup With nix, we get more reproducable builds. Everything that goes into making the build can be specified. The dependencies we need for development/CI can be made the same. And, we ought to be able to parameterize the build better. When we want to check if things compile on 8.4.x, we can change the compiler to that version in one place. Assuming this works out, it should make for a lower amount of overall work. --- Makefile | 74 ++++++++++++++++++++++------------------------------- default.nix | 21 +++++++++++++++ shell.nix | 43 +++++++++++++++++++++++++++++++ stack.yaml | 66 ----------------------------------------------- 4 files changed, 95 insertions(+), 109 deletions(-) create mode 100644 default.nix create mode 100644 shell.nix delete mode 100644 stack.yaml diff --git a/Makefile b/Makefile index f2ea6b6..31ba4fa 100644 --- a/Makefile +++ b/Makefile @@ -1,64 +1,52 @@ -BIN ?= bin CABAL ?= cabal +CABAL_FLAGS ?= DIST ?= dist -EMPTY ?= .make -PROJECT_NAME ?= rollbar-hs -STACK ?= stack -STACK_WORK ?= .stack-work -VERBOSITY ?= warn +GHCID ?= ghcid +GHCID_FLAGS ?= --ghc-options=-fno-code +HPACK ?= hpack +NIX_SHELL ?= nix-shell +NIX_SHELL_FLAGS ?= +PROJECT_NAME := rollbar-hs CABAL_FILE := $(PROJECT_NAME).cabal +CONFIGURE := $(DIST)/setup-config DOC_TEST := $(DIST)/build/doc-test/doc-test -GHCID := $(BIN)/ghcid -STACK_FLAGS := --verbosity $(VERBOSITY) .DEFAULT_GOAL := build -$(BIN) $(DIST) $(EMPTY): - mkdir -p $@ - $(CABAL_FILE): package.yaml - # `stack` has no way to run `hpack` directly. - # We can run `hpack` indirectly with little overhead. - $(STACK) $(STACK_FLAGS) build --dry-run + $(HPACK) -$(DOC_TEST): - rm -f $(EMPTY)/build - $(MAKE) $(EMPTY)/build +$(CONFIGURE): $(CABAL_FILE) + $(CABAL) $(CABAL_FLAGS) configure --enable-tests -$(EMPTY)/build: $(EMPTY)/stack-setup README.md Setup.hs package.yaml stack.yaml src/**/*.hs test/**/*.hs | $(DIST) - $(STACK) $(STACK_FLAGS) build --no-run-tests --test - cp -R $$($(STACK) $(STACK_FLAGS) path --dist-dir)/build $(DIST) - touch $@ +.PHONY: build +build $(DOC_TEST): $(CONFIGURE) default.nix + $(CABAL) $(CABAL_FLAGS) build -$(EMPTY)/stack-setup: | $(EMPTY) - $(STACK) $(STACK_FLAGS) setup - touch $@ - -$(GHCID): $(EMPTY)/stack-setup | $(BIN) - $(STACK) $(STACK_FLAGS) install ghcid --local-bin-path $(BIN) - -.PHONT: build -build: $(EMPTY)/build - -.PHONY: cabal-check -cabal-check: $(CABAL_FILE) - $(CABAL) check +.PHONY: check +check: $(CABAL_FILE) + $(CABAL) $(CABAL_FLAGS) check .PHONY: clean clean: rm -f $(CABAL_FILE) - rm -fr $(BIN) + rm -f default.nix rm -fr $(DIST) - rm -fr $(EMPTY) - rm -fr $(STACK_WORK) + +default.nix: $(CABAL_FILE) + cabal2nix . > $@ .PHONY: sdist -sdist: cabal-check | $(DIST) - $(CABAL) sdist +sdist: check + $(CABAL) $(CABAL_FLAGS) sdist + +.PHONY: shell +shell: + $(NIX_SHELL) --pure $(NIX_SHELL_FLAGS) .PHONY: test -test: $(EMPTY)/build test-doc-test +test: test-doc-test .PHONY: test-doc-test test-doc-test: $(DOC_TEST) @@ -66,8 +54,8 @@ test-doc-test: $(DOC_TEST) .PHONY: upload-hackage upload-hackage: sdist - @ $(CABAL) upload $(DIST)/$(PROJECT_NAME)-*.tar.gz + @ $(CABAL) $(CABAL_FLAGS) upload $(DIST)/$(PROJECT_NAME)-*.tar.gz .PHONY: watch -watch: $(GHCID) - $(GHCID) +watch: $(CONFIGURE) + $(GHCID) --command "cabal repl lib:$(PROJECT_NAME) $(GHCID_FLAGS)" diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..b4daf61 --- /dev/null +++ b/default.nix @@ -0,0 +1,21 @@ +{ mkDerivation, aeson, base, bytestring, case-insensitive, hostname +, hspec, hspec-golden-aeson, http-client, http-conduit, http-types +, network, QuickCheck, stdenv, text, time, unordered-containers +, uuid +}: +mkDerivation { + pname = "rollbar-hs"; + version = "0.3.1.0"; + src = ./.; + libraryHaskellDepends = [ + aeson base bytestring case-insensitive hostname http-client + http-conduit http-types network text time unordered-containers uuid + ]; + testHaskellDepends = [ + aeson base bytestring case-insensitive hspec hspec-golden-aeson + QuickCheck text unordered-containers + ]; + homepage = "https://github.com/joneshf/rollbar-hs#readme"; + description = "Core Rollbar data types and APIs"; + license = stdenv.lib.licenses.bsd3; +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..cd3fd44 --- /dev/null +++ b/shell.nix @@ -0,0 +1,43 @@ +{ compiler ? "ghc822" }: + +let + + default = haskell-packages.callPackage (import ./default.nix) {}; + + derivation = nixpkgs.haskell.lib.addBuildTools default tools; + + haskell-packages = nixpkgs.haskell.packages.${compiler}; + + nixpkgs = import nixpkgs-tarball {}; + + nixpkgs-revision = "120b013e0c082d58a5712cde0a7371ae8b25a601"; + + nixpkgs-sha256 = "0hk4y2vkgm1qadpsm4b0q1vxq889jhxzjx3ragybrlwwg54mzp4f"; + + nixpkgs-tarball = builtins.fetchTarball { + sha256 = nixpkgs-sha256; + url = "https://github.com/NixOS/nixpkgs/archive/${nixpkgs-revision}.tar.gz"; + }; + + # Tools affected by the compiler version. + tools-compiler = [ + haskell-packages.cabal-install + haskell-packages.ghc + haskell-packages.ghcid + haskell-packages.hpack + ]; + + # Tools that deal with the basic project infrastructure. + tools-infrastructure = [ + nixpkgs.cabal2nix + nixpkgs.curl + ]; + + # All of the tools. + tools = + tools-compiler ++ + tools-infrastructure; + +in + + derivation.env diff --git a/stack.yaml b/stack.yaml deleted file mode 100644 index 35d283a..0000000 --- a/stack.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# This file was automatically generated by 'stack init' -# -# Some commonly used options have been documented as comments in this file. -# For advanced use and comprehensive documentation of the format, please see: -# https://docs.haskellstack.org/en/stable/yaml_configuration/ - -# Resolver to choose a 'specific' stackage snapshot or a compiler version. -# A snapshot resolver dictates the compiler version and the set of packages -# to be used for project dependencies. For example: -# -# resolver: lts-3.5 -# resolver: nightly-2015-09-21 -# resolver: ghc-7.10.2 -# resolver: ghcjs-0.1.0_ghc-7.10.2 -# resolver: -# name: custom-snapshot -# location: "./custom-snapshot.yaml" -resolver: lts-10.5 - -# User packages to be built. -# Various formats can be used as shown in the example below. -# -# packages: -# - some-directory -# - https://example.com/foo/bar/baz-0.0.2.tar.gz -# - location: -# git: https://github.com/commercialhaskell/stack.git -# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a -# - location: https://github.com/commercialhaskell/stack/commit/e7b331f14bcffb8367cd58fbfc8b40ec7642100a -# extra-dep: true -# subdirs: -# - auto-update -# - wai -# -# A package marked 'extra-dep: true' will only be built if demanded by a -# non-dependency (i.e. a user package), and its test suites and benchmarks -# will not be run. This is useful for tweaking upstream packages. -packages: -- . -# Dependency packages to be pulled from upstream that are not in the resolver -# (e.g., acme-missiles-0.3) -# extra-deps: [] - -# Override default flag values for local packages and extra-deps -# flags: {} - -# Extra package databases containing global packages -# extra-package-dbs: [] - -# Control whether we use the GHC we find on the path -# system-ghc: true -# -# Require a specific version of stack, using version ranges -# require-stack-version: -any # Default -# require-stack-version: ">=1.6" -# -# Override the architecture used by stack, especially useful on Windows -# arch: i386 -# arch: x86_64 -# -# Extra directories used by stack for building -# extra-include-dirs: [/path/to/dir] -# extra-lib-dirs: [/path/to/dir] -# -# Allow a newer minor version of GHC than the snapshot specifies -# compiler-check: newer-minor \ No newline at end of file From 4b9cbbf9eb126a7f3fbb020ada6b8fa6df138e2a Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 13:45:29 -0700 Subject: [PATCH 02/11] Add `which` to the tools This is useful for debugging things. --- shell.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shell.nix b/shell.nix index cd3fd44..0f05588 100644 --- a/shell.nix +++ b/shell.nix @@ -31,6 +31,7 @@ let tools-infrastructure = [ nixpkgs.cabal2nix nixpkgs.curl + nixpkgs.which ]; # All of the tools. From 038c2842198fbf4bd8a83085c5ccec757d3aef6a Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 13:46:06 -0700 Subject: [PATCH 03/11] Update CI configuration --- .circleci/config.yml | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 389ff30..58899d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,39 +2,37 @@ version: 2 jobs: compile: docker: - - image: haskell:8.2.2 + - image: nixorg/nix:circleci steps: - checkout - restore_cache: keys: - - v1-stack-{{ checksum "package.yaml" }}-{{ checksum "stack.yaml" }} - - v1-stack-{{ checksum "package.yaml" }}- - - v1-stack- + - v1-nix-{{ checksum "package.yaml" }} + - v1-nix- name: Restoring stack cache - run: name: Compile - command: make build + command: nix-shell --run 'make build' - persist_to_workspace: root: . paths: - dist - save_cache: - key: v1-stack-{{ checksum "package.yaml" }}-{{ checksum "stack.yaml" }} - name: Caching stack + key: v1-nix-{{ checksum "package.yaml" }} + name: Caching nix paths: - - .stack-work - - /root/.stack + - /nix doc-test: docker: - - image: haskell:8.2.2 + - image: nixorg/nix:circleci steps: - checkout - attach_workspace: at: . - run: name: Doc test - command: make test-doc-test + command: nix-shell --run 'make test-doc-test' for-github: docker: @@ -46,18 +44,17 @@ jobs: sdist: docker: - - image: haskell:8.2.2 + - image: nixorg/nix:circleci steps: - checkout - restore_cache: keys: - - v1-stack-{{ checksum "package.yaml" }}-{{ checksum "stack.yaml" }} - - v1-stack-{{ checksum "package.yaml" }}- - - v1-stack- + - v1-nix-{{ checksum "package.yaml" }} + - v1-nix- name: Restoring stack cache - run: name: Build sdist - command: make sdist + command: nix-shell --run 'make sdist' workflows: version: 2 From 1e3e0f2a50c166a6385bb7caeabdf2a6f2b16d49 Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 21:58:03 -0700 Subject: [PATCH 04/11] Run everything in one job We want to start parameterizing jobs on the version of GHC. This is the first step. Should also make things quicker as there's no interleaving of jobs. --- .circleci/config.yml | 42 ++++-------------------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 58899d1..3b12caf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2 jobs: - compile: + ghc-8.2.2: docker: - image: nixorg/nix:circleci steps: @@ -12,28 +12,13 @@ jobs: name: Restoring stack cache - run: name: Compile - command: nix-shell --run 'make build' - - persist_to_workspace: - root: . - paths: - - dist + command: nix-shell --run 'make sdist test -j2' - save_cache: key: v1-nix-{{ checksum "package.yaml" }} name: Caching nix paths: - /nix - doc-test: - docker: - - image: nixorg/nix:circleci - steps: - - checkout - - attach_workspace: - at: . - - run: - name: Doc test - command: nix-shell --run 'make test-doc-test' - for-github: docker: - image: alpine @@ -42,30 +27,11 @@ jobs: name: Passed command: echo passed - sdist: - docker: - - image: nixorg/nix:circleci - steps: - - checkout - - restore_cache: - keys: - - v1-nix-{{ checksum "package.yaml" }} - - v1-nix- - name: Restoring stack cache - - run: - name: Build sdist - command: nix-shell --run 'make sdist' - workflows: version: 2 base: jobs: - - compile - for-github: requires: - - doc-test - - sdist - - sdist - - doc-test: - requires: - - compile + - ghc-8.2.2 + - ghc-8.2.2 From af3c7d6935f49f28c3a2f22fcfe7bb428182e938 Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 22:18:06 -0700 Subject: [PATCH 05/11] Cache the dist directory as well --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b12caf..bd7e882 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: keys: - v1-nix-{{ checksum "package.yaml" }} - v1-nix- - name: Restoring stack cache + name: Restoring cache - run: name: Compile command: nix-shell --run 'make sdist test -j2' @@ -18,6 +18,7 @@ jobs: name: Caching nix paths: - /nix + - dist for-github: docker: From 58cd90b10346d64ebba56c315fb68caa6e1f73ff Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 22:23:50 -0700 Subject: [PATCH 06/11] Bust the cache --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd7e882..738ca6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,8 @@ jobs: - checkout - restore_cache: keys: + - v2-nix-{{ checksum "package.yaml" }} + - v2-nix- - v1-nix-{{ checksum "package.yaml" }} - v1-nix- name: Restoring cache @@ -14,7 +16,7 @@ jobs: name: Compile command: nix-shell --run 'make sdist test -j2' - save_cache: - key: v1-nix-{{ checksum "package.yaml" }} + key: v2-nix-{{ checksum "package.yaml" }} name: Caching nix paths: - /nix From 0d599e3ff511ec97a90d8943fb7c86d833d5946b Mon Sep 17 00:00:00 2001 From: joneshf Date: Thu, 9 Aug 2018 22:30:59 -0700 Subject: [PATCH 07/11] Remove old cache keys --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 738ca6a..7d6818e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,6 @@ jobs: keys: - v2-nix-{{ checksum "package.yaml" }} - v2-nix- - - v1-nix-{{ checksum "package.yaml" }} - - v1-nix- name: Restoring cache - run: name: Compile From 3868f307774df41539e052128f01ce8a23f5ae6e Mon Sep 17 00:00:00 2001 From: joneshf Date: Sat, 11 Aug 2018 06:38:00 -0700 Subject: [PATCH 08/11] Don't remove the `default.nix` If we `make clean` outside of the nix shell, we can't necessarily get back in. Certainly, `make shell` doesn't work. But, building `default.nix` might not work if `hpack` doesn't exist. --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 31ba4fa..47405ad 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,6 @@ check: $(CABAL_FILE) .PHONY: clean clean: rm -f $(CABAL_FILE) - rm -f default.nix rm -fr $(DIST) default.nix: $(CABAL_FILE) From d096c8452707297e99779ec57425451e31282158 Mon Sep 17 00:00:00 2001 From: joneshf Date: Sun, 26 Aug 2018 10:27:53 -0700 Subject: [PATCH 09/11] Limit the number of simultaneous jobs We might be trying to overdo it with the number of CPU cores available. If we limit the jobs, it might speed up the build. --- .circleci/config.yml | 2 ++ Makefile | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d6818e..29d5893 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,8 @@ jobs: - run: name: Compile command: nix-shell --run 'make sdist test -j2' + environment: + CABAL_BUILD_FLAGS: --jobs=2 - save_cache: key: v2-nix-{{ checksum "package.yaml" }} name: Caching nix diff --git a/Makefile b/Makefile index 47405ad..3912826 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ CABAL ?= cabal CABAL_FLAGS ?= +CABAL_BUILD_FLAGS ?= DIST ?= dist GHCID ?= ghcid GHCID_FLAGS ?= --ghc-options=-fno-code @@ -22,7 +23,7 @@ $(CONFIGURE): $(CABAL_FILE) .PHONY: build build $(DOC_TEST): $(CONFIGURE) default.nix - $(CABAL) $(CABAL_FLAGS) build + $(CABAL) $(CABAL_FLAGS) build $(CABAL_BUILD_FLAGS) .PHONY: check check: $(CABAL_FILE) From c732a32f65e35bfdcf6ba96cc050dea9af60f7ae Mon Sep 17 00:00:00 2001 From: joneshf Date: Sun, 26 Aug 2018 10:34:35 -0700 Subject: [PATCH 10/11] Use the long flag Full English words are a bit easier to deal with, sometimes. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 29d5893..43ca566 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: name: Restoring cache - run: name: Compile - command: nix-shell --run 'make sdist test -j2' + command: nix-shell --run 'make sdist test --jobs=2' environment: CABAL_BUILD_FLAGS: --jobs=2 - save_cache: From 7ca92c00fca63e691a3a4d3387a1a228e263e3aa Mon Sep 17 00:00:00 2001 From: joneshf Date: Sun, 26 Aug 2018 10:46:20 -0700 Subject: [PATCH 11/11] Choose the image by sha Reproducability and all that. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 43ca566..e2a0b89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: ghc-8.2.2: docker: - - image: nixorg/nix:circleci + - image: nixorg/nix@sha256:96c3f56563a3b9549ac1da12ff7567c39cb6e648afd6a17d4a1a41f0596ad8f6 steps: - checkout - restore_cache: