Merge branch 'master' into staging-next

2024-12-26 04:43:09 +03:00 · 2018-11-26 10:28:00 +01:00 · 2018-11-26 10:28:00 +01:00 · a5de78b7d7
commit a5de78b7d7
parent e41154d1ab 4fef4fbb4b
198 changed files with 2486 additions and 1200 deletions
--- a/nixos/lib/make-system-tarball.nix
+++ b/nixos/lib/make-system-tarball.nix
@ -1,4 +1,4 @@
-{ stdenv, perl, pixz, pathsFromGraph
+{ stdenv, closureInfo, pixz

 , # The file name of the resulting tarball
  fileName ? "nixos-system-${stdenv.hostPlatform.system}"
@ -29,24 +29,28 @@
 , extraInputs ? [ pixz ]
 }:

+let
+  symlinks = map (x: x.symlink) storeContents;
+  objects = map (x: x.object) storeContents;
+in
+
 stdenv.mkDerivation {
  name = "tarball";
  builder = ./make-system-tarball.sh;
-  buildInputs = [ perl ] ++ extraInputs;
+  buildInputs = extraInputs;

-  inherit fileName pathsFromGraph extraArgs extraCommands compressCommand;
+  inherit fileName extraArgs extraCommands compressCommand;

  # !!! should use XML.
  sources = map (x: x.source) contents;
  targets = map (x: x.target) contents;

  # !!! should use XML.
-  objects = map (x: x.object) storeContents;
-  symlinks = map (x: x.symlink) storeContents;
+  inherit symlinks objects;

-  # For obtaining the closure of `storeContents'.
-  exportReferencesGraph =
-    map (x: [("closure-" + baseNameOf x.object) x.object]) storeContents;
+  closureInfo = closureInfo {
+    rootPaths = objects;
+  };

  extension = compressionExtension;
 }
--- a/nixos/lib/make-system-tarball.sh
+++ b/nixos/lib/make-system-tarball.sh
@ -3,7 +3,6 @@ source $stdenv/setup
 sources_=($sources)
 targets_=($targets)

-echo $objects
 objects=($objects)
 symlinks=($symlinks)

@ -14,8 +13,6 @@ stripSlash() {
    if test "${res:0:1}" = /; then res=${res:1}; fi
 }

-touch pathlist
-
 # Add the individual files.
 for ((i = 0; i < ${#targets_[@]}; i++)); do
    stripSlash "${targets_[$i]}"
@ -25,9 +22,9 @@ done


 # Add the closures of the top-level store objects.
+chmod +w .
 mkdir -p nix/store
-storePaths=$(perl $pathsFromGraph closure-*)
-for i in $storePaths; do
+for i in $(< $closureInfo/store-paths); do
    cp -a "$i" "${i:1}"
 done

@ -35,7 +32,7 @@ done
 # TODO tar ruxo
 # Also include a manifest of the closures in a format suitable for
 # nix-store --load-db.
-printRegistration=1 perl $pathsFromGraph closure-* > nix-path-registration
+cp $closureInfo/registration nix-path-registration

 # Add symlinks to the top-level store objects.
 for ((n = 0; n < ${#objects[*]}; n++)); do
--- a/nixos/modules/hardware/raid/hpsa.nix
+++ b/nixos/modules/hardware/raid/hpsa.nix
@ -8,7 +8,7 @@ let
    version = "2.40-13.0";

    src = pkgs.fetchurl {
-      url = "http://downloads.linux.hpe.com/SDR/downloads/MCP/Ubuntu/pool/non-free/${name}_amd64.deb";
+      url = "https://downloads.linux.hpe.com/SDR/downloads/MCP/Ubuntu/pool/non-free/${name}_amd64.deb";
      sha256 = "11w7fwk93lmfw0yya4jpjwdmgjimqxx6412sqa166g1pz4jil4sw";
    };

@ -34,7 +34,7 @@ let

    meta = with lib; {
      description = "HP Smart Array CLI";
-      homepage = http://downloads.linux.hpe.com/SDR/downloads/MCP/Ubuntu/pool/non-free/;
+      homepage = https://downloads.linux.hpe.com/SDR/downloads/MCP/Ubuntu/pool/non-free/;
      license = licenses.unfreeRedistributable;
      platforms = [ "x86_64-linux" ];
      maintainers = with maintainers; [ volth ];
--- a/nixos/modules/misc/ids.nix
+++ b/nixos/modules/misc/ids.nix
@ -175,7 +175,7 @@
      dnsmasq = 141;
      uhub = 142;
      yandexdisk = 143;
-      #collectd = 144; #unused
+      mxisd = 144; # was once collectd
      consul = 145;
      mailpile = 146;
      redmine = 147;
@ -484,7 +484,7 @@
      #dnsmasq = 141; # unused
      uhub = 142;
      #yandexdisk = 143; # unused
-      #collectd = 144; # unused
+      mxisd = 144; # was once collectd
      #consul = 145; # unused
      mailpile = 146;
      redmine = 147;
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@ -560,6 +560,7 @@
  ./services/networking/miredo.nix
  ./services/networking/mstpd.nix
  ./services/networking/murmur.nix
+  ./services/networking/mxisd.nix
  ./services/networking/namecoind.nix
  ./services/networking/nat.nix
  ./services/networking/ndppd.nix
--- a/nixos/modules/profiles/docker-container.nix
+++ b/nixos/modules/profiles/docker-container.nix
@ -15,15 +15,19 @@ in {

  # Create the tarball
  system.build.tarball = pkgs.callPackage ../../lib/make-system-tarball.nix {
-    contents = [];
+    contents = [
+      {
+        source = "${config.system.build.toplevel}/.";
+        target = "./";
+      }
+    ];
    extraArgs = "--owner=0";

    # Add init script to image
-    storeContents = [
-      { object = config.system.build.toplevel + "/init";
-        symlink = "/init";
-      }
-    ] ++ (pkgs2storeContents [ pkgs.stdenv ]);
+    storeContents = pkgs2storeContents [
+      config.system.build.toplevel
+      pkgs.stdenv
+    ];

    # Some container managers like lxc need these
    extraCommands = "mkdir -p proc sys dev";
--- a/nixos/modules/profiles/hardened.nix
+++ b/nixos/modules/profiles/hardened.nix
@ -12,6 +12,8 @@ with lib;

  boot.kernelPackages = mkDefault pkgs.linuxPackages_hardened;

+  nix.allowedUsers = mkDefault [ "@users" ];
+
  security.hideProcessInformation = mkDefault true;

  security.lockKernelModules = mkDefault true;
--- a/nixos/modules/services/mail/rspamd.nix
+++ b/nixos/modules/services/mail/rspamd.nix
@ -45,7 +45,9 @@ let
      else "${config.socket}${maybeOption "mode"}${maybeOption "owner"}${maybeOption "group"}";
  };

-  workerOpts = { name, ... }: {
+  traceWarning = w: x: builtins.trace "[1;31mwarning: ${w}[0m" x;
+
+  workerOpts = { name, options, ... }: {
    options = {
      enable = mkOption {
        type = types.nullOr types.bool;
@ -59,9 +61,18 @@ let
      };
      type = mkOption {
        type = types.nullOr (types.enum [
-          "normal" "controller" "fuzzy_storage" "rspamd_proxy" "lua"
+          "normal" "controller" "fuzzy_storage" "rspamd_proxy" "lua" "proxy"
        ]);
-        description = "The type of this worker";
+        description = ''
+          The type of this worker. The type <literal>proxy</literal> is
+          deprecated and only kept for backwards compatibility and should be
+          replaced with <literal>rspamd_proxy</literal>.
+        '';
+        apply = let
+            from = "services.rspamd.workers.\”${name}\".type";
+            files = options.type.files;
+            warning = "The option `${from}` defined in ${showFiles files} has enum value `proxy` which has been renamed to `rspamd_proxy`";
+          in x: if x == "proxy" then traceWarning warning "rspamd_proxy" else x;
      };
      bindSockets = mkOption {
        type = types.listOf (types.either types.str (types.submodule bindSocketOpts));
--- a/nixos/modules/services/monitoring/apcupsd.nix
+++ b/nixos/modules/services/monitoring/apcupsd.nix
@ -180,7 +180,7 @@ in
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${pkgs.apcupsd}/bin/apcupsd --killpower -f ${configFile}";
-        TimeoutSec = 0;
+        TimeoutSec = "infinity";
        StandardOutput = "tty";
        RemainAfterExit = "yes";
      };
--- a/nixos/modules/services/monitoring/osquery.nix
+++ b/nixos/modules/services/monitoring/osquery.nix
@ -78,7 +78,7 @@ in
        mkdir -p "$(dirname ${escapeShellArg cfg.databasePath})"
      '';
      serviceConfig = {
-        TimeoutStartSec = 0;
+        TimeoutStartSec = "infinity";
        ExecStart = "${pkgs.osquery}/bin/osqueryd --logger_path ${escapeShellArg cfg.loggerPath} --pidfile ${escapeShellArg cfg.pidfile} --database_path ${escapeShellArg cfg.databasePath}";
        KillMode = "process";
        KillSignal = "SIGTERM";
--- a/nixos/modules/services/monitoring/systemhealth.nix
+++ b/nixos/modules/services/monitoring/systemhealth.nix
@ -8,7 +8,7 @@ let
  systemhealth = with pkgs; stdenv.mkDerivation {
    name = "systemhealth-1.0";
    src = fetchurl {
-      url = "http://www.brianlane.com/static/downloads/systemhealth/systemhealth-1.0.tar.bz2";
+      url = "https://www.brianlane.com/downloads/systemhealth/systemhealth-1.0.tar.bz2";
      sha256 = "1q69lz7hmpbdpbz36zb06nzfkj651413n9icx0njmyr3xzq1j9qy";
    };
    buildInputs = [ python ];
--- a/nixos/modules/services/networking/consul.nix
+++ b/nixos/modules/services/networking/consul.nix
@ -185,7 +185,7 @@ in
          PermissionsStartOnly = true;
          User = if cfg.dropPrivileges then "consul" else null;
          Restart = "on-failure";
-          TimeoutStartSec = "0";
+          TimeoutStartSec = "infinity";
        } // (optionalAttrs (cfg.leaveOnStop) {
          ExecStop = "${cfg.package.bin}/bin/consul leave";
        });
--- a/nixos/modules/services/networking/flashpolicyd.nix
+++ b/nixos/modules/services/networking/flashpolicyd.nix
@ -11,7 +11,7 @@ let

    src = pkgs.fetchurl {
      name = "flashpolicyd_v0.6.zip";
-      url = "http://www.adobe.com/content/dotcom/en/devnet/flashplayer/articles/socket_policy_files/_jcr_content/articlePrerequistes/multiplefiles/node_1277808777771/file.res/flashpolicyd_v0.6%5B1%5D.zip";
+      url = "https://download.adobe.com/pub/adobe/devnet/flashplayer/articles/socket_policy_files/flashpolicyd_v0.6.zip";
      sha256 = "16zk237233npwfq1m4ksy4g5lzy1z9fp95w7pz0cdlpmv0fv9sm3";
    };

@ -35,9 +35,9 @@ in
  ###### interface

  options = {
-  
+
    services.flashpolicyd = {
-    
+
      enable = mkOption {
        default = false;
        description =
@ -47,13 +47,13 @@ in
            connections to your server.
          '';
      };
-      
+
      policy = mkOption {
        default =
          ''
            <?xml version="1.0"?>
            <!DOCTYPE cross-domain-policy SYSTEM "/xml/dtds/cross-domain-policy.dtd">
-            <cross-domain-policy> 
+            <cross-domain-policy>
              <site-control permitted-cross-domain-policies="master-only"/>
              <allow-access-from domain="*" to-ports="*" />
            </cross-domain-policy>
--- a/nixos/modules/services/networking/mxisd.nix
+++ b/nixos/modules/services/networking/mxisd.nix
@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.mxisd;
+
+  server = optionalAttrs (cfg.server.name != null) { inherit (cfg.server) name; }
+        // optionalAttrs (cfg.server.port != null) { inherit (cfg.server) port; };
+
+  baseConfig = {
+    matrix.domain = cfg.matrix.domain;
+    key.path = "${cfg.dataDir}/signing.key";
+    storage = {
+      provider.sqlite.database = "${cfg.dataDir}/mxisd.db";
+    };
+  } // optionalAttrs (server != {}) { inherit server; };
+
+  # merges baseConfig and extraConfig into a single file
+  fullConfig = recursiveUpdate baseConfig cfg.extraConfig;
+
+  configFile = pkgs.writeText "mxisd-config.yaml" (builtins.toJSON fullConfig);
+
+in {
+  options = {
+    services.mxisd = {
+      enable = mkEnableOption "mxisd matrix federated identity server";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.mxisd;
+        defaultText = "pkgs.mxisd";
+        description = "The mxisd package to use";
+      };
+
+      dataDir = mkOption {
+        type = types.str;
+        default = "/var/lib/mxisd";
+        description = "Where data mxisd uses resides";
+      };
+
+      extraConfig = mkOption {
+        type = types.attrs;
+        default = {};
+        description = "Extra options merged into the mxisd configuration";
+      };
+
+      matrix = {
+
+        domain = mkOption {
+          type = types.str;
+          description = ''
+            the domain of the matrix homeserver
+          '';
+        };
+
+      };
+
+      server = {
+
+        name = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = ''
+            Public hostname of mxisd, if different from the Matrix domain.
+          '';
+        };
+
+        port = mkOption {
+          type = types.nullOr types.int;
+          default = null;
+          description = ''
+            HTTP port to listen on (unencrypted)
+          '';
+        };
+
+      };
+
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.users = [
+      {
+        name = "mxisd";
+        group = "mxisd";
+        home = cfg.dataDir;
+        createHome = true;
+        shell = "${pkgs.bash}/bin/bash";
+        uid = config.ids.uids.mxisd;
+      }
+    ];
+
+    users.groups = [
+      {
+        name = "mxisd";
+        gid = config.ids.gids.mxisd;
+      }
+    ];
+
+    systemd.services.mxisd = {
+      description = "a federated identity server for the matrix ecosystem";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      # mxisd / spring.boot needs the configuration to be named "application.yaml"
+      preStart = ''
+        config=${cfg.dataDir}/application.yaml
+        cp ${configFile} $config
+        chmod 444 $config
+      '';
+
+      serviceConfig = {
+        Type = "simple";
+        User = "mxisd";
+        Group = "mxisd";
+        ExecStart = "${cfg.package}/bin/mxisd --spring.config.location=${cfg.dataDir}/ --spring.profiles.active=systemd --java.security.egd=file:/dev/./urandom";
+        WorkingDirectory = cfg.dataDir;
+        PermissionsStartOnly = true;
+        SuccessExitStatus = 143;
+        Restart = "on-failure";
+      };
+    };
+  };
+}
--- a/nixos/modules/services/security/tor.nix
+++ b/nixos/modules/services/security/tor.nix
@ -92,6 +92,7 @@ let
  # Hidden services
  + concatStrings (flip mapAttrsToList cfg.hiddenServices (n: v: ''
    HiddenServiceDir ${torDirectory}/onion/${v.name}
+    ${optionalString (v.version != null) "HiddenServiceVersion ${toString v.version}"}
    ${flip concatMapStrings v.map (p: ''
      HiddenServicePort ${toString p.port} ${p.destination}
    '')}
@ -667,6 +668,12 @@ in
                 };
               }));
             };
+
+             version = mkOption {
+               default = null;
+               description = "Rendezvous service descriptor version to publish for the hidden service. Currently, versions 2 and 3 are supported. (Default: 2)";
+               type = types.nullOr (types.enum [ 2 3 ]);
+             };
          };

          config = {
--- a/nixos/modules/services/system/cloud-init.nix
+++ b/nixos/modules/services/system/cloud-init.nix
@ -119,7 +119,7 @@ in
          { Type = "oneshot";
            ExecStart = "${pkgs.cloud-init}/bin/cloud-init init --local";
            RemainAfterExit = "yes";
-            TimeoutSec = "0";
+            TimeoutSec = "infinity";
            StandardOutput = "journal+console";
          };
      };
@ -137,7 +137,7 @@ in
          { Type = "oneshot";
            ExecStart = "${pkgs.cloud-init}/bin/cloud-init init";
            RemainAfterExit = "yes";
-            TimeoutSec = "0";
+            TimeoutSec = "infinity";
            StandardOutput = "journal+console";
          };
      };
@ -153,7 +153,7 @@ in
          { Type = "oneshot";
            ExecStart = "${pkgs.cloud-init}/bin/cloud-init modules --mode=config";
            RemainAfterExit = "yes";
-            TimeoutSec = "0";
+            TimeoutSec = "infinity";
            StandardOutput = "journal+console";
          };
      };
@ -169,7 +169,7 @@ in
          { Type = "oneshot";
            ExecStart = "${pkgs.cloud-init}/bin/cloud-init modules --mode=final";
            RemainAfterExit = "yes";
-            TimeoutSec = "0";
+            TimeoutSec = "infinity";
            StandardOutput = "journal+console";
          };
      };
--- a/nixos/modules/services/web-servers/apache-httpd/mediawiki.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/mediawiki.nix
@ -86,7 +86,7 @@ let
    name= "mediawiki-1.29.1";

    src = pkgs.fetchurl {
-      url = "http://download.wikimedia.org/mediawiki/1.29/${name}.tar.gz";
+      url = "https://releases.wikimedia.org/mediawiki/1.29/${name}.tar.gz";
      sha256 = "03mpazbxvb011s2nmlw5p6dc43yjgl5yrsilmj1imyykm57bwb3m";
    };

@ -311,7 +311,7 @@ in
      description = ''
        Any additional text to be appended to MediaWiki's
        configuration file.  This is a PHP script.  For configuration
-        settings, see <link xlink:href='http://www.mediawiki.org/wiki/Manual:Configuration_settings'/>.
+        settings, see <link xlink:href='https://www.mediawiki.org/wiki/Manual:Configuration_settings'/>.
      '';
    };

--- a/nixos/modules/virtualisation/docker-image.nix
+++ b/nixos/modules/virtualisation/docker-image.nix
@ -17,3 +17,41 @@
  # Socket activated ssh presents problem in Docker.
  services.openssh.startWhenNeeded = false;
 }
+
+# Example usage:
+#
+## default.nix
+# let
+#   nixos = import <nixpkgs/nixos> {
+#     configuration = ./configuration.nix;
+#     system = "x86_64-linux";
+#   };
+# in
+# nixos.config.system.build.tarball
+#
+## configuration.nix
+# { pkgs, config, lib, ... }:
+# {
+#   imports = [
+#     <nixpkgs/nixos/modules/virtualisation/docker-image.nix>
+#     <nixpkgs/nixos/modules/installer/cd-dvd/channel.nix>
+#   ];
+#
+#   documentation.doc.enable = false;
+#
+#   environment.systemPackages = with pkgs; [
+#     bashInteractive
+#     cacert
+#     nix
+#   ];
+# }
+#
+## Run
+# Build the tarball:
+# $ nix-build default.nix
+# Load into docker:
+# $ docker import result/tarball/nixos-system-*.tar.xz nixos-docker
+# Boots into systemd
+# $ docker run --privileged -it nixos-docker /init
+# Log into the container
+# $ docker exec -it <container-name> /run/current-system/sw/bin/bash
--- a/nixos/modules/virtualisation/google-compute-image.nix
+++ b/nixos/modules/virtualisation/google-compute-image.nix
@ -165,7 +165,7 @@ in
      ExecStop = "${gce}/bin/google_metadata_script_runner --debug --script-type shutdown";
      Type = "oneshot";
      RemainAfterExit = true;
-      TimeoutStopSec = 0;
+      TimeoutStopSec = "infinity";
    };
  };

--- a/nixos/tests/hardened.nix
+++ b/nixos/tests/hardened.nix
@ -10,6 +10,7 @@ import ./make-test.nix ({ pkgs, ...} : {
    { users.users.alice = { isNormalUser = true; extraGroups = [ "proc" ]; };
      users.users.sybil = { isNormalUser = true; group = "wheel"; };
      imports = [ ../modules/profiles/hardened.nix ];
+      nix.useSandbox = false;
      virtualisation.emptyDiskImages = [ 4096 ];
      boot.initrd.postDeviceCommands = ''
        ${pkgs.dosfstools}/bin/mkfs.vfat -n EFISYS /dev/vdb
@ -63,5 +64,11 @@ import ./make-test.nix ({ pkgs, ...} : {
        $machine->succeed("mount /dev/disk/by-label/EFISYS /efi");
        $machine->succeed("mountpoint -q /efi"); # now mounted
      };
+
+      # Test Nix dæmon usage
+      subtest "nix-daemon", sub {
+        $machine->fail("su -l nobody -s /bin/sh -c 'nix ping-store'");
+        $machine->succeed("su -l alice -c 'nix ping-store'") =~ "OK";
+      };
    '';
 })
--- a/nixos/tests/mxisd.nix
+++ b/nixos/tests/mxisd.nix
@ -0,0 +1,21 @@
+import ./make-test.nix ({ pkgs, ... } : {
+
+  name = "mxisd";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ mguentner ];
+  };
+
+  nodes = {
+    server_mxisd = args : {
+      services.mxisd.enable = true;
+      services.mxisd.matrix.domain = "example.org";
+    };
+  };
+
+  testScript = ''
+    startAll;
+    $server_mxisd->waitForUnit("mxisd.service");
+    $server_mxisd->waitForOpenPort(8090);
+    $server_mxisd->succeed("curl -Ssf \"http://127.0.0.1:8090/_matrix/identity/api/v1\"")
+  '';
+})
--- a/nixos/tests/rspamd.nix
+++ b/nixos/tests/rspamd.nix
@ -235,6 +235,7 @@ in
      services.rspamd = {
        enable = true;
        postfix.enable = true;
+        workers.rspamd_proxy.type = "proxy";
      };
    };
    testScript = ''
--- a/pkgs/applications/audio/a2jmidid/default.nix
+++ b/pkgs/applications/audio/a2jmidid/default.nix
@ -9,12 +9,12 @@ in stdenv.mkDerivation rec {
  version = "8";

  src = fetchurl {
-    url = "http://repo.or.cz/a2jmidid.git/snapshot/7383d268c4bfe85df9f10df6351677659211d1ca.tar.gz";
+    url = "https://repo.or.cz/a2jmidid.git/snapshot/7383d268c4bfe85df9f10df6351677659211d1ca.tar.gz";
    sha256 = "06dgf5655znbvrd7fhrv8msv6zw8vk0hjqglcqkh90960mnnmwz7";
  };

-  nativeBuildInputs = [ pkgconfig wafHook ];
-  buildInputs = [ makeWrapper alsaLib dbus libjack2 python dbus-python ];
+  nativeBuildInputs = [ pkgconfig makeWrapper wafHook ];
+  buildInputs = [ alsaLib dbus libjack2 python dbus-python ];

  postInstall = ''
    wrapProgram $out/bin/a2j_control --set PYTHONPATH $PYTHONPATH
--- a/pkgs/applications/audio/cadence/default.nix
+++ b/pkgs/applications/audio/cadence/default.nix
@ -1,5 +1,5 @@
 { stdenv
-, fetchurl
+, fetchzip
 , pkgconfig
 , qtbase
 , makeWrapper
@ -12,31 +12,13 @@
  version = "0.9.0";
  pname = "cadence";

-  src = fetchurl {
+  src = fetchzip {
    url = "https://github.com/falkTX/Cadence/archive/v${version}.tar.gz";
-    sha256 = "07z1mnb0bmldb3i31bgw816pnvlvr9gawr51rpx3mhixg5wpiqzb";
+    sha256 = "08vcggypkdfr70v49innahs5s11hi222dhhnm5wcqzdgksphqzwx";
  };

-  buildInputs = [
-    makeWrapper
-    pkgconfig
-    qtbase
-  ];
-
-  apps = [
-    "cadence"
-    "cadence-jacksettings"
-    "cadence-pulse2loopback"
-    "claudia"
-    "cadence-aloop-daemon"
-    "cadence-logs"
-    "cadence-render"
-    "catarina"
-    "claudia-launcher"
-    "cadence-pulse2jack"
-    "cadence-session-start"
-    "catia"
-  ];
+  nativeBuildInputs = [ makeWrapper pkgconfig ];
+  buildInputs = [ qtbase ];

  makeFlags = ''
    PREFIX=""
@ -46,20 +28,54 @@
  propagatedBuildInputs = with python3Packages; [ pyqt5 ];

  postInstall = ''
-    # replace with our own wrappers.
-    for app in $apps; do
-      rm $out/bin/$app
-      makeWrapper ${python3Packages.python.interpreter} $out/bin/$app \
-        --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
-        --add-flags "-O $out/share/cadence/src/$app.py"
-    done
+    # replace with our own wrappers. They need to be changed manually since it wouldn't work otherwise
+    rm $out/bin/cadence
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/cadence \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/cadence.py"
+    rm $out/bin/claudia
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/claudia \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/claudia.py"
+    rm $out/bin/catarina
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/catarina \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/catarina.py"
+    rm $out/bin/catia
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/catia \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/catia.py"
+    rm $out/bin/cadence-jacksettings
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/cadence-jacksettings \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/jacksettings.py"
+    rm $out/bin/cadence-aloop-daemon
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/cadence-aloop-daemon \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/cadence_aloop_daemon.py"
+    rm $out/bin/cadence-logs
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/cadence-logs \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/logs.py"
+    rm $out/bin/cadence-render
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/cadence-render \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/render.py"
+    rm $out/bin/claudia-launcher
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/claudia-launcher \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/claudia_launcher.py"
+    rm $out/bin/cadence-session-start
+    makeWrapper ${python3Packages.python.interpreter} $out/bin/cadence-session-start \
+      --set PYTHONPATH "$PYTHONPATH:$out/share/cadence" \
+      --add-flags "-O $out/share/cadence/src/cadence_session_start.py"
  '';

  meta = {
    homepage = https://github.com/falkTX/Cadence/;
    description = "Collection of tools useful for audio production";
-    license = stdenv.lib.licenses.mit;
+    license = stdenv.lib.licenses.gpl2Plus;
    maintainers = with stdenv.lib.maintainers; [ genesis ];
-    platforms = stdenv.lib.platforms.linux;
+    platforms = [ "x86_64-linux" ];
  };
 }
--- a/pkgs/applications/audio/renoise/default.nix
+++ b/pkgs/applications/audio/renoise/default.nix
@ -1,4 +1,5 @@
-{ stdenv, fetchurl, libX11, libXext, libXcursor, libXrandr, libjack2, alsaLib, releasePath ? null }:
+{ stdenv, fetchurl, libX11, libXext, libXcursor, libXrandr, libjack2, alsaLib,
+  mpg123, makeWrapper, releasePath ? null }:

 with stdenv.lib;

@ -35,6 +36,7 @@ stdenv.mkDerivation rec {
        releasePath
    else throw "Platform is not supported by Renoise";

+  nativeBuildInputs = [ makeWrapper ];
  buildInputs = [ libX11 libXext libXcursor libXrandr alsaLib libjack2 ];

  installPhase = ''
@ -56,11 +58,12 @@ stdenv.mkDerivation rec {
    ln -s $out/renoise $out/bin/renoise

    patchelf --set-interpreter $(cat $NIX_CC/nix-support/dynamic-linker) --set-rpath $out/lib $out/renoise
+    wrapProgram "$out/renoise" --prefix LD_LIBRARY_PATH : "${mpg123}/lib"
  '';

  meta = {
    description = "Modern tracker-based DAW";
-    homepage = http://www.renoise.com/;
+    homepage = https://www.renoise.com/;
    license = licenses.unfree;
    maintainers = [];
    platforms = [ "i686-linux" "x86_64-linux" ];
--- a/pkgs/applications/audio/seq24/default.nix
+++ b/pkgs/applications/audio/seq24/default.nix
@ -5,7 +5,7 @@ stdenv.mkDerivation  rec {
  version = "0.9.3";

  src = fetchurl {
-    url = "http://launchpad.net/seq24/trunk/${version}/+download/${name}.tar.gz";
+    url = "https://launchpad.net/seq24/trunk/${version}/+download/${name}.tar.gz";
    sha256 = "1qpyb7355s21sgy6gibkybxpzx4ikha57a8w644lca6qy9mhcwi3";
  };

--- a/pkgs/applications/audio/synthv1/default.nix
+++ b/pkgs/applications/audio/synthv1/default.nix
@ -2,11 +2,11 @@

 stdenv.mkDerivation rec {
  name = "synthv1-${version}";
-  version = "0.9.2";
+  version = "0.9.3";

  src = fetchurl {
    url = "mirror://sourceforge/synthv1/${name}.tar.gz";
-    sha256 = "1r60l286n8y4a4rrlnbc3h7xk4s2pvqykvskls89prxg0lkpz7kl";
+    sha256 = "0f58k5n2k667q8wsigg7bzl3lfgaf6jdj98r2a5nvyb18v1wpy2c";
  };

  buildInputs = [ qt5.qtbase qt5.qttools libjack2 alsaLib liblo lv2 ];
@ -15,7 +15,7 @@ stdenv.mkDerivation rec {

  meta = with stdenv.lib; {
    description = "An old-school 4-oscillator subtractive polyphonic synthesizer with stereo fx";
-    homepage = http://synthv1.sourceforge.net/;
+    homepage = https://synthv1.sourceforge.io/;
    license = licenses.gpl2Plus;
    platforms = platforms.linux;
    maintainers = [ maintainers.goibhniu ];
--- a/pkgs/applications/audio/transcribe/default.nix
+++ b/pkgs/applications/audio/transcribe/default.nix
@ -1,33 +1,31 @@
-{ stdenv, fetchzip, lib, makeWrapper, alsaLib, atk, cairo, gdk_pixbuf
-, glib, gst-ffmpeg, gst-plugins-bad, gst-plugins-base
-, gst-plugins-good, gst-plugins-ugly, gstreamer, gtk2, libSM, libX11
-, libpng12, pango, zlib }:
+{ stdenv, fetchzip, wrapGAppsHook, alsaLib, atk, cairo, gdk_pixbuf
+, glib, gst_all_1,  gtk3, libSM, libX11, libpng12, pango, zlib }:

 stdenv.mkDerivation rec {
  name = "transcribe-${version}";
-  version = "8.40";
+  version = "8.72";

  src = if stdenv.hostPlatform.system == "i686-linux" then
    fetchzip {
-      url = "https://www.seventhstring.com/xscribe/downlinux32_old/xscsetup.tar.gz";
-      sha256 = "1ngidmj9zz8bmv754s5xfsjv7v6xr03vck4kigzq4bpc9b1fdhjq";
+      url = "https://www.seventhstring.com/xscribe/downlinux32/xscsetup.tar.gz";
+      sha256 = "1h5l7ry9c9awpxfnd29b0wm973ifrhj17xl5d2fdsclw2swsickb";
    }
  else if stdenv.hostPlatform.system == "x86_64-linux" then
    fetchzip {
-      url = "https://www.seventhstring.com/xscribe/downlinux64_old/xsc64setup.tar.gz";
-      sha256 = "0svzi8svj6zn06gj0hr8mpnhq4416dvb4g5al0gpb1g3paywdaf9";
+      url = "https://www.seventhstring.com/xscribe/downlinux64/xsc64setup.tar.gz";
+      sha256 = "1rpd3ppnx5i5yrnfbjrx7h7dk48kwl99i9lnpa75ap7nxvbiznm0";
    }
  else throw "Platform not supported";

-  nativeBuildInputs = [ makeWrapper ];
+  nativeBuildInputs = [ wrapGAppsHook ];

-  buildInputs = [ gst-plugins-base gst-plugins-good
-    gst-plugins-bad gst-plugins-ugly gst-ffmpeg ];
+  buildInputs = with gst_all_1; [ gst-plugins-base gst-plugins-good
+    gst-plugins-bad gst-plugins-ugly ];

  dontPatchELF = true;

-  libPath = lib.makeLibraryPath [
-    stdenv.cc.cc glib gtk2 atk pango cairo gdk_pixbuf alsaLib
+  libPath = with gst_all_1; stdenv.lib.makeLibraryPath [
+    stdenv.cc.cc glib gtk3 atk pango cairo gdk_pixbuf alsaLib
    libX11 libSM libpng12 gstreamer gst-plugins-base zlib
  ];

@ -42,13 +40,18 @@ stdenv.mkDerivation rec {
    patchelf \
      --set-interpreter $(cat ${stdenv.cc}/nix-support/dynamic-linker) \
      $out/libexec/transcribe
+  '';

-    wrapProgram $out/libexec/transcribe \
-      --prefix GST_PLUGIN_SYSTEM_PATH : "$GST_PLUGIN_SYSTEM_PATH" \
+  preFixup = ''
+    gappsWrapperArgs+=(
+      --prefix GST_PLUGIN_SYSTEM_PATH : "$GST_PLUGIN_SYSTEM_PATH_1_0"
      --prefix LD_LIBRARY_PATH : "${libPath}"
+    )
+  '';

+  postFixup = ''
    ln -s $out/libexec/transcribe $out/bin/
-    '';
+  '';

  meta = with stdenv.lib; {
    description = "Software to help transcribe recorded music";
--- a/pkgs/applications/editors/monodevelop/default.nix
+++ b/pkgs/applications/editors/monodevelop/default.nix
@ -15,7 +15,7 @@ stdenv.mkDerivation rec {
  };

  nunit2510 = fetchurl {
-    url = "http://launchpad.net/nunitv2/2.5/2.5.10/+download/NUnit-2.5.10.11092.zip";
+    url = "https://launchpad.net/nunitv2/2.5/2.5.10/+download/NUnit-2.5.10.11092.zip";
    sha256 = "0k5h5bz1p2v3d0w0hpkpbpvdkcszgp8sr9ik498r1bs72w5qlwnc";
  };

--- a/pkgs/applications/graphics/deskew/default.nix
+++ b/pkgs/applications/graphics/deskew/default.nix
@ -0,0 +1,37 @@
+{ stdenv, fetchFromBitbucket, libtiff, fpc }:
+
+stdenv.mkDerivation rec {
+
+  name = "deskew-${version}";
+  version = "1.25";
+
+  src = fetchFromBitbucket {
+    owner = "galfar";
+    repo = "app-deskew";
+    rev = "v${version}";
+    sha256 = "0zjjj66qhgqkmfxl3q7p78dv4xl4ci918pgl4d5259pqdj1bfgc8";
+  };
+
+  nativeBuildInputs = [ fpc ];
+  buildInputs = [ libtiff ];
+
+  buildPhase = ''
+    rm -r Bin # Remove pre-compiled binary
+    mkdir Bin
+    chmod +x compile.sh
+    ./compile.sh
+  '';
+
+  installPhase = ''
+    install -Dt $out/bin Bin/*
+  '';
+
+  meta = with stdenv.lib; {
+    description = "A command line tool for deskewing scanned text documents";
+    homepage = https://bitbucket.org/galfar/app-deskew/overview;
+    license = licenses.mit;
+    maintainers = with maintainers; [ryantm];
+    platforms = platforms.all;
+  };
+
+}
--- a/pkgs/applications/misc/llpp/default.nix
+++ b/pkgs/applications/misc/llpp/default.nix
@ -39,7 +39,7 @@ stdenv.mkDerivation rec {
  '';

  meta = with stdenv.lib; {
-    homepage = http://repo.or.cz/w/llpp.git;
+    homepage = https://repo.or.cz/w/llpp.git;
    description = "A MuPDF based PDF pager written in OCaml";
    platforms = platforms.linux;
    maintainers = with maintainers; [ pSub ];
--- a/pkgs/applications/misc/mako/default.nix
+++ b/pkgs/applications/misc/mako/default.nix
@ -3,13 +3,13 @@

 stdenv.mkDerivation rec {
  name = "mako-${version}";
-  version = "1.1";
+  version = "1.2";

  src = fetchFromGitHub {
    owner = "emersion";
    repo = "mako";
    rev = "v${version}";
-    sha256 = "18krsyp9g6f689024dn1mq8dyj4yg8c3kcy5s88q1gm8py6c4493";
+    sha256 = "112b7s5bkvwlgsm2kng2vh8mn6wr3a6c7n1arl9adxlghdym449h";
  };

  nativeBuildInputs = [ meson ninja pkgconfig scdoc ];
--- a/pkgs/applications/misc/nnn/default.nix
+++ b/pkgs/applications/misc/nnn/default.nix
@ -4,13 +4,13 @@ with stdenv.lib;

 stdenv.mkDerivation rec {
  name = "nnn-${version}";
-  version = "2.0";
+  version = "2.1";

  src = fetchFromGitHub {
    owner = "jarun";
    repo = "nnn";
    rev = "v${version}";
-    sha256 = "16c6fimr1ayb2x3mvli70x2va3nz106jdfyqn53bhss7zjqvszxl";
+    sha256 = "1vkrhsdwgacln335rjywdf7nj7fg1x55szmm8xrvwda8y2qjqhc4";
  };

  configFile = optionalString (conf!=null) (builtins.toFile "nnn.h" conf);
--- a/pkgs/applications/misc/pinfo/default.nix
+++ b/pkgs/applications/misc/pinfo/default.nix
@ -5,7 +5,7 @@ stdenv.mkDerivation {

  src = fetchurl {
    # homepage needed you to login to download the tarball
-    url = "http://src.fedoraproject.org/repo/pkgs/pinfo/pinfo-0.6.10.tar.bz2"
+    url = "https://src.fedoraproject.org/repo/pkgs/pinfo/pinfo-0.6.10.tar.bz2"
      + "/fe3d3da50371b1773dfe29bf870dbc5b/pinfo-0.6.10.tar.bz2";
    sha256 = "0p8wyrpz9npjcbx6c973jspm4c3xz4zxx939nngbq49xqah8088j";
  };
--- a/pkgs/applications/misc/sakura/default.nix
+++ b/pkgs/applications/misc/sakura/default.nix
@ -5,7 +5,7 @@ stdenv.mkDerivation rec {
  version = "3.6.0";

  src = fetchurl {
-    url = "http://launchpad.net/sakura/trunk/${version}/+download/${name}.tar.bz2";
+    url = "https://launchpad.net/sakura/trunk/${version}/+download/${name}.tar.bz2";
    sha256 = "1q463qm41ym7jb3kbzjz7b6x549vmgkb70arpkhsf86yxly1y5m1";
  };

--- a/pkgs/applications/misc/tilda/default.nix
+++ b/pkgs/applications/misc/tilda/default.nix
@ -1,6 +1,6 @@
-{ stdenv, fetchurl, pkgconfig
+{ stdenv, fetchzip, pkgconfig
 , autoreconfHook, gettext, expat
-, confuse, vte, gtk
+, libconfuse, vte, gtk
 , makeWrapper }:

 stdenv.mkDerivation rec {
@ -8,13 +8,13 @@ stdenv.mkDerivation rec {
  name = "tilda-${version}";
  version = "1.4.1";

-  src = fetchurl {
+  src = fetchzip {
    url = "https://github.com/lanoxx/tilda/archive/${name}.tar.gz";
-    sha256 = "0w2hry2bqcqrkik4l100b1a9jlsih6sq8zwhfpl8zzfq20i00lfs";
+    sha256 = "154rsldqjv2m1bddisb930qicb0y35kx7bxq392n2hn68jr2pxkj";
  };

-  nativeBuildInputs = [ autoreconfHook pkgconfig ];
-  buildInputs = [ gettext confuse vte gtk makeWrapper ];
+  nativeBuildInputs = [ autoreconfHook makeWrapper pkgconfig ];
+  buildInputs = [ gettext libconfuse vte gtk ];

  LD_LIBRARY_PATH = "${expat.out}/lib"; # ugly hack for xgettext to work during build

--- a/pkgs/applications/networking/cluster/terragrunt/default.nix
+++ b/pkgs/applications/networking/cluster/terragrunt/default.nix
@ -2,7 +2,7 @@

 buildGoPackage rec {
  name = "terragrunt-${version}";
-  version = "0.17.2";
+  version = "0.17.3";

  goPackagePath = "github.com/gruntwork-io/terragrunt";

@ -10,7 +10,7 @@ buildGoPackage rec {
    owner  = "gruntwork-io";
    repo   = "terragrunt";
    rev    = "v${version}";
-    sha256 = "069l9ynyl96rfs9zw6w6n1yzjjin27731nj1ajr9jsyc8rhd84wv";
+    sha256 = "1b0fwql9nr00qpvcbsbdymxf1wrgr590gkms7yz3yirb4xfl3gl3";
  };

  goDeps = ./deps.nix;
--- a/pkgs/applications/networking/mailreaders/realpine/default.nix
+++ b/pkgs/applications/networking/mailreaders/realpine/default.nix
@ -36,7 +36,7 @@ stdenv.mkDerivation {
    license = stdenv.lib.licenses.asl20;
    maintainers = [stdenv.lib.maintainers.raskin];
    platforms = stdenv.lib.platforms.linux;
-    homepage = http://re-alpine.sf.net/;
-    downloadPage = "http://sourceforge.net/projects/re-alpine/files/";
+    homepage = https://sourceforge.net/projects/re-alpine/;
+    downloadPage = "https://sourceforge.net/projects/re-alpine/files/";
  };
 }
--- a/pkgs/applications/networking/seafile-client/default.nix
+++ b/pkgs/applications/networking/seafile-client/default.nix
@ -5,14 +5,14 @@
 with stdenv.lib;

 stdenv.mkDerivation rec {
-  version = "6.2.5";
+  version = "6.2.7";
  name = "seafile-client-${version}";

  src = fetchFromGitHub {
    owner = "haiwen";
    repo = "seafile-client";
    rev = "v${version}";
-    sha256 = "1g09gsaqr4swgwnjxz994xgjsv7mlfaypp6r37bbsiy89a7ppzfl";
+    sha256 = "16ikl6vkp9v16608bq2sfg48idn2p7ik3q8n6j866zxkmgdvkpsg";
  };

  nativeBuildInputs = [ pkgconfig cmake makeWrapper ];
--- a/pkgs/applications/office/kmymoney/default.nix
+++ b/pkgs/applications/office/kmymoney/default.nix
@ -60,12 +60,11 @@ stdenv.mkDerivation rec {
      "$out/share/kmymoney/weboob/kmymoneyweboob.py"
  '';

-  doInstallCheck = stdenv.hostPlatform == stdenv.buildPlatform;
-  installCheckPhase = let
-    pluginPath = "${qtbase.bin}/${qtbase.qtPluginPrefix}";
-  in lib.optionalString doInstallCheck ''
-    QT_PLUGIN_PATH=${lib.escapeShellArg pluginPath} CTEST_OUTPUT_ON_FAILURE=1 \
-      ${xvfb_run}/bin/xvfb-run -s '-screen 0 1024x768x24' make test \
+  doInstallCheck = true;
+  installCheckInputs = [ xvfb_run ];
+  installCheckPhase = ''
+    QT_PLUGIN_PATH=${lib.escapeShellArg "${qtbase.bin}/${qtbase.qtPluginPrefix}"} \
+      xvfb-run -s '-screen 0 1024x768x24' make test \
      ARGS="-E '(reports-chart-test)'" # Test fails, so exclude it for now.
  '';

--- a/pkgs/applications/science/chemistry/gwyddion/default.nix
+++ b/pkgs/applications/science/chemistry/gwyddion/default.nix
@ -6,7 +6,7 @@ let version = "2.48"; in
 stdenv.mkDerivation {
  name = "gwyddion-${version}";
  src = fetchurl {
-    url = "http://sourceforge.net/projects/gwyddion/files/gwyddion/${version}/gwyddion-${version}.tar.xz";
+    url = "mirror://sourceforge/gwyddion/files/gwyddion/${version}/gwyddion-${version}.tar.xz";
    sha256 = "119iw58ac2wn4cas6js8m7r1n4gmmkga6b1y711xzcyjp9hshgwx";
  };
  nativeBuildInputs = [ pkgconfig ];
--- a/pkgs/applications/science/math/fricas/default.nix
+++ b/pkgs/applications/science/math/fricas/default.nix
@ -8,7 +8,7 @@ stdenv.mkDerivation {
  inherit name;

  src = fetchurl {
-    url    = "http://sourceforge.net/projects/fricas/files/fricas/${version}/${name}-full.tar.bz2";
+    url    = "mirror://sourceforge/fricas/files/fricas/${version}/${name}-full.tar.bz2";
    sha256 = "156k9az1623y5808j845c56z2nvvdrm48dzg1v0ivpplyl7vp57x";
  };

--- a/pkgs/applications/science/math/sage/README.md
+++ b/pkgs/applications/science/math/sage/README.md
@ -2,46 +2,7 @@

 Sage is a pretty complex package that depends on many other complex packages and patches some of those. As a result, the sage nix package is also quite complex.

-Don't feel discouraged to fix, simplify or improve things though. Here's a quick overview over the functions of the individual files:
-
- `sage-src.nix`  
-  Downloads the source code and applies patches. This makes sure that all the other files work with the same sage source. If you want to apply a patch to sage or update sage to a new version, this is the place to do it.
-
- `env-locations.nix`  
-  Creates a bash file that sets a bunch of environment variables telling sage where to find various packages and files. The definitions of those environment variables can be found in the sage source in the `src/env.py` file. This bash file needs to be sourced before sage is started (done in `sage-env.nix` and `sagedoc.nix`).
-
- `sage-env.nix`  
-  Sets all environment variables sage needs to run. This includes the package locations defined in `env-locations.nix` as well as the location of sage itself and its various subdirectories.
-
- `sagelib.nix`  
-  Defines the main sage package (without setting the necessary environments or running any tests).
-
- `sage-with-env.nix`  
-  Wraps sage in the necessary environment.
-
- `sage.nix`  
-  Runs sages doctests.
-
- `sage-wrapper.nix`  
-  Optionally tells sage where do find the docs.
-
- `sagedoc.nix`  
-  Builds and tests the sage html documentation. Can be used for offline documentation viewing as well as the sage `browse_sage_doc` and `search_doc` functions.
-
- `sagenb.nix`  
-  The (semi deprecated) sage notebook.
-
- `default.nix`  
-  Introduces necessary overrides, defines new packages and ties everything together (returning the `sage` package).
-
- `flask-oldsessions.nix`, `flask-openid.nix`, `python-openid.nix`
-  These are python packages that were rejected from the main nixpkgs tree because they appear unmaintained. They are needed for the (semi-deprecated) sage notebook. Since that notebook is still needed to run the sage doctests, these packages are included but not exposed to the rest of nixpkgs.
-
- `pybrial.nix`  
-  pybrial is a dependency of sage. However, pybrial itself also has sage as a dependency. Because of that circular dependency, pybrial is hidden from the rest of nixpkgs (just as the flask packages and python-openid.
-
- `openblas-pc.nix`  
-  This creates a `.pc` file to be read by `pkg-config` that allows openblas to take on different roles, like `cblas` or `lapack`.
+Don't feel discouraged to fix, simplify or improve things though. The individual files have comments explaining their purpose. The most importent ones are `default.nix` linking everything together, `sage-src.nix` adding patches and `sagelib.nix` building the actual sage package.

 ## The sage build is broken

--- a/pkgs/applications/science/math/sage/default.nix
+++ b/pkgs/applications/science/math/sage/default.nix
@ -1,68 +1,101 @@
-{ nixpkgs
+{ pkgs
 , withDoc ? false
 }:

+# Here sage and its dependencies are put together. Some dependencies may be pinned
+# as a last resort. Patching sage for compatibility with newer dependency versions
+# is always preferred, see `sage-src.nix` for that.
+
 let
-  inherit (nixpkgs) fetchpatch fetchurl symlinkJoin callPackage nodePackages;
+  inherit (pkgs) fetchurl symlinkJoin callPackage nodePackages;

  # https://trac.sagemath.org/ticket/15980 for tracking of python3 support
-  python = nixpkgs.python2.override {
+  python = pkgs.python2.override {
    packageOverrides = self: super: {
      # python packages that appear unmaintained and were not accepted into the nixpkgs
      # tree because of that. These packages are only dependencies of the more-or-less
      # deprecated sagenb. However sagenb is still a default dependency and the doctests
      # depend on it.
      # See https://github.com/NixOS/nixpkgs/pull/38787 for a discussion.
+      # The dependency on the sage notebook (and therefore these packages) will be
+      # removed in the future:
+      # https://trac.sagemath.org/ticket/25837
      flask-oldsessions = self.callPackage ./flask-oldsessions.nix {};
      flask-openid = self.callPackage ./flask-openid.nix {};
      python-openid = self.callPackage ./python-openid.nix {};
-
-      pybrial = self.callPackage ./pybrial.nix {};
-
-      sagelib = self.callPackage ./sagelib.nix {
-        inherit flint ecl arb;
-        inherit sage-src openblas-blas-pc openblas-cblas-pc openblas-lapack-pc pynac singular;
-        linbox = nixpkgs.linbox.override { withSage = true; };
-      };
-
      sagenb = self.callPackage ./sagenb.nix {
        mathjax = nodePackages.mathjax;
      };

-      sagedoc = self.callPackage ./sagedoc.nix {
-        inherit sage-src;
-      };
+      # Package with a cyclic dependency with sage
+      pybrial = self.callPackage ./pybrial.nix {};

-      env-locations = self.callPackage ./env-locations.nix {
-        inherit pari_data ecl;
-        inherit singular;
-        three = nodePackages.three;
-        mathjax = nodePackages.mathjax;
-      };
-
-      sage-env = self.callPackage ./sage-env.nix {
-        inherit sage-src python rWrapper openblas-cblas-pc ecl singular palp flint pynac pythonEnv;
-        pkg-config = nixpkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
-      };
-
-      sage-with-env = self.callPackage ./sage-with-env.nix {
-        inherit pythonEnv;
-        inherit sage-src openblas-blas-pc openblas-cblas-pc openblas-lapack-pc pynac singular;
-        pkg-config = nixpkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
-        three = nodePackages.three;
-      };
-
-      sage = self.callPackage ./sage.nix { };
-
-      sage-wrapper = self.callPackage ./sage-wrapper.nix {
-        inherit sage-src withDoc;
+      # `sagelib`, i.e. all of sage except some wrappers and runtime dependencies
+      sagelib = self.callPackage ./sagelib.nix {
+        inherit flint ecl arb;
+        inherit sage-src pynac singular;
+        linbox = pkgs.linbox.override { withSage = true; };
      };
    };
  };

-  openblas-blas-pc = callPackage ./openblas-pc.nix { name = "blas"; };
-  openblas-cblas-pc = callPackage ./openblas-pc.nix { name = "cblas"; };
-  openblas-lapack-pc = callPackage ./openblas-pc.nix { name = "lapack"; };
+  jupyter-kernel-definition = {
+    displayName = "SageMath ${sage-src.version}";
+    argv = [
+      "${sage-with-env}/bin/sage" # FIXME which sage
+      "--python"
+      "-m"
+      "sage.repl.ipython_kernel"
+      "-f"
+      "{connection_file}"
+    ];
+    language = "sagemath";
+    # just one 16x16 logo is available
+    logo32 = "${sage-src}/doc/common/themes/sage/static/sageicon.png";
+    logo64 = "${sage-src}/doc/common/themes/sage/static/sageicon.png";
+  };
+
+  # A bash script setting various environment variables to tell sage where
+  # the files its looking fore are located. Also see `sage-env`.
+  env-locations = callPackage ./env-locations.nix {
+    inherit pari_data ecl;
+    inherit singular;
+    cysignals = python.pkgs.cysignals;
+    three = nodePackages.three;
+    mathjax = nodePackages.mathjax;
+  };
+
+  # The shell file that gets sourced on every sage start. Will also source
+  # the env-locations file.
+  sage-env = callPackage ./sage-env.nix {
+    sagelib = python.pkgs.sagelib;
+    inherit env-locations;
+    inherit python rWrapper ecl singular palp flint pynac pythonEnv;
+    pkg-config = pkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
+  };
+
+  # The documentation for sage, building it takes a lot of ram.
+  sagedoc = callPackage ./sagedoc.nix {
+    inherit sage-with-env;
+    inherit python;
+  };
+
+  # sagelib with added wrappers and a dependency on sage-tests to make sure thet tests were run.
+  sage-with-env = callPackage ./sage-with-env.nix {
+    inherit pythonEnv;
+    inherit sage-env;
+    inherit pynac singular;
+    pkg-config = pkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
+    three = nodePackages.three;
+  };
+
+  # Doesn't actually build anything, just runs sages testsuite. This is a
+  # separate derivation to make it possible to re-run the tests without
+  # rebuilding sagelib (which takes ~30 minutes).
+  # Running the tests should take something in the order of 1h.
+  sage-tests = callPackage ./sage-tests.nix {
+    inherit sage-with-env;
+  };

  sage-src = callPackage ./sage-src.nix {};

@ -77,6 +110,7 @@ let
    sympy
    fpylll
    matplotlib
+    tkinter # optional, as a matplotlib backend (use with `%matplotlib tk`)
    scipy
    ipywidgets
    rpy2
@ -91,9 +125,9 @@ let
  } // { extraLibs = pythonRuntimeDeps; }; # make the libs accessible

  # needs to be rWrapper, standard "R" doesn't include default packages
-  rWrapper = nixpkgs.rWrapper.override {
+  rWrapper = pkgs.rWrapper.override {
    # https://trac.sagemath.org/ticket/25674
-    R = nixpkgs.R.overrideAttrs (attrs: rec {
+    R = pkgs.R.overrideAttrs (attrs: rec {
      name = "R-3.4.4";
      src = fetchurl {
        url = "http://cran.r-project.org/src/base/R-3/${name}.tar.gz";
@ -102,44 +136,45 @@ let
    });
  };

-  arb = nixpkgs.arb.override { inherit flint; };
+  arb = pkgs.arb.override { inherit flint; };

-  singular = nixpkgs.singular.override { inherit flint; };
+  singular = pkgs.singular.override { inherit flint; };

  # *not* to confuse with the python package "pynac"
-  pynac = nixpkgs.pynac.override { inherit singular flint; };
+  pynac = pkgs.pynac.override { inherit singular flint; };

  # With openblas (64 bit), the tests fail the same way as when sage is build with
  # openblas instead of openblasCompat. Apparently other packages somehow use flints
  # blas when it is available. Alternative would be to override flint to use
  # openblasCompat.
-  flint = nixpkgs.flint.override { withBlas = false; };
+  flint = pkgs.flint.override { withBlas = false; };

  # Multiple palp dimensions need to be available and sage expects them all to be
  # in the same folder.
  palp = symlinkJoin {
-    name = "palp-${nixpkgs.palp.version}";
+    name = "palp-${pkgs.palp.version}";
    paths = [
-      (nixpkgs.palp.override { dimensions = 4; doSymlink = false; })
-      (nixpkgs.palp.override { dimensions = 5; doSymlink = false; })
-      (nixpkgs.palp.override { dimensions = 6; doSymlink = true; })
-      (nixpkgs.palp.override { dimensions = 11; doSymlink = false; })
+      (pkgs.palp.override { dimensions = 4; doSymlink = false; })
+      (pkgs.palp.override { dimensions = 5; doSymlink = false; })
+      (pkgs.palp.override { dimensions = 6; doSymlink = true; })
+      (pkgs.palp.override { dimensions = 11; doSymlink = false; })
    ];
  };

  # Sage expects those in the same directory.
  pari_data = symlinkJoin {
    name = "pari_data";
-    paths = with nixpkgs; [
+    paths = with pkgs; [
      pari-galdata
      pari-seadata-small
    ];
  };

  # https://trac.sagemath.org/ticket/22191
-  ecl = nixpkgs.ecl_16_1_2;
+  ecl = pkgs.ecl_16_1_2;
 in
-  python.pkgs.sage-wrapper // {
-    doc = python.pkgs.sagedoc;
-    lib = python.pkgs.sagelib;
-  }
+# A wrapper around sage that makes sure sage finds its docs (if they were build).
+callPackage ./sage.nix {
+  inherit sage-tests sage-with-env sagedoc jupyter-kernel-definition;
+  inherit withDoc;
+}
--- a/pkgs/applications/science/math/sage/env-locations.nix
+++ b/pkgs/applications/science/math/sage/env-locations.nix
@ -16,6 +16,8 @@
 , cysignals
 }:

+# A bash script setting various environment variables to tell sage where
+# the files its looking fore are located. Also see `sage-env`.
 writeTextFile rec {
  name = "sage-env-locations";
  destination = "/${name}";
--- a/pkgs/applications/science/math/sage/openblas-pc.nix
+++ b/pkgs/applications/science/math/sage/openblas-pc.nix
@ -1,17 +0,0 @@
-{ openblasCompat
-, writeTextFile
-, name
-}:
-
-writeTextFile {
-  name = "openblas-${name}-pc-${openblasCompat.version}";
-  destination = "/lib/pkgconfig/${name}.pc";
-  text = ''
-    Name: ${name}
-    Version: ${openblasCompat.version}
-
-    Description: ${name} for SageMath, provided by the OpenBLAS package.
-    Cflags: -I${openblasCompat}/include
-    Libs: -L${openblasCompat}/lib -lopenblas
-  '';
-}
--- a/pkgs/applications/science/math/sage/patches/eclib-20180710.patch
+++ b/pkgs/applications/science/math/sage/patches/eclib-20180710.patch
@ -1,16 +0,0 @@
-diff --git a/src/sage/interfaces/mwrank.py b/src/sage/interfaces/mwrank.py
-index 4417b59276..ae57ca2991 100644
--- a/src/sage/interfaces/mwrank.py
-+++ b/src/sage/interfaces/mwrank.py
-@@ -54,8 +54,9 @@ def Mwrank(options="", server=None, server_tmpdir=None):
-         sage: M = Mwrank('-v 0 -l')
-         sage: print(M('0 0 1 -1 0'))
-         Curve [0,0,1,-1,0] :    Rank = 1
-        Generator 1 is [0:-1:1]; height 0.0511114082399688
-        Regulator = 0.0511114082399688
-+        Generator 1 is [0:-1:1]; height 0.051111408239969
-+        Regulator = 0.051111408239969
-+
-     """
-     global instances
-     try:
--- a/pkgs/applications/science/math/sage/patches/eclib-regulator-precision.patch
+++ b/pkgs/applications/science/math/sage/patches/eclib-regulator-precision.patch
@ -1,98 +0,0 @@
-diff --git a/src/sage/libs/eclib/interface.py b/src/sage/libs/eclib/interface.py
-index f77000c478..9d17d412ae 100644
--- a/src/sage/libs/eclib/interface.py
-+++ b/src/sage/libs/eclib/interface.py
-@@ -1014,7 +1014,7 @@ class mwrank_MordellWeil(SageObject):
-             WARNING: saturation at primes p > 2 will not be done;
-             ...
-             Gained index 2
-            New regulator =  93.857300720636393209
-+            New regulator =  93.85730...
-             (False, 2, '[ ]')
-             sage: EQ.points()
-             [[-2, 3, 1], [2707496766203306, 864581029138191, 2969715140223272], [-13422227300, -49322830557, 12167000000]]
-@@ -1025,7 +1025,7 @@ class mwrank_MordellWeil(SageObject):
-             WARNING: saturation at primes p > 3 will not be done;
-             ...
-             Gained index 3
-            New regulator =  10.4285889689595992455
-+            New regulator =  10.42858...
-             (False, 3, '[ ]')
-             sage: EQ.points()
-             [[-2, 3, 1], [-14, 25, 8], [-13422227300, -49322830557, 12167000000]]
-@@ -1036,7 +1036,7 @@ class mwrank_MordellWeil(SageObject):
-             WARNING: saturation at primes p > 5 will not be done;
-             ...
-             Gained index 5
-            New regulator =  0.417143558758383969818
-+            New regulator =  0.41714...
-             (False, 5, '[ ]')
-             sage: EQ.points()
-             [[-2, 3, 1], [-14, 25, 8], [1, -1, 1]]
-@@ -1221,7 +1221,7 @@ class mwrank_MordellWeil(SageObject):
-             WARNING: saturation at primes p > 2 will not be done;
-             ...
-             Gained index 2
-            New regulator =  93.857300720636393209
-+            New regulator =  93.85730...
-             (False, 2, '[ ]')
-             sage: EQ
-             Subgroup of Mordell-Weil group: [[-2:3:1], [2707496766203306:864581029138191:2969715140223272], [-13422227300:-49322830557:12167000000]]
-@@ -1235,7 +1235,7 @@ class mwrank_MordellWeil(SageObject):
-             WARNING: saturation at primes p > 3 will not be done;
-             ...
-             Gained index 3
-            New regulator =  10.4285889689595992455
-+            New regulator =  10.42858...
-             (False, 3, '[ ]')
-             sage: EQ
-             Subgroup of Mordell-Weil group: [[-2:3:1], [-14:25:8], [-13422227300:-49322830557:12167000000]]
-@@ -1249,7 +1249,7 @@ class mwrank_MordellWeil(SageObject):
-             WARNING: saturation at primes p > 5 will not be done;
-             ...
-             Gained index 5
-            New regulator =  0.417143558758383969818
-+            New regulator =  0.41714...
-             (False, 5, '[ ]')
-             sage: EQ
-             Subgroup of Mordell-Weil group: [[-2:3:1], [-14:25:8], [1:-1:1]]
-diff --git a/src/sage/libs/eclib/mwrank.pyx b/src/sage/libs/eclib/mwrank.pyx
-index a4f89e1ca5..f8a22d2f55 100644
--- a/src/sage/libs/eclib/mwrank.pyx
-+++ b/src/sage/libs/eclib/mwrank.pyx
-@@ -1234,9 +1234,9 @@ cdef class _two_descent:
-             sage: D2.saturate()
-             Searching for points (bound = 8)...done:
-               found points which generate a subgroup of rank 3
-              and regulator 0.417143558758383969817119544618093396749810106098479
-+              and regulator 0.41714...
-             Processing points found during 2-descent...done:
-              now regulator = 0.417143558758383969817119544618093396749810106098479
-+              now regulator = 0.41714...
-             No saturation being done
-             sage: D2.getbasis()
-             '[[1:-1:1], [-2:3:1], [-14:25:8]]'
-@@ -1281,9 +1281,9 @@ cdef class _two_descent:
-             sage: D2.saturate()
-             Searching for points (bound = 8)...done:
-               found points which generate a subgroup of rank 3
-              and regulator 0.417143558758383969817119544618093396749810106098479
-+              and regulator 0.41714...
-             Processing points found during 2-descent...done:
-              now regulator = 0.417143558758383969817119544618093396749810106098479
-+              now regulator = 0.41714...
-             No saturation being done
-             sage: D2.getbasis()
-             '[[1:-1:1], [-2:3:1], [-14:25:8]]'
-@@ -1329,9 +1329,9 @@ cdef class _two_descent:
-             sage: D2.saturate()
-             Searching for points (bound = 8)...done:
-               found points which generate a subgroup of rank 3
-              and regulator 0.417143558758383969817119544618093396749810106098479
-+              and regulator 0.41714...
-             Processing points found during 2-descent...done:
-              now regulator = 0.417143558758383969817119544618093396749810106098479
-+              now regulator = 0.41714...
-             No saturation being done
-             sage: D2.getbasis()
-             '[[1:-1:1], [-2:3:1], [-14:25:8]]'
--- a/pkgs/applications/science/math/sage/patches/known-padics-bug.patch
+++ b/pkgs/applications/science/math/sage/patches/known-padics-bug.patch
@ -1,15 +0,0 @@
-diff --git a/build/pkgs/openblas/package-version.txt b/build/pkgs/openblas/package-version.txt
-index 3bc45c25d4..7c7c224887 100644
--- a/src/sage/schemes/elliptic_curves/padics.py
-+++ b/src/sage/schemes/elliptic_curves/padics.py
-@@ -292,8 +292,8 @@ def padic_regulator(self, p, prec=20, height=None, check_hypotheses=True):
- 
-         sage: max_prec = 30    # make sure we get past p^2    # long time
-         sage: full = E.padic_regulator(5, max_prec)           # long time
-        sage: for prec in range(1, max_prec):                 # long time
-        ....:     assert E.padic_regulator(5, prec) == full   # long time
-+        sage: for prec in range(1, max_prec):                 # known bug (#25969) # long time
-+        ....:     assert E.padic_regulator(5, prec) == full   # known bug (#25969) # long time
- 
-     A case where the generator belongs to the formal group already
-     (:trac:`3632`)::
--- a/pkgs/applications/science/math/sage/patches/matplotlib-normed-deprecated.patch
+++ b/pkgs/applications/science/math/sage/patches/matplotlib-normed-deprecated.patch
@ -1,12 +0,0 @@
-diff --git a/src/sage/all.py b/src/sage/all.py
-index 14cec431f1..25a35a0522 100644
--- a/src/sage/all.py
-+++ b/src/sage/all.py
-@@ -310,6 +310,7 @@ warnings.filters.remove(('ignore', None, DeprecationWarning, None, 0))
- # Ignore all deprecations from IPython etc.
- warnings.filterwarnings('ignore',
-     module='.*(IPython|ipykernel|jupyter_client|jupyter_core|nbformat|notebook|ipywidgets|storemagic)')
-+warnings.filterwarnings('ignore', "The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.") # matplotlib normed deprecation
- # However, be sure to keep OUR deprecation warnings
- warnings.filterwarnings('default',
-     '[\s\S]*See http://trac.sagemath.org/[0-9]* for details.')
--- a/pkgs/applications/science/math/sage/sage-env.nix
+++ b/pkgs/applications/science/math/sage/sage-env.nix
@ -2,7 +2,6 @@
 , lib
 , writeTextFile
 , python
-, sage-src
 , sagelib
 , env-locations
 , gfortran
@ -37,7 +36,7 @@
 , lcalc
 , rubiks
 , flintqs
-, openblas-cblas-pc
+, openblasCompat
 , flint
 , gmp
 , mpfr
@ -47,6 +46,10 @@
 , ntl
 }:

+# This generates a `sage-env` shell file that will be sourced by sage on startup.
+# It sets up various environment variables, telling sage where to find its
+# dependencies.
+
 let
  runtimepath = (lib.makeBinPath ([
    "@sage-local@"
@ -96,26 +99,27 @@ writeTextFile rec {
  destination = "/${name}";
  text = ''
    export PKG_CONFIG_PATH='${lib.concatStringsSep ":" (map (pkg: "${pkg}/lib/pkgconfig") [
-        # This is only needed in the src/sage/misc/cython.py test and I'm not sure if there's really a use-case
-        # for it outside of the tests. However since singular and openblas are runtime dependencies anyways
-        # and openblas-cblas-pc is tiny, it doesn't really hurt to include.
+        # This is only needed in the src/sage/misc/cython.py test and I'm not
+        # sure if there's really a usecase for it outside of the tests. However
+        # since singular and openblas are runtime dependencies anyways, it doesn't
+        # really hurt to include.
        singular
-        openblas-cblas-pc
+        openblasCompat
      ])
    }'
-    export SAGE_ROOT='${sage-src}'
+    export SAGE_ROOT='${sagelib.src}'
    export SAGE_LOCAL='@sage-local@'
    export SAGE_SHARE='${sagelib}/share'
    orig_path="$PATH"
    export PATH='${runtimepath}'

    # set dependent vars, like JUPYTER_CONFIG_DIR
-    source "${sage-src}/src/bin/sage-env"
+    source "${sagelib.src}/src/bin/sage-env"
    export PATH="${runtimepath}:$orig_path" # sage-env messes with PATH

    export SAGE_LOGS="$TMPDIR/sage-logs"
    export SAGE_DOC="''${SAGE_DOC_OVERRIDE:-doc-placeholder}"
-    export SAGE_DOC_SRC="''${SAGE_DOC_SRC_OVERRIDE:-${sage-src}/src/doc}"
+    export SAGE_DOC_SRC="''${SAGE_DOC_SRC_OVERRIDE:-${sagelib.src}/src/doc}"

    # set locations of dependencies
    . ${env-locations}/sage-env-locations
@ -154,9 +158,11 @@ writeTextFile rec {

    export SAGE_LIB='${sagelib}/${python.sitePackages}'

-    export SAGE_EXTCODE='${sage-src}/src/ext'
+    export SAGE_EXTCODE='${sagelib.src}/src/ext'

-    # for find_library
+  # for find_library
    export DYLD_LIBRARY_PATH="${lib.makeLibraryPath [stdenv.cc.libc singular]}:$DYLD_LIBRARY_PATH"
  '';
+} // {
+  lib = sagelib; # equivalent of `passthru`, which `writeTextFile` doesn't support
 }
--- a/pkgs/applications/science/math/sage/sage-src.nix
+++ b/pkgs/applications/science/math/sage/sage-src.nix
@ -2,6 +2,12 @@
 , fetchFromGitHub
 , fetchpatch
 }:
+
+# This file is responsible for fetching the sage source and adding necessary patches.
+# It does not actually build anything, it just copies the patched sources to $out.
+# This is done because multiple derivations rely on these sources and they should
+# all get the same sources with the same patches applied.
+
 stdenv.mkDerivation rec {
  version = "8.4";
  name = "sage-src-${version}";
@ -13,6 +19,8 @@ stdenv.mkDerivation rec {
    sha256 = "0gips1hagiz9m7s21bg5as8hrrm2x5k47h1bsq0pc46iplfwmv2d";
  };

+  # Patches needed because of particularities of nix or the way this is packaged.
+  # The goal is to upstream all of them and get rid of this list.
  nixPatches = [
    # https://trac.sagemath.org/ticket/25358
    (fetchpatch {
@ -31,6 +39,7 @@ stdenv.mkDerivation rec {
    # Revert the commit that made the sphinx build fork even in the single thread
    # case. For some yet unknown reason, that breaks the docbuild on nix and archlinux.
    # See https://groups.google.com/forum/#!msg/sage-packaging/VU4h8IWGFLA/mrmCMocYBwAJ.
+    # https://trac.sagemath.org/ticket/26608
    ./patches/revert-sphinx-always-fork.patch

    # Make sure py2/py3 tests are only run when their expected context (all "sage"
@ -39,10 +48,16 @@ stdenv.mkDerivation rec {
    ./patches/Only-test-py2-py3-optional-tests-when-all-of-sage-is.patch
  ];

+  # Patches needed because of package updates. We could just pin the versions of
+  # dependencies, but that would lead to rebuilds, confusion and the burdons of
+  # maintaining multiple versions of dependencies. Instead we try to make sage
+  # compatible with never dependency versions when possible. All these changes
+  # should come from or be proposed to upstream. This list will probably never
+  # be empty since dependencies update all the time.
  packageUpgradePatches = let
-    # fetch a diff between base and rev on sage's git server
-    # used to fetch trac tickets by setting the base to the release and the
-    # revision to the last commit that should be included
+    # Fetch a diff between `base` and `rev` on sage's git server.
+    # Used to fetch trac tickets by setting the `base` to the last release and the
+    # `rev` to the last commit of the ticket.
    fetchSageDiff = { base, rev, ...}@args: (
      fetchpatch ({
        url = "https://git.sagemath.org/sage.git/patch?id2=${base}&id=${rev}";
@ -54,7 +69,7 @@ stdenv.mkDerivation rec {
  in [
    # New glpk version has new warnings, filter those out until upstream sage has found a solution
    # https://trac.sagemath.org/ticket/24824
-    ./patches/pari-stackwarn.patch # not actually necessary since tha pari upgrade, but necessary for the glpk patch to apply
+    ./patches/pari-stackwarn.patch # not actually necessary since the pari upgrade, but necessary for the glpk patch to apply
    (fetchpatch {
      url = "https://salsa.debian.org/science-team/sagemath/raw/58bbba93a807ca2933ca317501d093a1bb4b84db/debian/patches/dt-version-glpk-4.65-ignore-warnings.patch";
      sha256 = "0b9293v73wb4x13wv5zwyjgclc01zn16msccfzzi6znswklgvddp";
@ -64,8 +79,8 @@ stdenv.mkDerivation rec {
    # https://trac.sagemath.org/ticket/25260
    ./patches/numpy-1.15.1.patch

-    # ntl upgrade
-    # https://trac.sagemath.org/ticket/25532#comment:29
+    # needed for ntl update
+    # https://trac.sagemath.org/ticket/25532
    (fetchpatch {
      name = "lcalc-c++11.patch";
      url = "https://git.archlinux.org/svntogit/community.git/plain/trunk/sagemath-lcalc-c++11.patch?h=packages/sagemath&id=0e31ae526ab7c6b5c0bfacb3f8b1c4fd490035aa";
@ -100,9 +115,7 @@ stdenv.mkDerivation rec {
    })
  ];

-  patches = nixPatches ++ packageUpgradePatches ++ [
-    ./patches/known-padics-bug.patch
-  ];
+  patches = nixPatches ++ packageUpgradePatches;

  postPatch = ''
    # make sure shebangs etc are fixed, but sage-python23 still works
--- a/pkgs/applications/science/math/sage/sage-tests.nix
+++ b/pkgs/applications/science/math/sage/sage-tests.nix
@ -0,0 +1,51 @@
+{ stdenv
+, lib
+, sage-with-env
+, makeWrapper
+, files ? null # "null" means run all tests
+, longTests ? true # run tests marked as "long time"
+}:
+
+# for a quick test of some source files:
+# nix-build -E 'with (import ./. {}); sage.tests.override { files = [ "src/sage/misc/cython.py" ];}'
+
+let
+  src = sage-with-env.env.lib.src;
+  runAllTests = files == null;
+  testArgs = if runAllTests then "--all" else testFileList;
+  patienceSpecifier = if longTests then "--long" else "";
+  relpathToArg = relpath: lib.escapeShellArg "${src}/${relpath}"; # paths need to be absolute
+  testFileList = lib.concatStringsSep " " (map relpathToArg files);
+in
+stdenv.mkDerivation rec {
+  version = src.version;
+  name = "sage-tests-${version}";
+  inherit src;
+
+  buildInputs = [
+    makeWrapper
+    sage-with-env
+  ];
+
+  unpackPhase = "#do nothing";
+  configurePhase = "#do nothing";
+  buildPhase = "#do nothing";
+
+  installPhase = ''
+    # This output is not actually needed for anything, the package just
+    # exists to decouple the sage build from its t ests.
+
+    mkdir -p "$out/bin"
+    # Like a symlink, but make sure that $0 points to the original.
+    makeWrapper "${sage-with-env}/bin/sage" "$out/bin/sage"
+  '';
+
+  doInstallCheck = true;
+  installCheckPhase = ''
+    export HOME="$TMPDIR/sage-home"
+    mkdir -p "$HOME"
+
+    # "--long" tests are in the order of 1h, without "--long" its 1/2h
+    "sage" -t --nthreads "$NIX_BUILD_CORES" --optional=sage ${patienceSpecifier} ${testArgs}
+  '';
+}
--- a/pkgs/applications/science/math/sage/sage-with-env.nix
+++ b/pkgs/applications/science/math/sage/sage-with-env.nix
@ -2,11 +2,7 @@
 , lib
 , makeWrapper
 , sage-env
-, sage-src
 , openblasCompat
-, openblas-blas-pc
-, openblas-cblas-pc
-, openblas-lapack-pc
 , pkg-config
 , three
 , singular
@ -26,15 +22,15 @@
 , pythonEnv
 }:

+# Wrapper that combined `sagelib` with `sage-env` to produce an actually
+# executable sage. No tests are run yet and no documentation is built.
+
 let
  buildInputs = [
    pythonEnv # for patchShebangs
    makeWrapper
    pkg-config
    openblasCompat # lots of segfaults with regular (64 bit) openblas
-    openblas-blas-pc
-    openblas-cblas-pc
-    openblas-lapack-pc
    singular
    three
    pynac
@ -92,13 +88,12 @@ let
  input_names = map (dep: pkg_to_spkg_name dep patch_names) transitiveDeps;
 in
 stdenv.mkDerivation rec {
-  version = sage-src.version;
+  version = src.version;
  name = "sage-with-env-${version}";
+  src = sage-env.lib.src;

  inherit buildInputs;

-  src = sage-src;
-
  configurePhase = "#do nothing";

  buildPhase = ''
@ -110,17 +105,24 @@ stdenv.mkDerivation rec {

  installPhase = ''
    mkdir -p "$out/var/lib/sage"
-    cp -r installed $out/var/lib/sage
+    cp -r installed "$out/var/lib/sage"

    mkdir -p "$out/etc"
    # sage tests will try to create this file if it doesn't exist
    touch "$out/etc/sage-started.txt"

    mkdir -p "$out/build"
+
+    # the scripts in src/bin will find the actual sage source files using environment variables set in `sage-env`
    cp -r src/bin "$out/bin"
    cp -r build/bin "$out/build/bin"
+
    cp -f '${sage-env}/sage-env' "$out/bin/sage-env"
    substituteInPlace "$out/bin/sage-env" \
      --subst-var-by sage-local "$out"
  '';
+
+  passthru = {
+    env = sage-env;
+  };
 }
--- a/pkgs/applications/science/math/sage/sage-wrapper.nix
+++ b/pkgs/applications/science/math/sage/sage-wrapper.nix
@ -1,41 +0,0 @@
-{ stdenv
-, makeWrapper
-, sage
-, sage-src
-, sagedoc
-, withDoc
-}:
-
-stdenv.mkDerivation rec {
-  version = sage.version;
-  name = "sage-${version}";
-
-  buildInputs = [
-    makeWrapper
-  ];
-
-  unpackPhase = "#do nothing";
-  configurePhase = "#do nothing";
-  buildPhase = "#do nothing";
-
-  installPhase = ''
-    mkdir -p "$out/bin"
-    makeWrapper "${sage}/bin/sage" "$out/bin/sage" \
-      --set SAGE_DOC_SRC_OVERRIDE "${sage-src}/src/doc" ${
-      stdenv.lib.optionalString withDoc "--set SAGE_DOC_OVERRIDE ${sagedoc}/share/doc/sage"
-    }
-  '';
-
-  doInstallCheck = withDoc;
-  installCheckPhase = ''
-    export HOME="$TMPDIR/sage-home"
-    mkdir -p "$HOME"
-    "$out/bin/sage" -c 'browse_sage_doc._open("reference", testing=True)'
-  '';
-
-  meta = with stdenv.lib; {
-    description = "Open Source Mathematics Software, free alternative to Magma, Maple, Mathematica, and Matlab";
-    license = licenses.gpl2;
-    maintainers = with maintainers; [ timokau ];
-  };
-}
--- a/pkgs/applications/science/math/sage/sage.nix
+++ b/pkgs/applications/science/math/sage/sage.nix
@ -1,14 +1,35 @@
 { stdenv
-, sage-with-env
 , makeWrapper
+, sage-tests
+, sage-with-env
+, jupyter-kernel-definition
+, jupyter-kernel
+, sagedoc
+, withDoc
 }:

+# A wrapper that makes sure sage finds its docs (if they were build) and the
+# jupyter kernel spec.
+
+let 
+  # generate kernel spec + default kernels
+  kernel-specs = jupyter-kernel.create {
+    definitions = jupyter-kernel.default // {
+      sagemath = jupyter-kernel-definition;
+    };
+  };
+in
 stdenv.mkDerivation rec {
-  version = sage-with-env.version;
-  name = "sage-tests-${version}";
+  version = src.version;
+  name = "sage-${version}";
+  src = sage-with-env.env.lib.src;

  buildInputs = [
    makeWrapper
+
+    # This is a hack to make sure sage-tests is evaluated. It doesn't acutally
+    # produce anything of value, it just decouples the tests from the build.
+    sage-tests
  ];

  unpackPhase = "#do nothing";
@ -17,16 +38,30 @@ stdenv.mkDerivation rec {

  installPhase = ''
    mkdir -p "$out/bin"
-    # Like a symlink, but make sure that $0 points to the original.
-    makeWrapper "${sage-with-env}/bin/sage" "$out/bin/sage"
+    makeWrapper "${sage-with-env}/bin/sage" "$out/bin/sage" \
+      --set SAGE_DOC_SRC_OVERRIDE "${src}/src/doc" ${
+        stdenv.lib.optionalString withDoc "--set SAGE_DOC_OVERRIDE ${sagedoc}/share/doc/sage"
+      } \
+      --prefix JUPYTER_PATH : "${kernel-specs}"
  '';

-  doInstallCheck = true;
+  doInstallCheck = withDoc;
  installCheckPhase = ''
    export HOME="$TMPDIR/sage-home"
    mkdir -p "$HOME"
-
-    # "--long" tests are in the order of 1h, without "--long" its 1/2h
-    "$out/bin/sage" -t --nthreads "$NIX_BUILD_CORES" --optional=sage --long --all
+    "$out/bin/sage" -c 'browse_sage_doc._open("reference", testing=True)'
  '';
+
+  passthru = {
+    tests = sage-tests;
+    doc = sagedoc;
+    lib = sage-with-env.env.lib;
+    kernelspec = jupyter-kernel-definition;
+  };
+
+  meta = with stdenv.lib; {
+    description = "Open Source Mathematics Software, free alternative to Magma, Maple, Mathematica, and Matlab";
+    license = licenses.gpl2;
+    maintainers = with maintainers; [ timokau ];
+  };
 }
--- a/pkgs/applications/science/math/sage/sagedoc.nix
+++ b/pkgs/applications/science/math/sage/sagedoc.nix
@ -1,63 +1,49 @@
 { stdenv
-, sage-src
 , sage-with-env
-, sagelib
-, python2
-, psutil
-, future
-, sphinx
-, sagenb
+, python
 , maxima-ecl
-, networkx
-, scipy
-, sympy
-, matplotlib
-, pillow
-, ipykernel
-, jupyter_client
 , tachyon
 , jmol
-, ipywidgets
-, typing
 , cddlib
-, pybrial
 }:

 stdenv.mkDerivation rec {
-  version = sage-src.version;
+  version = src.version;
  name = "sagedoc-${version}";
+  src = sage-with-env.env.lib.src;


  # Building the documentation has many dependencies, because all documented
  # modules are imported and because matplotlib is used to produce plots.
  buildInputs = [
-    sagelib
-    python2
+    sage-with-env.env.lib
+    python
+    maxima-ecl
+    tachyon
+    jmol
+    cddlib
+  ] ++ (with python.pkgs; [
    psutil
    future
    sphinx
    sagenb
-    maxima-ecl
-    networkx
    scipy
    sympy
    matplotlib
    pillow
+    networkx
    ipykernel
-    jupyter_client
-    tachyon
-    jmol
    ipywidgets
+    jupyter_client
    typing
-    cddlib
    pybrial
-  ];
+  ]);

  unpackPhase = ''
    export SAGE_DOC_OVERRIDE="$PWD/share/doc/sage"
    export SAGE_DOC_SRC_OVERRIDE="$PWD/docsrc"

-    cp -r "${sage-src}/src/doc" "$SAGE_DOC_SRC_OVERRIDE"
+    cp -r "${src}/src/doc" "$SAGE_DOC_SRC_OVERRIDE"
    chmod -R 755 "$SAGE_DOC_SRC_OVERRIDE"
  '';

--- a/pkgs/applications/science/math/sage/sagelib.nix
+++ b/pkgs/applications/science/math/sage/sagelib.nix
@ -3,9 +3,6 @@
 , buildPythonPackage
 , arb
 , openblasCompat
-, openblas-blas-pc
-, openblas-cblas-pc
-, openblas-lapack-pc
 , brial
 , cliquer
 , cypari2
@ -51,19 +48,20 @@
 , libbraiding
 }:

+# This is the core sage python package. Everything else is just wrappers gluing
+# stuff together. It is not very useful on its own though, since it will not
+# find many of its dependencies without `sage-env`, will not be tested without
+# `sage-tests` and will not have html docs without `sagedoc`.
+
 buildPythonPackage rec {
  format = "other";
-  version = sage-src.version;
-  pname = "sagelib";
-
+  version = src.version;
+  name = "sagelib-${version}";
  src = sage-src;

  nativeBuildInputs = [
    iml
    perl
-    openblas-blas-pc
-    openblas-cblas-pc
-    openblas-lapack-pc
    jupyter_core
  ];

--- a/pkgs/applications/science/math/sage/sagenb.nix
+++ b/pkgs/applications/science/math/sage/sagenb.nix
@ -1,4 +1,3 @@
-# Has a cyclic dependency with sage (not expressed here) and is not useful outside of sage
 { stdenv
 , fetchpatch
 , python
@ -13,6 +12,10 @@
 , flask-babel
 }:

+# Has a cyclic dependency with sage (not expressed here) and is not useful outside of sage.
+# Deprecated, hopefully soon to be removed. See
+# https://trac.sagemath.org/ticket/25837
+
 buildPythonPackage rec {
  pname = "sagenb";
  version = "2018-06-26"; # not 1.0.1 because of new flask syntax
--- a/pkgs/applications/science/misc/golly/beta.nix
+++ b/pkgs/applications/science/misc/golly/beta.nix
@ -46,6 +46,6 @@ stdenv.mkDerivation rec {
    license = stdenv.lib.licenses.gpl2;
    maintainers = [stdenv.lib.maintainers.raskin];
    platforms = stdenv.lib.platforms.linux;
-    downloadPage = "http://sourceforge.net/projects/golly/files/golly";
+    downloadPage = "https://sourceforge.net/projects/golly/files/golly";
  };
 }
--- a/pkgs/applications/science/misc/golly/default.nix
+++ b/pkgs/applications/science/misc/golly/default.nix
@ -35,6 +35,6 @@ stdenv.mkDerivation rec {
    license = stdenv.lib.licenses.gpl2;
    maintainers = [stdenv.lib.maintainers.raskin];
    platforms = stdenv.lib.platforms.linux;
-    downloadPage = "http://sourceforge.net/projects/golly/files/golly";
+    downloadPage = "https://sourceforge.net/projects/golly/files/golly";
  };
 }
--- a/pkgs/applications/science/misc/golly/default.upstream
+++ b/pkgs/applications/science/misc/golly/default.upstream
@ -1,4 +1,4 @@
-url http://sourceforge.net/projects/golly/files/golly/
+url https://sourceforge.net/projects/golly/files/golly/
 version_link '[-][0-9.]+/$'
 SF_version_tarball 'src'
 SF_redirect
--- a/pkgs/applications/version-management/bazaar/tools.nix
+++ b/pkgs/applications/version-management/bazaar/tools.nix
@ -5,7 +5,7 @@ python2Packages.buildPythonApplication rec {
  version = "2.6.0";

  src = fetchurl {
-    url = "http://launchpad.net/bzrtools/stable/${version}/+download/bzrtools-${version}.tar.gz";
+    url = "https://launchpad.net/bzrtools/stable/${version}/+download/bzrtools-${version}.tar.gz";
    sha256 = "0n3zzc6jf5866kfhmrnya1vdr2ja137a45qrzsz8vz6sc6xgn5wb";
  };

--- a/pkgs/applications/version-management/git-and-tools/fast-export/default.nix
+++ b/pkgs/applications/version-management/git-and-tools/fast-export/default.nix
@ -33,7 +33,7 @@ stdenv.mkDerivation {

  meta = {
    description = "Import svn, mercurial into git";
-    homepage = http://repo.or.cz/w/fast-export.git;
+    homepage = https://repo.or.cz/w/fast-export.git;
    license = licenses.gpl2;
    maintainers = [ maintainers.koral ];
    platforms = stdenv.lib.platforms.unix;
--- a/pkgs/applications/version-management/monotone-viz/default.nix
+++ b/pkgs/applications/version-management/monotone-viz/default.nix
@ -22,15 +22,15 @@ stdenv.mkDerivation rec {
  patchFlags = ["-p0"];
  patches = [
    (fetchurl {
-      url = "http://src.fedoraproject.org/cgit/rpms/monotone-viz.git/plain/monotone-viz-1.0.2-dot.patch";
+      url = "https://src.fedoraproject.org/cgit/rpms/monotone-viz.git/plain/monotone-viz-1.0.2-dot.patch";
      sha256 = "0risfy8iqmkr209hmnvpv57ywbd3rvchzzd0jy2lfyqrrrm6zknw";
    })
    (fetchurl {
-      url = "http://src.fedoraproject.org/cgit/rpms/monotone-viz.git/plain/monotone-viz-1.0.2-new-stdio.patch";
+      url = "https://src.fedoraproject.org/cgit/rpms/monotone-viz.git/plain/monotone-viz-1.0.2-new-stdio.patch";
      sha256 = "16bj0ppzqd45an154dr7sifjra7lv4m9anxfw3c56y763jq7fafa";
    })
    (fetchurl {
-      url = "http://src.fedoraproject.org/cgit/rpms/monotone-viz.git/plain/monotone-viz-1.0.2-typefix.patch";
+      url = "https://src.fedoraproject.org/cgit/rpms/monotone-viz.git/plain/monotone-viz-1.0.2-typefix.patch";
      sha256 = "1gfp82rc7pawb5x4hh2wf7xh1l1l54ib75930xgd1y437la4703r";
    })
  ];
--- a/pkgs/applications/video/openshot-qt/default.nix
+++ b/pkgs/applications/video/openshot-qt/default.nix
@ -4,13 +4,13 @@

 python3Packages.buildPythonApplication rec {
  name = "openshot-qt-${version}";
-  version = "2.4.2";
+  version = "2.4.3";

  src = fetchFromGitHub {
    owner = "OpenShot";
    repo = "openshot-qt";
    rev = "v${version}";
-    sha256 = "0m4fq9vj8gc5ngk8qf6ikj85qgzxhfk7nnz7n7362dzlfymaz18q";
+    sha256 = "1qdw1mli4y9qhrnllnkaf6ydgw5vfvdb90chs4i679k0x0jyb9a2";
  };

  nativeBuildInputs = [ doxygen wrapGAppsHook ];
--- a/pkgs/applications/video/pyca/default.nix
+++ b/pkgs/applications/video/pyca/default.nix
@ -0,0 +1,30 @@
+{ stdenv, buildPythonApplication, fetchFromGitHub, pycurl, dateutil, configobj, sqlalchemy, sdnotify, flask }:
+
+buildPythonApplication rec {
+  pname = "pyca";
+  version = "2.1";
+
+  src = fetchFromGitHub {
+    owner = "opencast";
+    repo = "pyCA";
+    rev = "v${version}";
+    sha256 = "0cvkmdlcax9da9iw4ls73vw0pxvm8wvchab5gwdy9w9ibqdpcmwh";
+  };
+
+  propagatedBuildInputs = [
+    pycurl
+    dateutil
+    configobj
+    sqlalchemy
+    sdnotify
+    flask
+  ];
+
+  meta = with stdenv.lib; {
+    description = "A fully functional Opencast capture agent written in Python";
+    homepage = https://github.com/opencast/pyCA;
+    license = licenses.lgpl3;
+    maintainers = with maintainers; [ pmiddend ];
+  };
+}
+
--- a/pkgs/applications/virtualization/driver/win-spice/default.nix
+++ b/pkgs/applications/virtualization/driver/win-spice/default.nix
@ -2,27 +2,27 @@

 let
  src_usbdk_x86 = fetchurl {
-    url = "http://www.spice-space.org/download/windows/usbdk/UsbDk_1.0.4_x86.msi";
+    url = "https://www.spice-space.org/download/windows/usbdk/UsbDk_1.0.4_x86.msi";
    sha256 = "17hv8034wk1xqnanm5jxs4741nl7asps1fdz6lhnrpp6gvj6yg9y";
  };

  src_usbdk_amd64 = fetchurl {
-    url = "http://www.spice-space.org/download/windows/usbdk/UsbDk_1.0.4_x64.msi";
+    url = "https://www.spice-space.org/download/windows/usbdk/UsbDk_1.0.4_x64.msi";
    sha256 = "0alcqsivp33pm8sy0lmkvq7m5yh6mmcmxdl39zjxjra67kw8r2sd";
  };

  src_qxlwddm = fetchurl {
-    url = "http://people.redhat.com/~vrozenfe/qxlwddm/qxlwddm-0.11.zip";
+    url = "https://people.redhat.com/~vrozenfe/qxlwddm/qxlwddm-0.11.zip";
    sha256 = "082zdpbh9i3bq2ds8g33rcbcw390jsm7cqf46rrlx02x8r03dm98";
  };

  src_vdagent_x86 = fetchurl {
-    url = "http://www.spice-space.org/download/windows/vdagent/vdagent-win-0.7.3/vdagent_0_7_3_x86.zip";
+    url = "https://www.spice-space.org/download/windows/vdagent/vdagent-win-0.7.3/vdagent_0_7_3_x86.zip";
    sha256 = "0d928g49rf4dl79jmvnqh6g864hp1flw1f0384sfp82himm3bxjs";
  };

  src_vdagent_amd64 = fetchurl {
-    url = "http://www.spice-space.org/download/windows/vdagent/vdagent-win-0.7.3/vdagent_0_7_3_x64.zip";
+    url = "https://www.spice-space.org/download/windows/vdagent/vdagent-win-0.7.3/vdagent_0_7_3_x64.zip";
    sha256 = "0djmvm66jcmcyhhbjppccbai45nqpva7vyvry6w8nyc0fwi1vm9l";
  };
 in
@ -61,8 +61,9 @@ stdenv.mkDerivation  {
      (copy "amd64" "w8.1") + (copy "x86" "w8.1");

  meta = with stdenv.lib; {
-    description = ''Windows SPICE Drivers'';
-    homepage = http://www.spice-space.org;
+    description = "Windows SPICE Drivers";
+    homepage = https://www.spice-space.org/;
+    license = [ licenses.asl20 ]; # See https://github.com/vrozenfe/qxl-dod
    maintainers = [ maintainers.tstrobel ];
    platforms = platforms.linux;
  };
--- a/pkgs/applications/virtualization/spice-vdagent/default.nix
+++ b/pkgs/applications/virtualization/spice-vdagent/default.nix
@ -24,7 +24,7 @@ stdenv.mkDerivation rec {
         to the client resolution
       * Multiple displays
    '';
-    homepage = http://www.spice-space.org/home.html;
+    homepage = https://www.spice-space.org/;
    license = stdenv.lib.licenses.gpl3;
    maintainers = [ stdenv.lib.maintainers.aboseley ];
    platforms = stdenv.lib.platforms.linux;
--- a/pkgs/applications/window-managers/dwm/dwm-status.nix
+++ b/pkgs/applications/window-managers/dwm/dwm-status.nix
@ -3,13 +3,13 @@

 rustPlatform.buildRustPackage rec {
  name = "dwm-status-${version}";
-  version = "1.2.0";
+  version = "1.4.0";

  src = fetchFromGitHub {
    owner = "Gerschtli";
    repo = "dwm-status";
    rev = version;
-    sha256 = "0bv1jkqkf509akg3dvdy8b2q1kh8i75vw4n6a9rjvslx9s9nh6ca";
+    sha256 = "1v9ksv8hdxhpm7vs71p9s1y3gnahczza0w4wyrk2fsc6x2kwlh6x";
  };

  nativeBuildInputs = [ makeWrapper pkgconfig ];
--- a/pkgs/applications/window-managers/i3/status.nix
+++ b/pkgs/applications/window-managers/i3/status.nix
@ -1,4 +1,4 @@
-{ fetchurl, stdenv, confuse, yajl, alsaLib, libpulseaudio, libnl, pkgconfig
+{ fetchurl, stdenv, libconfuse, yajl, alsaLib, libpulseaudio, libnl, pkgconfig
  }:

 stdenv.mkDerivation rec {
@ -10,7 +10,7 @@ stdenv.mkDerivation rec {
  };

  nativeBuildInputs = [ pkgconfig ];
-  buildInputs = [ confuse yajl alsaLib libpulseaudio libnl ];
+  buildInputs = [ libconfuse yajl alsaLib libpulseaudio libnl ];

  makeFlags = [ "all" "PREFIX=$(out)" ];

--- a/pkgs/applications/window-managers/stalonetray/default.nix
+++ b/pkgs/applications/window-managers/stalonetray/default.nix
@ -23,7 +23,7 @@ stdenv.mkDerivation rec {

  passthru = {
    updateInfo = {
-      downloadPage = "http://sourceforge.net/projects/stalonetray/files/";
+      downloadPage = "https://sourceforge.net/projects/stalonetray/files/";
    };
  };
 }
--- a/pkgs/data/fonts/noto-fonts/default.nix
+++ b/pkgs/data/fonts/noto-fonts/default.nix
@ -86,14 +86,14 @@ rec {
      maintainers = with maintainers; [ mathnerd314 ];
    };
  };
-  noto-fonts-emoji = let version = "2018-04-24-pistol-update"; in stdenv.mkDerivation {
+  noto-fonts-emoji = let version = "2018-08-10-unicode11"; in stdenv.mkDerivation {
    name = "noto-fonts-emoji-${version}";

    src = fetchFromGitHub {
      owner = "googlei18n";
      repo = "noto-emoji";
      rev = "v${version}";
-      sha256 = "1f9k182j0619xvwk60gw2hng3lcd483sva2fabjdhznk8yf9f7jg";
+      sha256 = "1y54zsvwf5pqhcd9cl2zz5l52qyswn6kycvrq03zm5kqqsngbw3p";
    };

    buildInputs = [ cairo ];
@ -116,7 +116,7 @@ rec {
      inherit version;
      description = "Color and Black-and-White emoji fonts";
      homepage = https://github.com/googlei18n/noto-emoji;
-      license = licenses.asl20;
+      license = with licenses; [ ofl asl20 ];
      platforms = platforms.all;
      maintainers = with maintainers; [ mathnerd314 ];
    };
--- a/pkgs/data/fonts/sarasa-gothic/default.nix
+++ b/pkgs/data/fonts/sarasa-gothic/default.nix
@ -1,23 +1,22 @@
 { stdenv, fetchurl, p7zip }:

-stdenv.mkDerivation rec {
+let
  version = "0.6.0";
+  sha256 = "08g3kzplp3v8kvni1vzl73fgh03xgfl8pwqyj7vwjihjdr1xfjyz";
+in fetchurl rec {
+  inherit sha256;
+
  name = "sarasa-gothic-${version}";

-  package = fetchurl {
-    url = "https://github.com/be5invis/Sarasa-Gothic/releases/download/v${version}/sarasa-gothic-ttf-${version}.7z";
-    sha256 = "00kyx03lpgycxaw0cyx96hhrx8gwkcmy3qs35q7r09y60vg5i0nv";
-  };
+  url = "https://github.com/be5invis/Sarasa-Gothic/releases/download/v${version}/sarasa-gothic-ttc-${version}.7z";

-  nativeBuildInputs = [ p7zip ];
+  recursiveHash = true;
+  downloadToTemp = true;

-  unpackPhase = ''
-    7z x $package
-  '';
-
-  installPhase = ''
-    mkdir -p $out/share/fonts/truetype
-    cp *.ttf $out/share/fonts/truetype
+  postFetch = ''
+    ${p7zip}/bin/7z x $downloadedFile
+    mkdir -p $out/share/fonts
+    install -m644 *.ttc $out/share/fonts/
  '';

  meta = with stdenv.lib; {
@ -26,7 +25,5 @@ stdenv.mkDerivation rec {
    license = licenses.ofl;
    maintainers = [ maintainers.ChengCat ];
    platforms = platforms.all;
-    # large package, mainly i/o bound
-    hydraPlatforms = [];
  };
 }
--- a/pkgs/data/fonts/twemoji-color-font/default.nix
+++ b/pkgs/data/fonts/twemoji-color-font/default.nix
@ -6,7 +6,7 @@ stdenv.mkDerivation rec {
    owner = "eosrei";
    repo = "twemoji-color-font";
    rev = "v${meta.version}";
-    sha256 = "0z8r7z2r0r2wng4a7hvqvkcpd43l0d57yl402r7ci5bnmb02yvsa";
+    sha256 = "07yawvbdkk15d7ac9dj7drs1rqln9sba1fd6jx885ms7ww2sfm7r";
  };

  nativeBuildInputs = [ inkscape imagemagick potrace svgo scfbuild ];
@ -21,7 +21,7 @@ stdenv.mkDerivation rec {
  '';

  meta = with stdenv.lib; {
-    version = "1.4";
+    version = "11.2.0";
    description = "Color emoji SVGinOT font using Twitter Unicode 10 emoji with diversity and country flags";
    longDescription = ''
      A color and B&W emoji SVGinOT font built from the Twitter Emoji for
--- a/pkgs/data/misc/hackage/default.nix
+++ b/pkgs/data/misc/hackage/default.nix
@ -1,6 +1,6 @@
 { fetchurl }:

 fetchurl {
-  url = "https://github.com/commercialhaskell/all-cabal-hashes/archive/3487551670de487866a34bd466b33b5146087882.tar.gz";
-  sha256 = "10kag8qmlsnj3qwq0zxb6apd2z7jg17srvhsax5lgbwvlymbnckb";
+  url = "https://github.com/commercialhaskell/all-cabal-hashes/archive/95366a34cd5c9b47444ac819562fff2f23d7d753.tar.gz";
+  sha256 = "184qrgb7jl1s79v4z1jz9ywihilf60jh93xhwf0n75vnxb4ibnfd";
 }
--- a/pkgs/data/misc/mailcap/default.nix
+++ b/pkgs/data/misc/mailcap/default.nix
@ -0,0 +1,28 @@
+{ lib, fetchzip }:
+
+let
+  version = "2.1.48";
+
+in fetchzip {
+  name = "mailcap-${version}";
+
+  url = "https://releases.pagure.org/mailcap/mailcap-${version}.tar.xz";
+  sha256 = "0m1rls4z85aby9fggwx2x70b4y6l0jjyiqdv30p8g91nv8hrq9fw";
+
+  postFetch = ''
+    tar -xavf $downloadedFile --strip-components=1
+    substituteInPlace mailcap --replace "/usr/bin/" ""
+    gzip mailcap.4
+
+    install -D -m0644 -t $out/etc mailcap mime.types
+    install -D -m0644 -t $out/share/man/man4 mailcap.4.gz
+  '';
+
+  meta = with lib; {
+    description = "Helper application and MIME type associations for file types";
+    homepage = "https://pagure.io/mailcap";
+    license = licenses.mit;
+    maintainers = with maintainers; [ c0bw3b ];
+    platforms = platforms.all;
+  };
+}
--- a/pkgs/development/compilers/ghc/8.4.4.nix
+++ b/pkgs/development/compilers/ghc/8.4.4.nix
@ -2,7 +2,7 @@

 # build-tools
 , bootPkgs
-, autoconf, automake, coreutils, fetchurl, fetchpatch, perl, python3, m4
+, autoconf, automake, coreutils, fetchurl, fetchpatch, perl, python3, m4, sphinx

 , libiconv ? null, ncurses

@ -183,7 +183,7 @@ stdenv.mkDerivation (rec {
  strictDeps = true;

  nativeBuildInputs = [
-    perl autoconf automake m4 python3
+    perl autoconf automake m4 python3 sphinx
    ghc bootPkgs.alex bootPkgs.happy bootPkgs.hscolour
  ];

--- a/pkgs/development/compilers/ghc/8.6.1.nix
+++ b/pkgs/development/compilers/ghc/8.6.1.nix
@ -2,7 +2,7 @@

 # build-tools
 , bootPkgs
-, autoconf, automake, coreutils, fetchurl, fetchpatch, perl, python3, m4
+, autoconf, automake, coreutils, fetchurl, fetchpatch, perl, python3, m4, sphinx

 , libiconv ? null, ncurses

@ -168,7 +168,7 @@ stdenv.mkDerivation (rec {
  strictDeps = true;

  nativeBuildInputs = [
-    perl autoconf automake m4 python3
+    perl autoconf automake m4 python3 sphinx
    ghc bootPkgs.alex bootPkgs.happy bootPkgs.hscolour
  ];

--- a/pkgs/development/compilers/ghc/8.6.2.nix
+++ b/pkgs/development/compilers/ghc/8.6.2.nix
@ -2,7 +2,7 @@

 # build-tools
 , bootPkgs
-, autoconf, automake, coreutils, fetchurl, fetchpatch, perl, python3, m4
+, autoconf, automake, coreutils, fetchurl, fetchpatch, perl, python3, m4, sphinx

 , libiconv ? null, ncurses

@ -168,7 +168,7 @@ stdenv.mkDerivation (rec {
  strictDeps = true;

  nativeBuildInputs = [
-    perl autoconf automake m4 python3
+    perl autoconf automake m4 python3 sphinx
    ghc bootPkgs.alex bootPkgs.happy bootPkgs.hscolour
  ];

--- a/pkgs/development/compilers/ghc/head.nix
+++ b/pkgs/development/compilers/ghc/head.nix
@ -2,7 +2,7 @@

 # build-tools
 , bootPkgs
-, autoconf, automake, coreutils, fetchgit, perl, python3, m4
+, autoconf, automake, coreutils, fetchgit, perl, python3, m4, sphinx

 , libiconv ? null, ncurses

--- a/pkgs/development/haskell-modules/configuration-common.nix
+++ b/pkgs/development/haskell-modules/configuration-common.nix
@ -86,7 +86,7 @@ self: super: {
      name = "git-annex-${super.git-annex.version}-src";
      url = "git://git-annex.branchable.com/";
      rev = "refs/tags/" + super.git-annex.version;
-      sha256 = "0dnrihpdshrldais74jm5wjfw650i4va8znc1k2zq8gl9p4i8p39";
+      sha256 = "0f0pp0d5q4122cjh4j7iasnjh234fmkvlwgb3f49087cg8rr2czh";
    };
  }).override {
    dbus = if pkgs.stdenv.isLinux then self.dbus else null;
--- a/pkgs/development/haskell-modules/configuration-ghc-8.6.x.nix
+++ b/pkgs/development/haskell-modules/configuration-ghc-8.6.x.nix
@ -46,7 +46,7 @@ self: super: {

  # LTS-12.x versions do not compile.
  base-orphans = self.base-orphans_0_8;
-  brick = self.brick_0_41_3;
+  brick = self.brick_0_41_4;
  cassava-megaparsec = doJailbreak super.cassava-megaparsec;
  config-ini = doJailbreak super.config-ini;   # https://github.com/aisamanra/config-ini/issues/18
  contravariant = self.contravariant_1_5;
--- a/pkgs/development/haskell-modules/configuration-ghc-head.nix
+++ b/pkgs/development/haskell-modules/configuration-ghc-head.nix
@ -40,7 +40,7 @@ self: super: {
  xhtml = null;

  # jailbreak-cabal can use the native Cabal library.
-  jailbreak-cabal = pkgs.haskell.packages.ghc802.jailbreak-cabal;
+  jailbreak-cabal = super.jailbreak-cabal.override { Cabal = null; };

  # haddock: No input file(s).
  nats = dontHaddock super.nats;
--- a/pkgs/development/haskell-modules/hackage-packages.nix
+++ b/pkgs/development/haskell-modules/hackage-packages.nix
--- a/pkgs/development/libraries/SDL/default.nix
+++ b/pkgs/development/libraries/SDL/default.nix
@ -91,7 +91,7 @@ stdenv.mkDerivation rec {
    # Ticket: https://bugs.freedesktop.org/show_bug.cgi?id=27222
    (fetchpatch {
      name = "SDL_SetGamma.patch";
-      url = "http://src.fedoraproject.org/cgit/rpms/SDL.git/plain/SDL-1.2.15-x11-Bypass-SetGammaRamp-when-changing-gamma.patch?id=04a3a7b1bd88c2d5502292fad27e0e02d084698d";
+      url = "https://src.fedoraproject.org/cgit/rpms/SDL.git/plain/SDL-1.2.15-x11-Bypass-SetGammaRamp-when-changing-gamma.patch?id=04a3a7b1bd88c2d5502292fad27e0e02d084698d";
      sha256 = "0x52s4328kilyq43i7psqkqg7chsfwh0aawr50j566nzd7j51dlv";
    })
    # Fix a build failure on OS X Mavericks
--- a/pkgs/development/libraries/confuse/default.nix
+++ b/pkgs/development/libraries/confuse/default.nix
@ -1,17 +0,0 @@
-{stdenv, fetchurl}:
-
-stdenv.mkDerivation rec {
-  name = "confuse-${version}";
-  version = "3.2.1";
-  src = fetchurl {
-    url = "https://github.com/martinh/libconfuse/releases/download/v${version}/${name}.tar.xz";
-    sha256 = "0pnjmlj9i0alp407qd7c0vq83sz7gpsjrbdgpcn4xvzjp9r35ii3";
-  };
-
-  meta = {
-    homepage = http://www.nongnu.org/confuse/;
-    description = "Configuration file parser library";
-    license = stdenv.lib.licenses.isc;
-    platforms = stdenv.lib.platforms.unix;
-  };
-}
--- a/pkgs/development/libraries/cyrus-sasl/default.nix
+++ b/pkgs/development/libraries/cyrus-sasl/default.nix
@ -24,7 +24,7 @@ stdenv.mkDerivation rec {
    ./missing-size_t.patch # https://bugzilla.redhat.com/show_bug.cgi?id=906519
    (fetchpatch {
      name = "CVE-2013-4122.patch";
-      url = "http://sourceforge.net/projects/miscellaneouspa/files/glibc217/cyrus-sasl-2.1.26-glibc217-crypt.diff";
+      url = "mirror://sourceforge/miscellaneouspa/files/glibc217/cyrus-sasl-2.1.26-glibc217-crypt.diff";
      sha256 = "05l7dh1w9d5fvzg0pjwzqh0fy4ah8y5cv6v67s4ssbq8xwd4pkf2";
    })
  ] ++ lib.optional stdenv.isFreeBSD (
--- a/pkgs/development/libraries/dssi/default.nix
+++ b/pkgs/development/libraries/dssi/default.nix
@ -24,6 +24,6 @@ stdenv.mkDerivation rec {
    ];
    platforms = platforms.linux;
    license = licenses.lgpl21;
-    downloadPage = "http://sourceforge.net/projects/dssi/files/dssi/";
+    downloadPage = "https://sourceforge.net/projects/dssi/files/dssi/";
  };
 }
--- a/pkgs/development/libraries/kyotocabinet/default.nix
+++ b/pkgs/development/libraries/kyotocabinet/default.nix
@ -26,7 +26,7 @@ stdenv.mkDerivation rec {

  patches = [(fetchurl {
    name = "gcc6.patch";
-    url = "http://src.fedoraproject.org/rpms/kyotocabinet/raw/master/f/kyotocabinet-1.2.76-gcc6.patch";
+    url = "https://src.fedoraproject.org/rpms/kyotocabinet/raw/master/f/kyotocabinet-1.2.76-gcc6.patch";
    sha256 = "1h5k38mkiq7lz8nd2gbn7yvimcz49g3z7phn1cr560bzjih8rz23";
  })];

--- a/pkgs/development/libraries/libftdi/1.x.nix
+++ b/pkgs/development/libraries/libftdi/1.x.nix
@ -1,4 +1,4 @@
-{ stdenv, fetchurl, cmake, pkgconfig, libusb1, confuse
+{ stdenv, fetchurl, cmake, pkgconfig, libusb1, libconfuse
 , cppSupport ? true, boost ? null
 , pythonSupport ? true, python ? null, swig ? null
 , docSupport ? true, doxygen ? null
@ -16,8 +16,8 @@ stdenv.mkDerivation rec {
    sha256 = "0x0vncf6i92slgrn0h7ghkskqbglbs534220qa84d0qg114zndpc";
  };

-  nativeBuildInputs = [ pkgconfig ];
-  buildInputs = with stdenv.lib; [ cmake confuse ]
+  nativeBuildInputs = [ cmake pkgconfig ];
+  buildInputs = with stdenv.lib; [ libconfuse ]
    ++ optionals cppSupport [ boost ]
    ++ optionals pythonSupport [ python swig ]
    ++ optionals docSupport [ doxygen ];
--- a/pkgs/development/libraries/libipfix/default.nix
+++ b/pkgs/development/libraries/libipfix/default.nix
@ -4,7 +4,7 @@ stdenv.mkDerivation rec {
  name = "libipfix-${version}";
  version = "110209";
  src = fetchurl {
-    url = "http://sourceforge.net/projects/libipfix/files/libipfix/libipfix_110209.tgz";
+    url = "mirror://sourceforge/libipfix/files/libipfix/libipfix_110209.tgz";
    sha256 = "0h7v0sxjjdc41hl5vq2x0yhyn04bczl11bqm97825mivrvfymhn6";
  };
  meta = with stdenv.lib; {
--- a/pkgs/development/libraries/libmatheval/default.nix
+++ b/pkgs/development/libraries/libmatheval/default.nix
@ -15,26 +15,26 @@ stdenv.mkDerivation rec {
  # Patches coming from debian package
  # https://packages.debian.org/source/sid/libs/libmatheval
  patches = [ (fetchpatch {
-                url = "http://anonscm.debian.org/cgit/debian-science/packages/libmatheval.git/plain/debian/patches/002-skip-docs.patch";
+                url = "https://salsa.debian.org/science-team/libmatheval/raw/debian/1.1.11+dfsg-3/debian/patches/002-skip-docs.patch";
                sha256 = "1nnkk9aw4jj6nql46zhwq6vx74zrmr1xq5ix0xyvpawhabhgjg62";
              } )
              (fetchpatch {
-                url = "http://anonscm.debian.org/cgit/debian-science/packages/libmatheval.git/plain/debian/patches/003-guile2.0.patch";
+                url = "https://salsa.debian.org/science-team/libmatheval/raw/debian/1.1.11+dfsg-3/debian/patches/003-guile2.0.patch";
                sha256 = "1xgfw4finfvr20kjbpr4yl2djxmyr4lmvfa11pxirfvhrdi602qj";
               } )
              (fetchpatch {
-                url = "http://anonscm.debian.org/cgit/debian-science/packages/libmatheval.git/plain/debian/patches/disable_coth_test.patch";
+                url = "https://salsa.debian.org/science-team/libmatheval/raw/debian/1.1.11+dfsg-3/debian/patches/disable_coth_test.patch";
                sha256 = "0bai8jrd5azfz5afmjixlvifk34liq58qb7p9kb45k6kc1fqqxzm";
               } )
            ];
-  
+
  meta = {
    description = "A library to parse and evaluate symbolic expressions input as text";
    longDescription = ''
-      GNU libmatheval is a library (callable from C and Fortran) to parse and evaluate symbolic 
-      expressions input as text. It supports expressions in any number of variables of arbitrary 
-      names, decimal and symbolic constants, basic unary and binary operators, and elementary 
-      mathematical functions. In addition to parsing and evaluation, libmatheval can also compute 
+      GNU libmatheval is a library (callable from C and Fortran) to parse and evaluate symbolic
+      expressions input as text. It supports expressions in any number of variables of arbitrary
+      names, decimal and symbolic constants, basic unary and binary operators, and elementary
+      mathematical functions. In addition to parsing and evaluation, libmatheval can also compute
      symbolic derivatives and output expressions to strings.
    '';
    homepage = https://www.gnu.org/software/libmatheval/;
--- a/pkgs/development/libraries/libp11/default.nix
+++ b/pkgs/development/libraries/libp11/default.nix
@ -2,13 +2,13 @@

 stdenv.mkDerivation rec {
  name = "libp11-${version}";
-  version = "0.4.7";
+  version = "0.4.9";

  src = fetchFromGitHub {
    owner = "OpenSC";
    repo = "libp11";
    rev = name;
-    sha256 = "0n1i0pxj6l0vdq8gpdwfp5p9qd7wkymg0lpy6a17ix8hpqsljlhr";
+    sha256 = "1f0ir1mnr4wxxnql8ld2aa6288fn04fai5pr0sics7kbdm1g0cki";
  };

  makeFlags = [ "DESTDIR=$(out)" "PREFIX=" ];
--- a/pkgs/development/libraries/libtar/default.nix
+++ b/pkgs/development/libraries/libtar/default.nix
@ -14,7 +14,7 @@ stdenv.mkDerivation rec {
  patches = let
    fp =  name: sha256:
      fetchpatch {
-        url = "http://sources.debian.net/data/main/libt/libtar/1.2.20-4/debian/patches/${name}.patch";
+        url = "https://sources.debian.net/data/main/libt/libtar/1.2.20-4/debian/patches/${name}.patch";
        inherit sha256;
      };
    in [
@ -29,7 +29,7 @@ stdenv.mkDerivation rec {

  meta = with stdenv.lib; {
    description = "C library for manipulating POSIX tar files";
-    homepage = http://repo.or.cz/libtar;
+    homepage = https://repo.or.cz/libtar;
    license = licenses.bsd3;
    platforms = with platforms; linux ++ darwin;
    maintainers = [ maintainers.bjornfor ];
--- a/pkgs/development/libraries/ntrack/default.nix
+++ b/pkgs/development/libraries/ntrack/default.nix
@ -8,7 +8,7 @@ stdenv.mkDerivation rec {
  name = "ntrack-${version}";

  src = fetchurl {
-    url = "http://launchpad.net/ntrack/main/${version}/+download/${name}.tar.gz";
+    url = "https://launchpad.net/ntrack/main/${version}/+download/${name}.tar.gz";
    sha256 = "037ig5y0mp327m0hh4pnfr3vmsk3wrxgfjy3645q4ws9vdhx807w";
  };

--- a/pkgs/development/libraries/spice-gtk/default.nix
+++ b/pkgs/development/libraries/spice-gtk/default.nix
@ -74,7 +74,7 @@ in stdenv.mkDerivation rec {
      Python bindings are available too.
    '';

-    homepage = http://www.spice-space.org/;
+    homepage = https://www.spice-space.org/;
    license = licenses.lgpl21;
    maintainers = [ maintainers.xeji ];
    platforms = platforms.linux;
--- a/pkgs/development/libraries/spice-protocol/default.nix
+++ b/pkgs/development/libraries/spice-protocol/default.nix
@ -15,7 +15,7 @@ stdenv.mkDerivation rec {

  meta = with stdenv.lib; {
    description = "Protocol headers for the SPICE protocol";
-    homepage = http://www.spice-space.org;
+    homepage = https://www.spice-space.org/;
    license = licenses.bsd3;
    maintainers = with maintainers; [ bluescreen303 ];
    platforms = platforms.linux;
--- a/pkgs/development/libraries/spice/default.nix
+++ b/pkgs/development/libraries/spice/default.nix
@ -41,7 +41,7 @@ stdenv.mkDerivation rec {
      VD-Interfaces. The VD-Interfaces (VDI) enable both ends of the solution to be easily
      utilized by a third-party component.
    '';
-    homepage = http://www.spice-space.org/;
+    homepage = https://www.spice-space.org/;
    license = licenses.lgpl21;

    maintainers = [ maintainers.bluescreen303 ];
--- a/pkgs/development/libraries/taglib/1.9.nix
+++ b/pkgs/development/libraries/taglib/1.9.nix
@ -4,7 +4,7 @@ stdenv.mkDerivation rec {
  name = "taglib-1.9.1";

  src = fetchurl {
-    url = http://taglib.github.io/releases/taglib-1.9.1.tar.gz;
+    url = https://taglib.github.io/releases/taglib-1.9.1.tar.gz;
    sha256 = "06n7gnbcqa3r6c9gv00y0y1r48dyyazm6yj403i7ma0r2k6p3lvj";
  };

@ -13,7 +13,7 @@ stdenv.mkDerivation rec {
  buildInputs = [ zlib ];

  meta = {
-    homepage = http://developer.kde.org/~wheeler/taglib.html;
+    homepage = https://taglib.org/;
    repositories.git = git://github.com/taglib/taglib.git;
    description = "A library for reading and editing the meta-data of several popular audio formats";
    inherit (cmake.meta) platforms;
--- a/Show More
+++ b/Show More