Change and document the behaviour of --diff-cutoff-percent

benchmark/bench-report/BenchReport.hs

@@ -209,25 +209,23 @@ selectBench
     -> (SortColumn -> Maybe GroupStyle -> Either String [(String, Double)])
     -> [String]
 selectBench Options{..} f =
-    reverse
-    $ fmap fst
-    $ filter (\(_,y) -> filterPred y)
-    $ either
-      (const $ either error sortFunc $ f (ColumnIndex 0) (Just PercentDiff))
-      sortFunc
-      $ f (ColumnIndex 1) (Just PercentDiff)
+    -- Apply filterPred only if at least 2 columns exist
+    let colVals =
+            case f (ColumnIndex 1) (Just PercentDiff) of
+                Left _ -> either error id $ f (ColumnIndex 0) (Just PercentDiff)
+                Right xs -> filter (filterPred . snd) xs
+    in reverse
+           $ fmap fst
+           $ sortFunc colVals
 
     where
 
     sortFunc = if sortByName then sortOn fst else sortOn snd
 
-    cutOffMultiples = 1 + cutOffPercent / 100
-
-    filterPred x =
-        case benchType of
-            Just (Compare _) ->
-                x <= negate cutOffPercent || x >= cutOffPercent
-            _ -> True
+    filterPred x
+        | cutOffPercent > 0 = x >= cutOffPercent
+        | cutOffPercent < 0 = x <= cutOffPercent
+        | otherwise = True
 
 benchShow ::
        Options

benchmark/bench-report/bin/bench-runner.sh

@@ -21,7 +21,7 @@ print_help () {
   echo "       [--sort-by-name]"
   echo "       [--compare]"
   echo "       [--diff-style <absolute|percent|multiples>]"
-  echo "       [--cutoff-percent <percent-value>]"
+  echo "       [--diff-cutoff-percent <percent-value>]"
   echo "       [--graphs]"
   echo "       [--silent]"
   echo "       [--no-measure]"
@@ -40,6 +40,13 @@ print_help () {
   echo
   echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks"
   echo "--compare: compare the specified benchmarks with each other"
+
+  echo "--diff-cutoff-percent: Diff percentage used for benchmark selection."
+  echo "This applies only to the second column of the report and makes sense"
+  echo "only while comparing benchmarks."
+  echo "A positive cutoff value selects only regressions, whereas, a negative"
+  echo "cutoff value selects only improvements."
+
   echo "--fields: measurement fields to report, use 'help' for a list"
   echo "--graphs: Generate graphical reports"
   echo "--no-measure: Don't run benchmarks, run reports from previous results"