From c6e5cf2e7472ab872a537327a03ef9eb9fcef2a1 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 17 Mar 2021 13:34:17 -0700
Subject: Benchmark improvements.

* Build `+RTS -A256m -RTS` into default ghc-options for benchmark,
  so we don't have to specify this separately on the command line.
  This is necessary to get accurate benchmark results; otherwise
  we are largely measuring garbage collecting, some not related
  to the current benchmark.
* Switch back from gauge to tasty-bench.
* Allow specifying BASELINE file in 'make bench' for comparison
  (otherwise the latest is chosen by default).
* Remove obsolete reference to weigh-pandoc from CONTRIBUTING.md.
* Remove `-Rghc-timing` from 'make bench'.
---
 CONTRIBUTING.md               |  3 ---
 Makefile                      | 18 +++++++-----------
 benchmark/benchmark-pandoc.hs |  4 ++--
 pandoc.cabal                  |  6 ++++--
 4 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 090cc0a4f..cf8d3aa03 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -318,9 +318,6 @@ With stack:
 
     stack bench
 
-You can also build `weigh-pandoc` (`stack build pandoc:weigh-pandoc`)
-to get some statistics on memory usage.  (Eventually this should
-be incorporated into the benchmark suite.)
 
 Using the REPL
 --------------
diff --git a/Makefile b/Makefile
index 0ef0406f8..5e5bc7c51 100644
--- a/Makefile
+++ b/Makefile
@@ -7,21 +7,19 @@ DOCKERIMAGE=registry.gitlab.b-data.ch/ghc/ghc4pandoc:8.10.4
 COMMIT=$(shell git rev-parse --short HEAD)
 TIMESTAMP=$(shell date "+%Y%m%d_%H%M")
 LATESTBENCH=$(word 1,$(shell ls -t bench_*.csv 2>/dev/null))
-ifeq ($(LATESTBENCH),)
-BASELINE=
+BASELINE?=$(LATESTBENCH)
+ifeq ($(BASELINE),)
+BASELINECMD=
 else
-BASELINE=--baseline $(LATESTBENCH)
+BASELINECMD=--baseline $(BASELINE)
 endif
 GHCOPTS=-fdiagnostics-color=always -j4 +RTS -A256m -RTS
 WEBSITE=../../web/pandoc.org
 REVISION?=1
-# Note: for benchmarks we use +RTS -A256m -I0 -RTS ; otherwise the benchmarks
-# are measuring garbage collecting, and this can vary depending on which
-# other benchmarks are run.
 # For gauge:
-BENCHARGS?=--small --ci=0.90 --match=pattern $(PATTERN) +RTS -T -A256m -I0 -RTS
+# BENCHARGS?=--small --ci=0.90 --match=pattern $(PATTERN)
 # For tasty-bench:
-# BENCHARGS?=--csv bench_$(TIMESTAMP).csv --timeout=6 +RTS -T -A256m -I0 -RTS $(if $(PATTERN),--pattern "$(PATTERN)",)
+BENCHARGS?=--csv bench_$(TIMESTAMP).csv $(BASELINECMD) --timeout=6 +RTS -T -RTS $(if $(PATTERN),--pattern "$(PATTERN)",)
 
 quick:
 	stack install --ghc-options='$(GHCOPTS)' --install-ghc --flag 'pandoc:embed_data_files' --fast --test --ghc-options='$(GHCOPTS)' --test-arguments='-j4 --hide-successes $(TESTARGS)'
@@ -58,11 +56,9 @@ ghcid-test:
 
 bench:
 	stack bench \
-	  --ghc-options '-Rghc-timing $(GHCOPTS)' \
+	  --ghc-options '$(GHCOPTS)' \
 	  --benchmark-arguments='$(BENCHARGS)' 2>&1 | \
 	  tee "bench_latest.txt"
-	perl -ne 'if (/\r/) { s/\x1b\[[0-9;]*[mGK]//g;s/^.*\r//;print; }' \
-	  bench_latest.txt > "bench_$(TIMESTAMP).txt"
 
 reformat:
 	for f in $(SOURCEFILES); do echo $$f; stylish-haskell -i $$f ; done
diff --git a/benchmark/benchmark-pandoc.hs b/benchmark/benchmark-pandoc.hs
index 111e63274..1890a998f 100644
--- a/benchmark/benchmark-pandoc.hs
+++ b/benchmark/benchmark-pandoc.hs
@@ -23,8 +23,8 @@ import Control.Monad.Except (throwError)
 import qualified Text.Pandoc.UTF8 as UTF8
 import qualified Data.ByteString as B
 import qualified Data.Text as T
--- import Test.Tasty.Bench
-import Gauge
+import Test.Tasty.Bench
+-- import Gauge
 import qualified Data.ByteString.Lazy as BL
 import Data.Maybe (mapMaybe)
 import Data.List (sortOn)
diff --git a/pandoc.cabal b/pandoc.cabal
index 2f49f0ac3..3a7436370 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -826,9 +826,11 @@ benchmark benchmark-pandoc
   hs-source-dirs:  benchmark
   build-depends:   bytestring,
                    containers,
-                   gauge       >= 0.2     && < 0.3,
-                   -- tasty-bench >= 0.2     && <= 0.3,
+                   -- gauge       >= 0.2     && < 0.3,
+                   tasty-bench >= 0.2     && <= 0.3,
                    mtl         >= 2.2     && < 2.3,
                    text        >= 1.1.1.0 && < 1.3,
                    time,
                    deepseq
+  -- we increase heap size to avoid benchmarking garbage collection:
+  ghc-options:     -rtsopts -with-rtsopts=-A256m -threaded
-- 
cgit v1.2.3