From 385b6a3b215124fd2dfa044b8847d69a6cf14a73 Mon Sep 17 00:00:00 2001
From: David Martschenko <62178322+davidmrt98@users.noreply.github.com>
Date: Tue, 5 Jan 2021 19:15:59 +0100
Subject: Implement defaults file inheritance (#6924)

Allow defaults files to inherit options from other defaults files by
specifying them with the following syntax:
`defaults: [list of defaults files or single defaults file]`.
---
 MANUAL.txt                                |  10 +++
 pandoc.cabal                              |   7 ++
 src/Text/Pandoc/App/CommandLineOptions.hs |  36 +++-----
 src/Text/Pandoc/App/Opt.hs                | 136 ++++++++++++++++++++++++++++--
 test/command/defaults-inheritance-1.md    |   6 ++
 test/command/defaults-inheritance-2.md    |   5 ++
 test/command/defaults-inheritance-3.md    |   6 ++
 test/command/defaults3.yaml               |   4 +
 test/command/defaults4.yaml               |   3 +
 test/command/defaults5.yaml               |   2 +
 test/command/defaults6.yaml               |   2 +
 test/command/defaults7.yaml               |   2 +
 test/command/defaults8.yaml               |   2 +
 test/command/defaults9.yaml               |   1 +
 14 files changed, 189 insertions(+), 33 deletions(-)
 create mode 100644 test/command/defaults-inheritance-1.md
 create mode 100644 test/command/defaults-inheritance-2.md
 create mode 100644 test/command/defaults-inheritance-3.md
 create mode 100644 test/command/defaults3.yaml
 create mode 100644 test/command/defaults4.yaml
 create mode 100644 test/command/defaults5.yaml
 create mode 100644 test/command/defaults6.yaml
 create mode 100644 test/command/defaults7.yaml
 create mode 100644 test/command/defaults8.yaml
 create mode 100644 test/command/defaults9.yaml

diff --git a/MANUAL.txt b/MANUAL.txt
index 53cd54e5f..831ffb0b0 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -1507,6 +1507,16 @@ input-files:
 - content.md
 # or you may use input-file: with a single value
 
+# Include options from the specified defaults files.
+# The files will be searched for first in the working directory
+# and then in the defaults subdirectory of the user data directory.
+# The files are included in the same order in which they appear in
+# the list. Options specified in this defaults file always have
+# priority over the included ones.
+defaults:
+- defsA
+- defsB
+
 template: letter
 standalone: true
 self-contained: false
diff --git a/pandoc.cabal b/pandoc.cabal
index 794eef91d..cce3c1a58 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -216,6 +216,13 @@ extra-source-files:
                  test/command/01.csv
                  test/command/defaults1.yaml
                  test/command/defaults2.yaml
+                 test/command/defaults3.yaml
+                 test/command/defaults4.yaml
+                 test/command/defaults5.yaml
+                 test/command/defaults6.yaml
+                 test/command/defaults7.yaml
+                 test/command/defaults8.yaml
+                 test/command/defaults9.yaml
                  test/command/3533-rst-csv-tables.csv
                  test/command/3880.txt
                  test/command/5182.txt
diff --git a/src/Text/Pandoc/App/CommandLineOptions.hs b/src/Text/Pandoc/App/CommandLineOptions.hs
index 906fcc4c0..21ee47b7b 100644
--- a/src/Text/Pandoc/App/CommandLineOptions.hs
+++ b/src/Text/Pandoc/App/CommandLineOptions.hs
@@ -25,6 +25,7 @@ module Text.Pandoc.App.CommandLineOptions (
 import Control.Monad
 import Control.Monad.Trans
 import Control.Monad.Except (throwError)
+import Control.Monad.State.Strict
 import Data.Aeson.Encode.Pretty (encodePretty', Config(..), keyOrder,
          defConfig, Indent(..), NumberFormat(..))
 import Data.Bifunctor (second)
@@ -46,10 +47,12 @@ import System.FilePath
 import System.IO (stdout)
 import Text.DocTemplates (Context (..), ToContext (toVal), Val (..))
 import Text.Pandoc
-import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..), addMeta)
+import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..),
+                            DefaultsState (..), addMeta, applyDefaults,
+                            fullDefaultsPath)
 import Text.Pandoc.Filter (Filter (..))
 import Text.Pandoc.Highlighting (highlightingStyles)
-import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs, findM)
+import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs)
 import Text.Printf
 
 #ifdef EMBED_DATA_FILES
@@ -64,7 +67,6 @@ import qualified Data.ByteString as BS
 import qualified Data.ByteString.Lazy as B
 import qualified Data.Map as M
 import qualified Data.Text as T
-import qualified Data.YAML as Y
 import qualified Text.Pandoc.UTF8 as UTF8
 
 parseOptions :: [OptDescr (Opt -> IO Opt)] -> Opt -> IO Opt
@@ -166,7 +168,11 @@ options =
 
     , Option "d" ["defaults"]
                  (ReqArg
-                  (\arg opt -> applyDefaults opt arg
+                  (\arg opt -> runIOorExplode $ do
+                     let defsState = DefaultsState { curDefaults = Nothing,
+                                                     inheritanceGraph = [] }
+                     fp <- fullDefaultsPath (optDataDir opt) arg
+                     evalStateT (applyDefaults opt fp) defsState
                   )
                   "FILE")
                 ""
@@ -1012,28 +1018,6 @@ writersNames = sort
 splitField :: String -> (String, String)
 splitField = second (tailDef "true") . break (`elemText` ":=")
 
--- | Apply defaults from --defaults file.
-applyDefaults :: Opt -> FilePath -> IO Opt
-applyDefaults opt file = runIOorExplode $ do
-  let fp = if null (takeExtension file)
-              then addExtension file "yaml"
-              else file
-  setVerbosity $ optVerbosity opt
-  dataDirs <- liftIO defaultUserDataDirs
-  let fps = fp : case optDataDir opt of
-              Nothing -> map (</> ("defaults" </> fp))
-                               dataDirs
-              Just dd -> [dd </> "defaults" </> fp]
-  fp' <- fromMaybe fp <$> findM fileExists fps
-  inp <- readFileLazy fp'
-  case Y.decode1 inp of
-      Right (f :: Opt -> Opt) -> return $ f opt
-      Left (errpos, errmsg)  -> throwError $
-         PandocParseError $ T.pack $
-         "Error parsing " ++ fp' ++ " line " ++
-          show (Y.posLine errpos) ++ " column " ++
-          show (Y.posColumn errpos) ++ ":\n" ++ errmsg
-
 lookupHighlightStyle :: PandocMonad m => String -> m Style
 lookupHighlightStyle s
   | takeExtension s == ".theme" = -- attempt to load KDE theme
diff --git a/src/Text/Pandoc/App/Opt.hs b/src/Text/Pandoc/App/Opt.hs
index 00b4b5523..6dd19758e 100644
--- a/src/Text/Pandoc/App/Opt.hs
+++ b/src/Text/Pandoc/App/Opt.hs
@@ -20,10 +20,17 @@ module Text.Pandoc.App.Opt (
             Opt(..)
           , LineEnding (..)
           , IpynbOutput (..)
+          , DefaultsState (..)
           , defaultOpts
           , addMeta
+          , applyDefaults
+          , fullDefaultsPath
           ) where
+import Control.Monad.Except (MonadIO, liftIO, throwError, (>=>), foldM)
+import Control.Monad.State.Strict (StateT, modify, gets)
+import System.FilePath ( addExtension, (</>), takeExtension )
 import Data.Char (isLower, toLower)
+import Data.Maybe (fromMaybe)
 import GHC.Generics hiding (Meta)
 import Text.Pandoc.Builder (setMeta)
 import Text.Pandoc.Filter (Filter (..))
@@ -34,7 +41,9 @@ import Text.Pandoc.Options (TopLevelDivision (TopLevelDefault),
                             ReferenceLocation (EndOfDocument),
                             ObfuscationMethod (NoObfuscation),
                             CiteMethod (Citeproc))
-import Text.Pandoc.Shared (camelCaseStrToHyphenated)
+import Text.Pandoc.Class (readFileLazy, fileExists, setVerbosity, PandocMonad)
+import Text.Pandoc.Error (PandocError (PandocParseError, PandocSomeError))
+import Text.Pandoc.Shared (camelCaseStrToHyphenated, defaultUserDataDirs, findM, ordNub)
 import qualified Text.Pandoc.Parsing as P
 import Text.Pandoc.Readers.Metadata (yamlMap)
 import Text.Pandoc.Class.PandocPure
@@ -150,16 +159,77 @@ data Opt = Opt
     } deriving (Generic, Show)
 
 instance FromYAML (Opt -> Opt) where
-  parseYAML (Mapping _ _ m) =
-    foldr (.) id <$> mapM doOpt (M.toList m)
+  parseYAML (Mapping _ _ m) = chain doOpt (M.toList m)
   parseYAML n = failAtNode n "Expected a mapping"
 
+data DefaultsState = DefaultsState
+    {
+      curDefaults      :: Maybe FilePath -- currently parsed file
+    , inheritanceGraph :: [[FilePath]]   -- defaults file inheritance graph
+    } deriving (Show)
+
+instance (PandocMonad m, MonadIO m)
+      => FromYAML (Opt -> StateT DefaultsState m Opt) where
+  parseYAML (Mapping _ _ m) = do
+    let opts = M.mapKeys toText m
+    dataDir <- case M.lookup "data-dir" opts of
+      Nothing -> return Nothing
+      Just v -> Just . unpack <$> parseYAML v
+    f <- parseOptions $ M.toList m
+    case M.lookup "defaults" opts of
+      Just v -> do
+        g <- parseDefaults v dataDir
+        return  $ g >=> f
+      Nothing -> return f
+    where
+      toText (Scalar _ (SStr s)) = s
+      toText _ = ""
+  parseYAML n = failAtNode n "Expected a mapping"
+
+parseDefaults :: (PandocMonad m, MonadIO m)
+              => Node Pos
+              -> Maybe FilePath
+              -> Parser (Opt -> StateT DefaultsState m Opt)
+parseDefaults n dataDir = parseDefsNames n >>= \ds -> return $ \o -> do
+  -- get parent defaults:
+  defsParent <- gets $ fromMaybe "" . curDefaults
+  -- get child defaults:
+  defsChildren <- mapM (fullDefaultsPath dataDir) ds
+  -- expand parent in defaults inheritance graph by children:
+  defsGraph <- gets inheritanceGraph
+  let defsGraphExp = expand defsGraph defsChildren defsParent
+  modify $ \defsState -> defsState{ inheritanceGraph = defsGraphExp }
+  -- check for cyclic inheritance:
+  if cyclic defsGraphExp
+    then throwError $
+      PandocSomeError $ T.pack $
+        "Error: Circular defaults file reference in " ++
+        "'" ++ defsParent ++ "'"
+    else foldM applyDefaults o defsChildren
+  where parseDefsNames x = (parseYAML x >>= \xs -> return $ map unpack xs)
+                       <|> (parseYAML x >>= \x' -> return [unpack x'])
+
+parseOptions :: Monad m
+             => [(Node Pos, Node Pos)]
+             -> Parser (Opt -> StateT DefaultsState m Opt)
+parseOptions ns = do
+  f <- chain doOpt' ns
+  return $ return . f
+
+chain :: Monad m => (a -> m (b -> b)) -> [a] -> m (b -> b)
+chain f = foldM g id
+  where g o n = f n >>= \o' -> return $ o' . o
+
+doOpt' :: (Node Pos, Node Pos) -> Parser (Opt -> Opt)
+doOpt' (k',v) = do
+  k <- parseStringKey k'
+  case k of
+    "defaults" -> return id
+    _ -> doOpt (k',v)
+
 doOpt :: (Node Pos, Node Pos) -> Parser (Opt -> Opt)
 doOpt (k',v) = do
-  k <- case k' of
-         Scalar _ (SStr t) -> return t
-         Scalar _ _ -> failAtNode k' "Non-string key"
-         _ -> failAtNode k' "Non-scalar key"
+  k <- parseStringKey k'
   case k of
     "tab-stop" ->
       parseYAML v >>= \x -> return (\o -> o{ optTabStop = x })
@@ -494,6 +564,12 @@ defaultOpts = Opt
     , optStripComments         = False
     }
 
+parseStringKey ::  Node Pos -> Parser Text
+parseStringKey k = case k of
+  Scalar _ (SStr t) -> return t
+  Scalar _ _ -> failAtNode k "Non-string key"
+  _ -> failAtNode k "Non-scalar key"
+
 yamlToMeta :: Node Pos -> Parser Meta
 yamlToMeta (Mapping _ _ m) =
     either (fail . show) return $ runEverything (yamlMap pMetaString m)
@@ -524,6 +600,52 @@ readMetaValue s
   | s == "FALSE" = MetaBool False
   | otherwise    = MetaString $ T.pack s
 
+-- | Apply defaults from --defaults file.
+applyDefaults :: (PandocMonad m, MonadIO m)
+              => Opt
+              -> FilePath
+              -> StateT DefaultsState m Opt
+applyDefaults opt file = do
+  setVerbosity $ optVerbosity opt
+  modify $ \defsState -> defsState{ curDefaults = Just file }
+  inp <- readFileLazy file
+  case decode1 inp of
+      Right f -> f opt
+      Left (errpos, errmsg)  -> throwError $
+         PandocParseError $ T.pack $
+         "Error parsing " ++ file ++ " line " ++
+          show (posLine errpos) ++ " column " ++
+          show (posColumn errpos) ++ ":\n" ++ errmsg
+
+fullDefaultsPath :: (PandocMonad m, MonadIO m)
+                 => Maybe FilePath
+                 -> FilePath
+                 -> m FilePath
+fullDefaultsPath dataDir file = do
+  let fp = if null (takeExtension file)
+              then addExtension file "yaml"
+              else file
+  dataDirs <- liftIO defaultUserDataDirs
+  let fps = fp : case dataDir of
+                   Nothing -> map (</> ("defaults" </> fp))
+                                    dataDirs
+                   Just dd -> [dd </> "defaults" </> fp]
+  fromMaybe fp <$> findM fileExists fps
+
+-- | In a list of lists, append another list in front of every list which
+-- starts with specific element.
+expand :: Ord a => [[a]] -> [a] -> a -> [[a]]
+expand [] ns n = fmap (\x -> x : [n]) ns
+expand ps ns n = concatMap (ext n ns) ps
+  where
+    ext x xs p = case p of
+      (l : _) | x == l -> fmap (: p) xs
+      _ -> [p]
+
+cyclic :: Ord a => [[a]] -> Bool
+cyclic = any hasDuplicate
+  where
+    hasDuplicate xs = length (ordNub xs) /= length xs
 
 -- see https://github.com/jgm/pandoc/pull/4083
 -- using generic deriving caused long compilation times
diff --git a/test/command/defaults-inheritance-1.md b/test/command/defaults-inheritance-1.md
new file mode 100644
index 000000000..0760e2ec0
--- /dev/null
+++ b/test/command/defaults-inheritance-1.md
@@ -0,0 +1,6 @@
+```
+% pandoc -d command/defaults3
+# Header
+^D
+# Header
+```
diff --git a/test/command/defaults-inheritance-2.md b/test/command/defaults-inheritance-2.md
new file mode 100644
index 000000000..8b26a2613
--- /dev/null
+++ b/test/command/defaults-inheritance-2.md
@@ -0,0 +1,5 @@
+```
+% pandoc -d command/defaults6
+^D
+Error: Circular defaults file reference in 'command/defaults7.yaml'
+```
diff --git a/test/command/defaults-inheritance-3.md b/test/command/defaults-inheritance-3.md
new file mode 100644
index 000000000..81cac7baa
--- /dev/null
+++ b/test/command/defaults-inheritance-3.md
@@ -0,0 +1,6 @@
+```
+% pandoc -d command/defaults8
+<h1>Header</h1>
+^D
+# Header
+```
diff --git a/test/command/defaults3.yaml b/test/command/defaults3.yaml
new file mode 100644
index 000000000..d8b6f9144
--- /dev/null
+++ b/test/command/defaults3.yaml
@@ -0,0 +1,4 @@
+defaults:
+  - command/defaults4
+  - command/defaults5
+to: markdown
diff --git a/test/command/defaults4.yaml b/test/command/defaults4.yaml
new file mode 100644
index 000000000..a6caa985b
--- /dev/null
+++ b/test/command/defaults4.yaml
@@ -0,0 +1,3 @@
+from: html
+defaults:
+  - command/defaults5
diff --git a/test/command/defaults5.yaml b/test/command/defaults5.yaml
new file mode 100644
index 000000000..bb48b9708
--- /dev/null
+++ b/test/command/defaults5.yaml
@@ -0,0 +1,2 @@
+from: markdown
+to: html
diff --git a/test/command/defaults6.yaml b/test/command/defaults6.yaml
new file mode 100644
index 000000000..ac4a819e5
--- /dev/null
+++ b/test/command/defaults6.yaml
@@ -0,0 +1,2 @@
+defaults:
+  - command/defaults7
diff --git a/test/command/defaults7.yaml b/test/command/defaults7.yaml
new file mode 100644
index 000000000..19ca8f09e
--- /dev/null
+++ b/test/command/defaults7.yaml
@@ -0,0 +1,2 @@
+defaults:
+  - command/defaults6
diff --git a/test/command/defaults8.yaml b/test/command/defaults8.yaml
new file mode 100644
index 000000000..e418d33b2
--- /dev/null
+++ b/test/command/defaults8.yaml
@@ -0,0 +1,2 @@
+from: html
+defaults: command/defaults9
diff --git a/test/command/defaults9.yaml b/test/command/defaults9.yaml
new file mode 100644
index 000000000..d732eb066
--- /dev/null
+++ b/test/command/defaults9.yaml
@@ -0,0 +1 @@
+to: markdown
-- 
cgit v1.2.3