From 1319bbfe4ab3ddd321bcbb902bba7392ad868324 Mon Sep 17 00:00:00 2001 From: Jasper Van der Jeugt Date: Fri, 9 Nov 2012 16:34:45 +0100 Subject: Remove Resource type --- src/Hakyll/Core/Compiler.hs | 36 ++--- src/Hakyll/Core/Compiler/Internal.hs | 2 +- src/Hakyll/Core/Identifier.hs | 17 ++- src/Hakyll/Core/Metadata.hs | 24 ++++ src/Hakyll/Core/Resource.hs | 51 ------- src/Hakyll/Core/Resource/Metadata.hs | 118 ---------------- src/Hakyll/Core/Resource/MetadataCache.hs | 61 --------- src/Hakyll/Core/Resource/Modified.hs | 82 ----------- src/Hakyll/Core/Resource/Pattern.hs | 160 ---------------------- src/Hakyll/Core/Resource/Provider.hs | 45 ------ src/Hakyll/Core/Resource/Provider/Internal.hs | 84 ------------ src/Hakyll/Core/ResourceProvider.hs | 46 +++++++ src/Hakyll/Core/ResourceProvider/Internal.hs | 86 ++++++++++++ src/Hakyll/Core/ResourceProvider/Metadata.hs | 119 ++++++++++++++++ src/Hakyll/Core/ResourceProvider/MetadataCache.hs | 62 +++++++++ src/Hakyll/Core/ResourceProvider/Modified.hs | 83 +++++++++++ src/Hakyll/Core/Rules.hs | 27 ++-- src/Hakyll/Core/Rules/Internal.hs | 7 +- src/Hakyll/Core/Run.hs | 7 +- src/Hakyll/Core/Writable/CopyFile.hs | 19 ++- 20 files changed, 479 insertions(+), 657 deletions(-) create mode 100644 src/Hakyll/Core/Metadata.hs delete mode 100644 src/Hakyll/Core/Resource.hs delete mode 100644 src/Hakyll/Core/Resource/Metadata.hs delete mode 100644 src/Hakyll/Core/Resource/MetadataCache.hs delete mode 100644 src/Hakyll/Core/Resource/Modified.hs delete mode 100644 src/Hakyll/Core/Resource/Pattern.hs delete mode 100644 src/Hakyll/Core/Resource/Provider.hs delete mode 100644 src/Hakyll/Core/Resource/Provider/Internal.hs create mode 100644 src/Hakyll/Core/ResourceProvider.hs create mode 100644 src/Hakyll/Core/ResourceProvider/Internal.hs create mode 100644 src/Hakyll/Core/ResourceProvider/Metadata.hs create mode 100644 src/Hakyll/Core/ResourceProvider/MetadataCache.hs create mode 100644 src/Hakyll/Core/ResourceProvider/Modified.hs (limited to 'src/Hakyll/Core') diff --git a/src/Hakyll/Core/Compiler.hs b/src/Hakyll/Core/Compiler.hs index 92fcff8..e1eab79 100644 --- a/src/Hakyll/Core/Compiler.hs +++ b/src/Hakyll/Core/Compiler.hs @@ -93,7 +93,6 @@ module Hakyll.Core.Compiler ( Compiler , runCompiler , getIdentifier - , getResource , getRoute , getRouteFor , getResourceString @@ -135,8 +134,7 @@ import Hakyll.Core.Identifier import Hakyll.Core.Identifier.Pattern import Hakyll.Core.CompiledItem import Hakyll.Core.Writable -import Hakyll.Core.Resource -import Hakyll.Core.Resource.Provider +import Hakyll.Core.ResourceProvider import Hakyll.Core.Compiler.Internal import Hakyll.Core.Store (Store) import Hakyll.Core.Rules.Internal @@ -181,11 +179,6 @@ getIdentifier :: Compiler a (Identifier b) getIdentifier = fromJob $ const $ CompilerM $ castIdentifier . compilerIdentifier <$> ask --- | Get the resource that is currently being compiled --- -getResource :: Compiler a Resource -getResource = getIdentifier >>> arr fromIdentifier - -- | Get the route we are using for this item -- getRoute :: Compiler a (Maybe FilePath) @@ -200,22 +193,23 @@ getRouteFor = fromJob $ \identifier -> CompilerM $ do -- | Get the resource we are compiling as a string -- -getResourceString :: Compiler Resource String +getResourceString :: Compiler a String getResourceString = getResourceWith resourceString -- | Get the resource we are compiling as a lazy bytestring -- -getResourceLBS :: Compiler Resource ByteString +getResourceLBS :: Compiler a ByteString getResourceLBS = getResourceWith resourceLBS -- | Overloadable function for 'getResourceString' and 'getResourceLBS' -- -getResourceWith :: (Resource -> IO a) -> Compiler Resource a -getResourceWith reader = fromJob $ \r -> CompilerM $ do - let filePath = unResource r +getResourceWith :: (Identifier a -> IO b) -> Compiler c b +getResourceWith reader = fromJob $ \_ -> CompilerM $ do provider <- compilerResourceProvider <$> ask + r <- compilerIdentifier <$> ask + let filePath = toFilePath r if resourceExists provider r - then liftIO $ reader r + then liftIO $ reader $ castIdentifier r else throwError $ error' filePath where error' id' = "Hakyll.Core.Compiler.getResourceWith: resource " @@ -299,17 +293,17 @@ requireAllA pattern = (id &&& requireAll_ pattern >>>) cached :: (Binary a, Typeable a, Writable a) => String - -> Compiler Resource a - -> Compiler Resource a + -> Compiler () a + -> Compiler () a cached name (Compiler d j) = Compiler d $ const $ CompilerM $ do - logger <- compilerLogger <$> ask + logger <- compilerLogger <$> ask identifier <- castIdentifier . compilerIdentifier <$> ask - store <- compilerStore <$> ask - modified <- compilerResourceModified <$> ask - progName <- liftIO getProgName + store <- compilerStore <$> ask + modified <- compilerResourceModified <$> ask + progName <- liftIO getProgName report logger $ "Checking cache: " ++ if modified then "modified" else "OK" if modified - then do v <- unCompilerM $ j $ fromIdentifier identifier + then do v <- unCompilerM $ j () liftIO $ Store.set store [name, show identifier] v return v else do v <- liftIO $ Store.get store [name, show identifier] diff --git a/src/Hakyll/Core/Compiler/Internal.hs b/src/Hakyll/Core/Compiler/Internal.hs index 8ed822d..a8c0989 100644 --- a/src/Hakyll/Core/Compiler/Internal.hs +++ b/src/Hakyll/Core/Compiler/Internal.hs @@ -26,7 +26,7 @@ import Control.Category (Category, (.), id) import Control.Arrow (Arrow, ArrowChoice, arr, first, left) import Hakyll.Core.Identifier -import Hakyll.Core.Resource.Provider +import Hakyll.Core.ResourceProvider import Hakyll.Core.Store import Hakyll.Core.Routes import Hakyll.Core.Logger diff --git a/src/Hakyll/Core/Identifier.hs b/src/Hakyll/Core/Identifier.hs index 90f0eea..d7bb8c6 100644 --- a/src/Hakyll/Core/Identifier.hs +++ b/src/Hakyll/Core/Identifier.hs @@ -34,6 +34,7 @@ module Hakyll.Core.Identifier ( Identifier (..) , castIdentifier , parseIdentifier + , fromFilePath , toFilePath , setGroup ) where @@ -77,16 +78,24 @@ castIdentifier :: Identifier a -> Identifier b castIdentifier (Identifier g p) = Identifier g p {-# INLINE castIdentifier #-} + +-------------------------------------------------------------------------------- -- | Parse an identifier from a string --- parseIdentifier :: String -> Identifier a -parseIdentifier = Identifier Nothing - . intercalate "/" . filter (not . null) . split' +parseIdentifier = Identifier Nothing . + intercalate "/" . filter (not . null) . split' where split' = map dropTrailingPathSeparator . splitPath + +-------------------------------------------------------------------------------- +-- | Create an identifier from a filepath +fromFilePath :: FilePath -> Identifier a +fromFilePath = parseIdentifier + + +-------------------------------------------------------------------------------- -- | Convert an identifier to a relative 'FilePath' --- toFilePath :: Identifier a -> FilePath toFilePath = identifierPath diff --git a/src/Hakyll/Core/Metadata.hs b/src/Hakyll/Core/Metadata.hs new file mode 100644 index 0000000..79922e1 --- /dev/null +++ b/src/Hakyll/Core/Metadata.hs @@ -0,0 +1,24 @@ +-------------------------------------------------------------------------------- +module Hakyll.Core.Metadata + ( Metadata + , MonadMetadata (..) + ) where + + +-------------------------------------------------------------------------------- +import Data.Map (Map) + + +-------------------------------------------------------------------------------- +import Hakyll.Core.Identifier + + +-------------------------------------------------------------------------------- +type Metadata = Map String String + + +-------------------------------------------------------------------------------- +class MonadMetadata m where + identifierMetadata :: Identifier a -> m Metadata + -- allMetadata :: m [(Resource, Metadata)] + -- patternMetadata :: Pattern a -> m [(Resource, Metadata)] diff --git a/src/Hakyll/Core/Resource.hs b/src/Hakyll/Core/Resource.hs deleted file mode 100644 index 0a43fc2..0000000 --- a/src/Hakyll/Core/Resource.hs +++ /dev/null @@ -1,51 +0,0 @@ --------------------------------------------------------------------------------- --- | Module exporting the simple 'Resource' type -module Hakyll.Core.Resource - ( -- * Constructing and deconstructing resources - Resource - , resource - , unResource - - -- * Conversions to and from identifiers - , fromIdentifier - , toIdentifier - - -- * TODO: Move me - , Metadata - ) where - - --------------------------------------------------------------------------------- -import Data.Map (Map) - - --------------------------------------------------------------------------------- -import Hakyll.Core.Identifier - - --------------------------------------------------------------------------------- --- | A resource -newtype Resource = Resource {unResource :: FilePath} - deriving (Eq, Show, Ord) - - --------------------------------------------------------------------------------- --- | Smart constructor to ensure we have @/@ as path separator -resource :: FilePath -> Resource -resource = fromIdentifier . parseIdentifier - - --------------------------------------------------------------------------------- --- | Find the resource for an identifier -fromIdentifier :: Identifier a -> Resource -fromIdentifier = Resource . toFilePath - - --------------------------------------------------------------------------------- --- | Convert a resource to an identifier -toIdentifier :: Resource -> Identifier a -toIdentifier = parseIdentifier . unResource - - --------------------------------------------------------------------------------- -type Metadata = Map String String diff --git a/src/Hakyll/Core/Resource/Metadata.hs b/src/Hakyll/Core/Resource/Metadata.hs deleted file mode 100644 index 44b0721..0000000 --- a/src/Hakyll/Core/Resource/Metadata.hs +++ /dev/null @@ -1,118 +0,0 @@ --------------------------------------------------------------------------------- --- | Internal module to parse metadata -module Hakyll.Core.Resource.Metadata - ( loadMetadata - ) where - - --------------------------------------------------------------------------------- -import Control.Applicative ((<$>), (<*), (<*>)) -import Control.Arrow (second) -import qualified Data.ByteString.Char8 as BC -import qualified Data.Map as M -import System.IO as IO -import Text.Parsec (()) -import qualified Text.Parsec as P -import Text.Parsec.String (Parser) - - --------------------------------------------------------------------------------- -import Hakyll.Core.Resource -import Hakyll.Core.Resource.Provider.Internal -import Hakyll.Core.Util.String - - --------------------------------------------------------------------------------- -loadMetadata :: ResourceProvider -> Resource -> IO (Metadata, Maybe String) -loadMetadata rp r = do - hasHeader <- probablyHasMetadataHeader fp - (md, body) <- if hasHeader - then second Just <$> loadMetadataHeader fp - else return (M.empty, Nothing) - - emd <- if resourceExists rp mr then loadMetadataFile mfp else return M.empty - - return (M.union md emd, body) - where - fp = unResource r - mr = resourceMetadataResource r - mfp = unResource mr - - --------------------------------------------------------------------------------- -loadMetadataHeader :: FilePath -> IO (Metadata, String) -loadMetadataHeader fp = do - contents <- readFile fp - case P.parse page fp contents of - Left err -> error (show err) - Right (md, b) -> return (M.fromList md, b) - - --------------------------------------------------------------------------------- -loadMetadataFile :: FilePath -> IO Metadata -loadMetadataFile fp = do - contents <- readFile fp - case P.parse metadata fp contents of - Left err -> error (show err) - Right md -> return $ M.fromList md - - --------------------------------------------------------------------------------- --- | Check if a file "probably" has a metadata header. The main goal of this is --- to exclude binary files (which are unlikely to start with "---"). -probablyHasMetadataHeader :: FilePath -> IO Bool -probablyHasMetadataHeader fp = do - handle <- IO.openFile fp IO.ReadMode - bs <- BC.hGet handle 1024 - IO.hClose handle - return $ isMetadataHeader bs - where - isMetadataHeader bs = - let pre = BC.takeWhile (\x -> x /= '\n' && x /= '\r') bs - in BC.length pre >= 3 && BC.all (== '-') pre - - --------------------------------------------------------------------------------- --- | Space or tab, no newline -inlineSpace :: Parser Char -inlineSpace = P.oneOf ['\t', ' '] "space" - - --------------------------------------------------------------------------------- --- | Parse a single metadata field -metadataField :: Parser (String, String) -metadataField = do - key <- P.manyTill P.alphaNum $ P.char ':' - P.skipMany1 inlineSpace "space followed by metadata for: " ++ key - value <- P.manyTill P.anyChar P.newline - trailing' <- P.many trailing - return (key, trim $ value ++ concat trailing') - where - trailing = (++) <$> P.many1 inlineSpace <*> P.manyTill P.anyChar P.newline - - --------------------------------------------------------------------------------- --- | Parse a metadata block -metadata :: Parser [(String, String)] -metadata = P.many metadataField - - --------------------------------------------------------------------------------- --- | Parse a metadata block, including delimiters and trailing newlines -metadataBlock :: Parser [(String, String)] -metadataBlock = do - open <- P.many1 (P.char '-') <* P.many inlineSpace <* P.newline - metadata' <- metadata - _ <- P.choice $ map (P.string . replicate (length open)) ['-', '.'] - P.skipMany inlineSpace - P.skipMany1 P.newline - return metadata' - - --------------------------------------------------------------------------------- --- | Parse a page consisting of a metadata header and a body -page :: Parser ([(String, String)], String) -page = do - metadata' <- P.option [] metadataBlock - body <- P.many P.anyChar - return (metadata', body) diff --git a/src/Hakyll/Core/Resource/MetadataCache.hs b/src/Hakyll/Core/Resource/MetadataCache.hs deleted file mode 100644 index b459674..0000000 --- a/src/Hakyll/Core/Resource/MetadataCache.hs +++ /dev/null @@ -1,61 +0,0 @@ --------------------------------------------------------------------------------- -module Hakyll.Core.Resource.MetadataCache - ( resourceMetadata - , resourceBody - , resourceInvalidateMetadataCache - ) where - - --------------------------------------------------------------------------------- -import Hakyll.Core.Resource -import Hakyll.Core.Resource.Metadata -import Hakyll.Core.Resource.Provider.Internal -import qualified Hakyll.Core.Store as Store - - --------------------------------------------------------------------------------- -resourceMetadata :: ResourceProvider -> Resource -> IO Metadata -resourceMetadata rp r = do - load rp r - Store.Found md <- Store.get (resourceStore rp) - [name, unResource r, "metadata"] - return md - - --------------------------------------------------------------------------------- -resourceBody :: ResourceProvider -> Resource -> IO String -resourceBody rp r = do - load rp r - Store.Found bd <- Store.get (resourceStore rp) - [name, unResource r, "body"] - maybe (resourceString r) return bd - - --------------------------------------------------------------------------------- -resourceInvalidateMetadataCache :: ResourceProvider -> Resource -> IO () -resourceInvalidateMetadataCache rp r = do - Store.delete (resourceStore rp) [name, unResource r, "metadata"] - Store.delete (resourceStore rp) [name, unResource r, "body"] - - --------------------------------------------------------------------------------- -load :: ResourceProvider -> Resource -> IO () -load rp r = do - mmd <- Store.get store mdk :: IO (Store.Result Metadata) - case mmd of - -- Already loaded - Store.Found _ -> return () - -- Not yet loaded - _ -> do - (metadata, body) <- loadMetadata rp r - Store.set store mdk metadata - Store.set store bk body - where - store = resourceStore rp - mdk = [name, unResource r, "metadata"] - bk = [name, unResource r, "body"] - - --------------------------------------------------------------------------------- -name :: String -name = "Hakyll.Core.Resource.Provider.MetadataCache" diff --git a/src/Hakyll/Core/Resource/Modified.hs b/src/Hakyll/Core/Resource/Modified.hs deleted file mode 100644 index 8492108..0000000 --- a/src/Hakyll/Core/Resource/Modified.hs +++ /dev/null @@ -1,82 +0,0 @@ --------------------------------------------------------------------------------- -module Hakyll.Core.Resource.Modified - ( resourceModified - , resourceModificationTime - ) where - - --------------------------------------------------------------------------------- -import Control.Applicative ((<$>), (<*>)) -import Control.Monad (when) -import qualified Crypto.Hash.MD5 as MD5 -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL -import Data.IORef -import qualified Data.Map as M -import Data.Time (UTCTime) -import System.Directory (getModificationTime) - - --------------------------------------------------------------------------------- -import Hakyll.Core.Resource -import Hakyll.Core.Resource.MetadataCache -import Hakyll.Core.Resource.Provider.Internal -import Hakyll.Core.Store (Store) -import qualified Hakyll.Core.Store as Store - - --------------------------------------------------------------------------------- --- | A resource is modified if it or its metadata has changed -resourceModified :: ResourceProvider -> Resource -> IO Bool -resourceModified rp r - | not exists = return False - | otherwise = do - cache <- readIORef cacheRef - case M.lookup r cache of - Just m -> return m - Nothing -> do - -- Check if the actual file was modified, and do a recursive - -- call to check if the metadata file was modified - m <- (||) - <$> fileDigestModified store (unResource r) - <*> resourceModified rp (resourceMetadataResource r) - modifyIORef cacheRef (M.insert r m) - - -- Important! (But ugly) - when m $ resourceInvalidateMetadataCache rp r - - return m - where - exists = resourceExists rp r - store = resourceStore rp - cacheRef = resourceModifiedCache rp - - --------------------------------------------------------------------------------- --- | Utility: Check if a the digest of a file was modified -fileDigestModified :: Store -> FilePath -> IO Bool -fileDigestModified store fp = do - -- Get the latest seen digest from the store, and calculate the current - -- digest for the - lastDigest <- Store.get store key - newDigest <- fileDigest fp - if Store.Found newDigest == lastDigest - -- All is fine, not modified - then return False - -- Resource modified; store new digest - else do - Store.set store key newDigest - return True - where - key = ["Hakyll.Core.Resource.Provider.fileModified", fp] - - --------------------------------------------------------------------------------- --- | Utility: Retrieve a digest for a given file -fileDigest :: FilePath -> IO B.ByteString -fileDigest = fmap MD5.hashlazy . BL.readFile - - --------------------------------------------------------------------------------- -resourceModificationTime :: Resource -> IO UTCTime -resourceModificationTime = getModificationTime . unResource diff --git a/src/Hakyll/Core/Resource/Pattern.hs b/src/Hakyll/Core/Resource/Pattern.hs deleted file mode 100644 index c2f1132..0000000 --- a/src/Hakyll/Core/Resource/Pattern.hs +++ /dev/null @@ -1,160 +0,0 @@ --------------------------------------------------------------------------------- --- | Module providing pattern matching and capturing on file names. --- 'Pattern's come in two kinds: --- --- * Simple glob patterns, like @foo\/*@; --- --- * Custom, arbitrary predicates of the type @Identifier -> Bool@. --- --- They both have advantages and disadvantages. By default, globs are used, --- unless you construct your 'Pattern' using the 'predicate' function. --- --- A very simple pattern could be, for example, @foo\/bar@. This pattern will --- only match the exact @foo\/bar@ identifier. --- --- To match more than one identifier, there are different captures that one can --- use: --- --- * @*@: matches at most one element of an identifier; --- --- * @**@: matches one or more elements of an identifier. --- --- Some examples: --- --- * @foo\/*@ will match @foo\/bar@ and @foo\/foo@, but not @foo\/bar\/qux@; --- --- * @**@ will match any identifier; --- --- * @foo\/**@ will match @foo\/bar@ and @foo\/bar\/qux@, but not @bar\/foo@; --- --- * @foo\/*.html@ will match all HTML files in the @foo\/@ directory. --- --- The 'capture' function allows the user to get access to the elements captured --- by the capture elements in the pattern. --- --- Like an 'Identifier', a 'Pattern' also has a type parameter. This is simply --- an extra layer of safety, and can be discarded using the 'castPattern' --- function. -module Hakyll.Core.Resource.Pattern - ( Pattern - , parsePattern - , capture - ) where - - --------------------------------------------------------------------------------- -import Control.Arrow ((&&&), (>>>)) -import Control.Monad (msum) -import Data.List (inits, isPrefixOf, tails) -import GHC.Exts (IsString, fromString) - - --------------------------------------------------------------------------------- -import Hakyll.Core.Resource - - --------------------------------------------------------------------------------- --- | One base element of a pattern -data GlobComponent - = Capture - | CaptureMany - | Literal String - deriving (Eq, Show) - - --------------------------------------------------------------------------------- --- | Type that allows matching on identifiers -newtype Pattern = Pattern [GlobComponent] - deriving (Show) - - --------------------------------------------------------------------------------- -instance IsString Pattern where - fromString = parsePattern - - --------------------------------------------------------------------------------- --- | Parse a pattern from a string -parsePattern :: String -> Pattern -parsePattern = Pattern . parse - where - parse str = - let (chunk, rest) = break (`elem` "\\*") str - in case rest of - ('\\' : x : xs) -> Literal (chunk ++ [x]) : parse xs - ('*' : '*' : xs) -> Literal chunk : CaptureMany : parse xs - ('*' : xs) -> Literal chunk : Capture : parse xs - xs -> Literal chunk : Literal xs : [] - - --------------------------------------------------------------------------------- --- | Split a list at every possible point, generate a list of (init, tail) --- cases. The result is sorted with inits decreasing in length. -splits :: [a] -> [([a], [a])] -splits = inits &&& tails >>> uncurry zip >>> reverse - - --------------------------------------------------------------------------------- --- | Match a glob against a pattern, generating a list of captures -capture :: Pattern -> Resource -> Maybe [String] -capture (Pattern p) rs = capture' p (unResource rs) - - --------------------------------------------------------------------------------- --- | Internal verion of 'capture' -capture' :: [GlobComponent] -> String -> Maybe [String] -capture' [] [] = Just [] -- An empty match -capture' [] _ = Nothing -- No match -capture' (Literal l : ms) str - -- Match the literal against the string - | l `isPrefixOf` str = capture' ms $ drop (length l) str - | otherwise = Nothing -capture' (Capture : ms) str = - -- Match until the next / - let (chunk, rest) = break (== '/') str - in msum $ [ fmap (i :) (capture' ms (t ++ rest)) | (i, t) <- splits chunk ] -capture' (CaptureMany : ms) str = - -- Match everything - msum $ [ fmap (i :) (capture' ms t) | (i, t) <- splits str ] - - --------------------------------------------------------------------------------- --- | Create an identifier from a pattern by filling in the captures with a given --- string --- --- Example: --- --- > fromCapture (parsePattern "tags/*") "foo" --- --- Result: --- --- > "tags/foo" -{- -fromCapture :: Pattern -> String -> Identifier -fromCapture pattern = fromCaptures pattern . repeat --} - - --------------------------------------------------------------------------------- --- | Create an identifier from a pattern by filling in the captures with the --- given list of strings --- -{- -fromCaptures :: Pattern -> [String] -> String -fromCaptures (Pattern p) = fromCaptures' p --} - - --------------------------------------------------------------------------------- --- | Internally used version of 'fromCaptures' -{- -fromCaptures' :: [GlobComponent] -> [String] -> String -fromCaptures' [] _ = mempty -fromCaptures' (m : ms) [] = case m of - Literal l -> l `mappend` fromCaptures' ms [] - _ -> error $ "Hakyll.Core.Identifier.Pattern.fromCaptures': " - ++ "identifier list exhausted" -fromCaptures' (m : ms) ids@(i : is) = case m of - Literal l -> l `mappend` fromCaptures' ms ids - _ -> i `mappend` fromCaptures' ms is --} diff --git a/src/Hakyll/Core/Resource/Provider.hs b/src/Hakyll/Core/Resource/Provider.hs deleted file mode 100644 index 8f4c83f..0000000 --- a/src/Hakyll/Core/Resource/Provider.hs +++ /dev/null @@ -1,45 +0,0 @@ --------------------------------------------------------------------------------- --- | This module provides an wrapper API around the file system which does some --- caching. -module Hakyll.Core.Resource.Provider - ( -- * Constructing resource providers - ResourceProvider - , newResourceProvider - - -- * Querying resource properties - , resourceList - , resourceExists - , resourceModified - , resourceModificationTime - - -- * Access to raw resource content - , resourceString - , resourceLBS - - -- * Access to metadata and body content - , resourceMetadata - , resourceBody - ) where - - --------------------------------------------------------------------------------- -import Hakyll.Core.Resource -import qualified Hakyll.Core.Resource.MetadataCache as Internal -import Hakyll.Core.Resource.Modified -import Hakyll.Core.Resource.Provider.Internal - - --------------------------------------------------------------------------------- --- | Wrapper to ensure metadata cache is invalidated if necessary -resourceMetadata :: ResourceProvider -> Resource -> IO Metadata -resourceMetadata rp r = do - _ <- resourceModified rp r - Internal.resourceMetadata rp r - - --------------------------------------------------------------------------------- --- | Wrapper to ensure metadata cache is invalidated if necessary -resourceBody :: ResourceProvider -> Resource -> IO String -resourceBody rp r = do - _ <- resourceModified rp r - Internal.resourceBody rp r diff --git a/src/Hakyll/Core/Resource/Provider/Internal.hs b/src/Hakyll/Core/Resource/Provider/Internal.hs deleted file mode 100644 index fb93fcc..0000000 --- a/src/Hakyll/Core/Resource/Provider/Internal.hs +++ /dev/null @@ -1,84 +0,0 @@ --------------------------------------------------------------------------------- -module Hakyll.Core.Resource.Provider.Internal - ( ResourceProvider (..) - , newResourceProvider - - , resourceList - , resourceExists - , resourceMetadataResource - - , resourceString - , resourceLBS - ) where - - --------------------------------------------------------------------------------- -import Control.Applicative ((<$>)) -import qualified Data.ByteString.Lazy as BL -import Data.IORef -import Data.Map (Map) -import qualified Data.Map as M -import Data.Set (Set) -import qualified Data.Set as S -import System.FilePath (addExtension) - - --------------------------------------------------------------------------------- -import Hakyll.Core.Resource -import Hakyll.Core.Store -import Hakyll.Core.Util.File - - --------------------------------------------------------------------------------- --- | Responsible for retrieving and listing resources -data ResourceProvider = ResourceProvider - { -- | A list of all files found - resourceSet :: Set Resource - , -- | Cache keeping track of modified files - resourceModifiedCache :: IORef (Map Resource Bool) - , -- | Underlying persistent store for caching - resourceStore :: Store - } - - --------------------------------------------------------------------------------- --- | Create a resource provider -newResourceProvider :: Store -- ^ Store to use - -> (FilePath -> Bool) -- ^ Should we ignore this file? - -> FilePath -- ^ Search directory - -> IO ResourceProvider -- ^ Resulting provider -newResourceProvider store ignore directory = do - list <- map resource . filter (not . ignore) <$> - getRecursiveContents False directory - cache <- newIORef M.empty - return $ ResourceProvider (S.fromList list) cache store - - --------------------------------------------------------------------------------- -resourceList :: ResourceProvider -> [Resource] -resourceList = S.toList . resourceSet - - --------------------------------------------------------------------------------- --- | Check if a given resiyrce exists -resourceExists :: ResourceProvider -> Resource -> Bool -resourceExists provider = (`S.member` resourceSet provider) - - --------------------------------------------------------------------------------- --- | Each resource may have an associated metadata resource (with a @.metadata@ --- filename) -resourceMetadataResource :: Resource -> Resource -resourceMetadataResource = resource . flip addExtension "metadata" . unResource - - --------------------------------------------------------------------------------- --- | Get the raw body of a resource as string -resourceString :: Resource -> IO String -resourceString = readFile . unResource - - --------------------------------------------------------------------------------- --- | Get the raw body of a resource of a lazy bytestring -resourceLBS :: Resource -> IO BL.ByteString -resourceLBS = BL.readFile . unResource diff --git a/src/Hakyll/Core/ResourceProvider.hs b/src/Hakyll/Core/ResourceProvider.hs new file mode 100644 index 0000000..f18d462 --- /dev/null +++ b/src/Hakyll/Core/ResourceProvider.hs @@ -0,0 +1,46 @@ +-------------------------------------------------------------------------------- +-- | This module provides an wrapper API around the file system which does some +-- caching. +module Hakyll.Core.ResourceProvider + ( -- * Constructing resource providers + ResourceProvider + , newResourceProvider + + -- * Querying resource properties + , resourceList + , resourceExists + , resourceModified + , resourceModificationTime + + -- * Access to raw resource content + , resourceString + , resourceLBS + + -- * Access to metadata and body content + , resourceMetadata + , resourceBody + ) where + + +-------------------------------------------------------------------------------- +import Hakyll.Core.Identifier +import Hakyll.Core.Metadata +import Hakyll.Core.ResourceProvider.Internal +import qualified Hakyll.Core.ResourceProvider.MetadataCache as Internal +import Hakyll.Core.ResourceProvider.Modified + + +-------------------------------------------------------------------------------- +-- | Wrapper to ensure metadata cache is invalidated if necessary +resourceMetadata :: ResourceProvider -> Identifier a -> IO Metadata +resourceMetadata rp r = do + _ <- resourceModified rp r + Internal.resourceMetadata rp r + + +-------------------------------------------------------------------------------- +-- | Wrapper to ensure metadata cache is invalidated if necessary +resourceBody :: ResourceProvider -> Identifier a -> IO String +resourceBody rp r = do + _ <- resourceModified rp r + Internal.resourceBody rp r diff --git a/src/Hakyll/Core/ResourceProvider/Internal.hs b/src/Hakyll/Core/ResourceProvider/Internal.hs new file mode 100644 index 0000000..1f8f776 --- /dev/null +++ b/src/Hakyll/Core/ResourceProvider/Internal.hs @@ -0,0 +1,86 @@ +-------------------------------------------------------------------------------- +module Hakyll.Core.ResourceProvider.Internal + ( ResourceProvider (..) + , newResourceProvider + + , resourceList + , resourceExists + , resourceMetadataResource + + , resourceString + , resourceLBS + ) where + + +-------------------------------------------------------------------------------- +import Control.Applicative ((<$>)) +import qualified Data.ByteString.Lazy as BL +import Data.IORef +import Data.Map (Map) +import qualified Data.Map as M +import Data.Set (Set) +import qualified Data.Set as S +import System.FilePath (addExtension) + + +-------------------------------------------------------------------------------- +import Hakyll.Core.Store +import Hakyll.Core.Util.File +import Hakyll.Core.Identifier + + +-------------------------------------------------------------------------------- +-- | Responsible for retrieving and listing resources +data ResourceProvider = ResourceProvider + { -- | A list of all files found + resourceSet :: Set (Identifier ()) + , -- | Cache keeping track of modified files + resourceModifiedCache :: IORef (Map (Identifier ()) Bool) + , -- | Underlying persistent store for caching + resourceStore :: Store + } + + +-------------------------------------------------------------------------------- +-- | Create a resource provider +newResourceProvider :: Store -- ^ Store to use + -> (FilePath -> Bool) -- ^ Should we ignore this file? + -> FilePath -- ^ Search directory + -> IO ResourceProvider -- ^ Resulting provider +newResourceProvider store ignore directory = do + list <- map parseIdentifier . filter (not . ignore) <$> + getRecursiveContents False directory + cache <- newIORef M.empty + return $ ResourceProvider (S.fromList list) cache store + + +-------------------------------------------------------------------------------- +resourceList :: ResourceProvider -> [Identifier ()] +resourceList = S.toList . resourceSet + + +-------------------------------------------------------------------------------- +-- | Check if a given resiyrce exists +resourceExists :: ResourceProvider -> Identifier a -> Bool +resourceExists provider = + (`S.member` resourceSet provider) . setGroup Nothing . castIdentifier + + +-------------------------------------------------------------------------------- +-- | Each resource may have an associated metadata resource (with a @.metadata@ +-- filename) +resourceMetadataResource :: Identifier a -> Identifier () +resourceMetadataResource = + parseIdentifier . flip addExtension "metadata" . toFilePath + + +-------------------------------------------------------------------------------- +-- | Get the raw body of a resource as string +resourceString :: Identifier a -> IO String +resourceString = readFile . toFilePath + + +-------------------------------------------------------------------------------- +-- | Get the raw body of a resource of a lazy bytestring +resourceLBS :: Identifier a -> IO BL.ByteString +resourceLBS = BL.readFile . toFilePath diff --git a/src/Hakyll/Core/ResourceProvider/Metadata.hs b/src/Hakyll/Core/ResourceProvider/Metadata.hs new file mode 100644 index 0000000..e297f2c --- /dev/null +++ b/src/Hakyll/Core/ResourceProvider/Metadata.hs @@ -0,0 +1,119 @@ +-------------------------------------------------------------------------------- +-- | Internal module to parse metadata +module Hakyll.Core.ResourceProvider.Metadata + ( loadMetadata + ) where + + +-------------------------------------------------------------------------------- +import Control.Applicative ((<$>), (<*), (<*>)) +import Control.Arrow (second) +import qualified Data.ByteString.Char8 as BC +import qualified Data.Map as M +import System.IO as IO +import Text.Parsec (()) +import qualified Text.Parsec as P +import Text.Parsec.String (Parser) + + +-------------------------------------------------------------------------------- +import Hakyll.Core.Identifier +import Hakyll.Core.Metadata +import Hakyll.Core.ResourceProvider.Internal +import Hakyll.Core.Util.String + + +-------------------------------------------------------------------------------- +loadMetadata :: ResourceProvider -> Identifier a -> IO (Metadata, Maybe String) +loadMetadata rp identifier = do + hasHeader <- probablyHasMetadataHeader fp + (md, body) <- if hasHeader + then second Just <$> loadMetadataHeader fp + else return (M.empty, Nothing) + + emd <- if resourceExists rp mi then loadMetadataFile mfp else return M.empty + + return (M.union md emd, body) + where + fp = toFilePath identifier + mi = resourceMetadataResource identifier + mfp = toFilePath mi + + +-------------------------------------------------------------------------------- +loadMetadataHeader :: FilePath -> IO (Metadata, String) +loadMetadataHeader fp = do + contents <- readFile fp + case P.parse page fp contents of + Left err -> error (show err) + Right (md, b) -> return (M.fromList md, b) + + +-------------------------------------------------------------------------------- +loadMetadataFile :: FilePath -> IO Metadata +loadMetadataFile fp = do + contents <- readFile fp + case P.parse metadata fp contents of + Left err -> error (show err) + Right md -> return $ M.fromList md + + +-------------------------------------------------------------------------------- +-- | Check if a file "probably" has a metadata header. The main goal of this is +-- to exclude binary files (which are unlikely to start with "---"). +probablyHasMetadataHeader :: FilePath -> IO Bool +probablyHasMetadataHeader fp = do + handle <- IO.openFile fp IO.ReadMode + bs <- BC.hGet handle 1024 + IO.hClose handle + return $ isMetadataHeader bs + where + isMetadataHeader bs = + let pre = BC.takeWhile (\x -> x /= '\n' && x /= '\r') bs + in BC.length pre >= 3 && BC.all (== '-') pre + + +-------------------------------------------------------------------------------- +-- | Space or tab, no newline +inlineSpace :: Parser Char +inlineSpace = P.oneOf ['\t', ' '] "space" + + +-------------------------------------------------------------------------------- +-- | Parse a single metadata field +metadataField :: Parser (String, String) +metadataField = do + key <- P.manyTill P.alphaNum $ P.char ':' + P.skipMany1 inlineSpace "space followed by metadata for: " ++ key + value <- P.manyTill P.anyChar P.newline + trailing' <- P.many trailing + return (key, trim $ value ++ concat trailing') + where + trailing = (++) <$> P.many1 inlineSpace <*> P.manyTill P.anyChar P.newline + + +-------------------------------------------------------------------------------- +-- | Parse a metadata block +metadata :: Parser [(String, String)] +metadata = P.many metadataField + + +-------------------------------------------------------------------------------- +-- | Parse a metadata block, including delimiters and trailing newlines +metadataBlock :: Parser [(String, String)] +metadataBlock = do + open <- P.many1 (P.char '-') <* P.many inlineSpace <* P.newline + metadata' <- metadata + _ <- P.choice $ map (P.string . replicate (length open)) ['-', '.'] + P.skipMany inlineSpace + P.skipMany1 P.newline + return metadata' + + +-------------------------------------------------------------------------------- +-- | Parse a page consisting of a metadata header and a body +page :: Parser ([(String, String)], String) +page = do + metadata' <- P.option [] metadataBlock + body <- P.many P.anyChar + return (metadata', body) diff --git a/src/Hakyll/Core/ResourceProvider/MetadataCache.hs b/src/Hakyll/Core/ResourceProvider/MetadataCache.hs new file mode 100644 index 0000000..85062a0 --- /dev/null +++ b/src/Hakyll/Core/ResourceProvider/MetadataCache.hs @@ -0,0 +1,62 @@ +-------------------------------------------------------------------------------- +module Hakyll.Core.ResourceProvider.MetadataCache + ( resourceMetadata + , resourceBody + , resourceInvalidateMetadataCache + ) where + + +-------------------------------------------------------------------------------- +import Hakyll.Core.Identifier +import Hakyll.Core.Metadata +import Hakyll.Core.ResourceProvider.Internal +import Hakyll.Core.ResourceProvider.Metadata +import qualified Hakyll.Core.Store as Store + + +-------------------------------------------------------------------------------- +resourceMetadata :: ResourceProvider -> Identifier a -> IO Metadata +resourceMetadata rp r = do + load rp r + Store.Found md <- Store.get (resourceStore rp) + [name, toFilePath r, "metadata"] + return md + + +-------------------------------------------------------------------------------- +resourceBody :: ResourceProvider -> Identifier a -> IO String +resourceBody rp r = do + load rp r + Store.Found bd <- Store.get (resourceStore rp) + [name, toFilePath r, "body"] + maybe (resourceString r) return bd + + +-------------------------------------------------------------------------------- +resourceInvalidateMetadataCache :: ResourceProvider -> Identifier a -> IO () +resourceInvalidateMetadataCache rp r = do + Store.delete (resourceStore rp) [name, toFilePath r, "metadata"] + Store.delete (resourceStore rp) [name, toFilePath r, "body"] + + +-------------------------------------------------------------------------------- +load :: ResourceProvider -> Identifier a -> IO () +load rp r = do + mmd <- Store.get store mdk :: IO (Store.Result Metadata) + case mmd of + -- Already loaded + Store.Found _ -> return () + -- Not yet loaded + _ -> do + (metadata, body) <- loadMetadata rp r + Store.set store mdk metadata + Store.set store bk body + where + store = resourceStore rp + mdk = [name, toFilePath r, "metadata"] + bk = [name, toFilePath r, "body"] + + +-------------------------------------------------------------------------------- +name :: String +name = "Hakyll.Core.Resource.Provider.MetadataCache" diff --git a/src/Hakyll/Core/ResourceProvider/Modified.hs b/src/Hakyll/Core/ResourceProvider/Modified.hs new file mode 100644 index 0000000..837bc8c --- /dev/null +++ b/src/Hakyll/Core/ResourceProvider/Modified.hs @@ -0,0 +1,83 @@ +-------------------------------------------------------------------------------- +module Hakyll.Core.ResourceProvider.Modified + ( resourceModified + , resourceModificationTime + ) where + + +-------------------------------------------------------------------------------- +import Control.Applicative ((<$>), (<*>)) +import Control.Monad (when) +import qualified Crypto.Hash.MD5 as MD5 +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as BL +import Data.IORef +import qualified Data.Map as M +import Data.Time (UTCTime) +import System.Directory (getModificationTime) + + +-------------------------------------------------------------------------------- +import Hakyll.Core.Identifier +import Hakyll.Core.ResourceProvider.Internal +import Hakyll.Core.ResourceProvider.MetadataCache +import Hakyll.Core.Store (Store) +import qualified Hakyll.Core.Store as Store + + +-------------------------------------------------------------------------------- +-- | A resource is modified if it or its metadata has changed +resourceModified :: ResourceProvider -> Identifier a -> IO Bool +resourceModified rp r + | not exists = return False + | otherwise = do + cache <- readIORef cacheRef + case M.lookup normalized cache of + Just m -> return m + Nothing -> do + -- Check if the actual file was modified, and do a recursive + -- call to check if the metadata file was modified + m <- (||) + <$> fileDigestModified store (toFilePath r) + <*> resourceModified rp (resourceMetadataResource r) + modifyIORef cacheRef (M.insert normalized m) + + -- Important! (But ugly) + when m $ resourceInvalidateMetadataCache rp r + + return m + where + normalized = castIdentifier $ setGroup Nothing r + exists = resourceExists rp r + store = resourceStore rp + cacheRef = resourceModifiedCache rp + + +-------------------------------------------------------------------------------- +-- | Utility: Check if a the digest of a file was modified +fileDigestModified :: Store -> FilePath -> IO Bool +fileDigestModified store fp = do + -- Get the latest seen digest from the store, and calculate the current + -- digest for the + lastDigest <- Store.get store key + newDigest <- fileDigest fp + if Store.Found newDigest == lastDigest + -- All is fine, not modified + then return False + -- Resource modified; store new digest + else do + Store.set store key newDigest + return True + where + key = ["Hakyll.Core.Resource.Provider.fileModified", fp] + + +-------------------------------------------------------------------------------- +-- | Utility: Retrieve a digest for a given file +fileDigest :: FilePath -> IO B.ByteString +fileDigest = fmap MD5.hashlazy . BL.readFile + + +-------------------------------------------------------------------------------- +resourceModificationTime :: Identifier a -> IO UTCTime +resourceModificationTime = getModificationTime . toFilePath diff --git a/src/Hakyll/Core/Rules.hs b/src/Hakyll/Core/Rules.hs index ff68c56..5ac63bc 100644 --- a/src/Hakyll/Core/Rules.hs +++ b/src/Hakyll/Core/Rules.hs @@ -51,11 +51,9 @@ import Hakyll.Core.CompiledItem import Hakyll.Core.Compiler.Internal import Hakyll.Core.Identifier import Hakyll.Core.Identifier.Pattern -import Hakyll.Core.Resource -import Hakyll.Core.Resource.Provider +import Hakyll.Core.ResourceProvider import Hakyll.Core.Routes import Hakyll.Core.Rules.Internal -import Hakyll.Core.Util.Arrow import Hakyll.Core.Writable @@ -80,10 +78,10 @@ tellCompilers compilers = RulesM $ do -------------------------------------------------------------------------------- -- | Add resources -tellResources :: [Resource] +tellResources :: [Identifier a] -> Rules tellResources resources' = RulesM $ tell $ - RuleSet mempty mempty $ S.fromList resources' + RuleSet mempty mempty $ S.fromList $ map castIdentifier resources' -------------------------------------------------------------------------------- @@ -139,13 +137,12 @@ group g = RulesM . local setGroup' . unRulesM -- no resources match the current selection, nothing will happen. In this case, -- you might want to have a look at 'create'. compile :: (Binary a, Typeable a, Writable a) - => Compiler Resource a -> RulesM (Pattern a) + => Compiler () a -> RulesM (Pattern a) compile compiler = do ids <- resources - tellCompilers $ flip map ids $ \identifier -> - (identifier, constA (fromIdentifier identifier) >>> compiler) - tellResources $ map fromIdentifier ids - return $ list ids + tellCompilers [(castIdentifier id', compiler) | id' <- ids] + tellResources ids + return $ list $ map castIdentifier ids -------------------------------------------------------------------------------- @@ -182,14 +179,12 @@ route route' = RulesM $ do -------------------------------------------------------------------------------- -- | Get a list of resources matching the current pattern. This will also set -- the correct group to the identifiers. -resources :: RulesM [Identifier a] +resources :: RulesM [Identifier ()] resources = RulesM $ do - pattern <- rulesPattern <$> ask + pattern <- rulesPattern <$> ask provider <- rulesResourceProvider <$> ask - group' <- rulesGroup <$> ask - return $ filterMatches pattern $ map (toId group') $ resourceList provider - where - toId g = setGroup g . toIdentifier + g <- rulesGroup <$> ask + return $ filterMatches pattern $ map (setGroup g) $ resourceList provider -------------------------------------------------------------------------------- diff --git a/src/Hakyll/Core/Rules/Internal.hs b/src/Hakyll/Core/Rules/Internal.hs index 9d6a979..245d935 100644 --- a/src/Hakyll/Core/Rules/Internal.hs +++ b/src/Hakyll/Core/Rules/Internal.hs @@ -26,8 +26,7 @@ import Hakyll.Core.CompiledItem import Hakyll.Core.Compiler.Internal import Hakyll.Core.Identifier import Hakyll.Core.Identifier.Pattern -import Hakyll.Core.Resource -import Hakyll.Core.Resource.Provider +import Hakyll.Core.ResourceProvider import Hakyll.Core.Routes @@ -49,8 +48,8 @@ data RuleSet = RuleSet rulesRoutes :: Routes , -- | Compilation rules rulesCompilers :: [(Identifier (), Compiler () CompileRule)] - , -- | A list of the used resources - rulesResources :: Set Resource + , -- | A set of the actually used files + rulesResources :: Set (Identifier ()) } diff --git a/src/Hakyll/Core/Run.hs b/src/Hakyll/Core/Run.hs index 4842ea7..ff7acac 100644 --- a/src/Hakyll/Core/Run.hs +++ b/src/Hakyll/Core/Run.hs @@ -25,8 +25,7 @@ import Hakyll.Core.DependencyAnalyzer import Hakyll.Core.DirectedGraph import Hakyll.Core.Identifier import Hakyll.Core.Logger -import Hakyll.Core.Resource -import Hakyll.Core.Resource.Provider +import Hakyll.Core.ResourceProvider import Hakyll.Core.Routes import Hakyll.Core.Rules.Internal import Hakyll.Core.Store (Store) @@ -132,7 +131,7 @@ addNewCompilers newCompilers = Runtime $ do -- Check which items have been modified modified <- fmap S.fromList $ flip filterM (map fst newCompilers) $ - liftIO . resourceModified provider . fromIdentifier + liftIO . resourceModified provider let checkModified = if firstRun then const True else (`S.member` modified) -- Create a new analyzer and append it to the currect one @@ -183,7 +182,7 @@ build id' = Runtime $ do let compiler = compilers M.! id' -- Check if the resource was modified - isModified <- liftIO $ resourceModified provider $ fromIdentifier id' + isModified <- liftIO $ resourceModified provider id' -- Run the compiler result <- timed logger "Total compile time" $ liftIO $ diff --git a/src/Hakyll/Core/Writable/CopyFile.hs b/src/Hakyll/Core/Writable/CopyFile.hs index ab9c698..6cc08f2 100644 --- a/src/Hakyll/Core/Writable/CopyFile.hs +++ b/src/Hakyll/Core/Writable/CopyFile.hs @@ -1,29 +1,36 @@ +-------------------------------------------------------------------------------- -- | Exports simple compilers to just copy files --- {-# LANGUAGE GeneralizedNewtypeDeriving, DeriveDataTypeable #-} module Hakyll.Core.Writable.CopyFile ( CopyFile (..) , copyFileCompiler ) where + +-------------------------------------------------------------------------------- import Control.Arrow ((>>^)) import System.Directory (copyFile) - import Data.Typeable (Typeable) import Data.Binary (Binary) -import Hakyll.Core.Resource + +-------------------------------------------------------------------------------- import Hakyll.Core.Writable import Hakyll.Core.Compiler import Hakyll.Core.Identifier + +-------------------------------------------------------------------------------- -- | Newtype construct around 'FilePath' which will copy the file directly --- newtype CopyFile = CopyFile {unCopyFile :: FilePath} - deriving (Show, Eq, Ord, Binary, Typeable) + deriving (Show, Eq, Ord, Binary, Typeable) + +-------------------------------------------------------------------------------- instance Writable CopyFile where write dst (CopyFile src) = copyFile src dst -copyFileCompiler :: Compiler Resource CopyFile + +-------------------------------------------------------------------------------- +copyFileCompiler :: Compiler a CopyFile copyFileCompiler = getIdentifier >>^ CopyFile . toFilePath -- cgit v1.2.3