How to read all content of a file with Iteratee

Question

I have the next code in Haskell, and this is fine for to read a first line of a files, but I need to read the all content of a file in a directory (many files recursively). I am trying to change the firstLineE function, I do not understand How can I to change the line: EIO.enumFile 1024 filename $ joinI $ ((mapChunks B.pack) ><> EC.enumLinesBS) . do you have some documentation for this or can you help me with some examples?.

I am reviewing the the documentation, but Iteratee is very new for me:

http://www.mew.org/~kazu/proj/enumerator/

http://blog-mno2.csie.org/blog/2011/11/19/yet-another-guide-to-understand-iteratee-for-haskell-programmers/

import Control.Monad
import Control.Monad.IO.Class
import Control.Applicative
import System.Environment
import System.Directory
import System.FilePath
import qualified Data.List as L
import qualified Data.ByteString.Char8 as B
import qualified Data.Iteratee as I
import Data.Iteratee.Iteratee
import qualified Data.Iteratee.Char as EC
import qualified Data.Iteratee.IO.Fd as EIO
import qualified Data.Iteratee.ListLike as EL


getValidContents :: FilePath -> IO [String]
getValidContents path =
    filter (`notElem` [".", "..",".git", ".svn",".metadata",".idea",".project",".gitignore",".settings",".hsproject",".dist-scion",".dist-buildwrapper"])
    <$> getDirectoryContents path

isSearchableDir :: FilePath -> IO Bool
isSearchableDir dir = doesDirectoryExist dir
    -- (&&) <$> doesDirectoryExist dir
    --     <*> (searchable <$> getPermissions dir)


doesFileExistAndFilter :: FilePath -> IO Bool
doesFileExistAndFilter dir =
    (&&) <$> doesFileExist dir
         <*> return (snd (splitExtension dir) == ".java" || snd (splitExtension dir) == ".mora")


printI :: Iteratee [B.ByteString] IO ()
printI = do
    mx <- EL.tryHead
    case mx of
         Nothing -> return ()
         Just l -> do
             liftIO . B.putStrLn $ l
             printI


firstLineE :: Enumeratee [FilePath] [B.ByteString] IO ()
firstLineE = mapChunksM $ \filenames -> do
    forM filenames $ \filename -> do
        i <- EIO.enumFile 1024 filename $ joinI $ ((mapChunks B.pack) ><> EC.enumLinesBS) EL.head
        result <- run i
        return result

enumDir :: FilePath -> Enumerator [FilePath] IO b
enumDir dir iter = runIter iter idoneM onCont
    where
        onCont k Nothing = do
            (files, dirs) <- liftIO getFilesDirs
            if null dirs
                then return $ k (Chunk files)
                else walk dirs $ k (Chunk files)
        walk dirs = foldr1 (>>>) $ map enumDir dirs
        getFilesDirs = do
            cnts <- map (dir </>) <$> getValidContents dir
            (,) <$> filterM doesFileExist  cnts
                <*> filterM isSearchableDir cnts

allFirstLines :: FilePath -> IO ()
allFirstLines dir = do
    i' <- enumDir dir $ joinI $ firstLineE printI
    run i'


main = do
    dir:_ <- getArgs
    allFirstLines dir

Answer 1

I don't know how to do it with iteratees, but here is the pipes-based solution. Some advantages of this version of the solution are:

• It will never bring more than one bytestring chunk into memory at a time

• It streams the list of files so it won't choke on a directory with a large number of immediate children

• It recursively traverses the directory tree, like you asked

Some of this stuff will be in pipes utility libraries very soon:

import Control.Monad (when, unless)
import Control.Proxy
import Control.Proxy.Safe hiding (readFileS)
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as B8
import System.Directory (readable, getPermissions, doesDirectoryExist)
import System.FilePath ((</>), takeFileName)
import System.Posix (openDirStream, readDirStream, closeDirStream)
import System.IO (openFile, hClose, IOMode(ReadMode), hIsEOF)

contents
     :: (CheckP p)
     => FilePath -> () -> Producer (ExceptionP p) FilePath SafeIO ()
contents path () = do
     canRead <- tryIO $ fmap readable $ getPermissions path
     when canRead $ bracket id (openDirStream path) closeDirStream $ \dirp -> do
         let loop = do
                 file <- tryIO $ readDirStream dirp
                 case file of
                     [] -> return ()
                     _  -> do
                         respond (path </> file)
                         loop
         loop

contentsRecursive
     :: (CheckP p)
     => FilePath -> () -> Producer (ExceptionP p) FilePath SafeIO ()
contentsRecursive path () = loop path
   where
     loop path = do
         contents path () //> \newPath -> do
             respond newPath
             isDir <- tryIO $ doesDirectoryExist newPath
             let isChild = not $ takeFileName newPath `elem` [".", ".."]
             when (isDir && isChild) $ loop newPath

readFileS
    :: (CheckP p)
    => Int -> FilePath -> () -> Producer (ExceptionP p) B.ByteString SafeIO ()
readFileS chunkSize path () =
    bracket id (openFile path ReadMode) hClose $ \handle -> do
        let loop = do
                eof <- tryIO $ hIsEOF handle
                unless eof $ do
                    bs <- tryIO $ B.hGetSome handle chunkSize
                    respond bs
                    loop
        loop

firstLine :: (Proxy p) => () -> Consumer p B.ByteString IO ()
firstLine () = runIdentityP loop
  where
    loop = do
        bs <- request ()
        let (prefix, suffix) = B8.span (/= '\n') bs
        lift $ B8.putStr prefix
        if (B.null suffix) then loop else lift $ B8.putStr (B8.pack "\n")

handler :: (CheckP p) => FilePath -> Session (ExceptionP p) SafeIO ()
handler path = do
    canRead <- tryIO $ fmap readable $ getPermissions path
    isDir   <- tryIO $ doesDirectoryExist path
    when (not isDir && canRead) $
        (readFileS 1024 path >-> try . firstLine) ()

main = runSafeIO $ runProxy $ runEitherK $
      contentsRecursive "/home" />/ handler

If you want to learn more about pipes, you can begin with the tutorial.