- (\handle ->
- do buf <- hGetStringBufferBlock handle blockSize
- loop handle buf)
- where blockSize = 1024
- loop handle buf
- | len buf == 0 = return []
- | otherwise
- = case getOptions' dflags buf filename of
- (Nothing, opts) -> return opts
- (Just buf', opts) -> do nextBlock <- hGetStringBufferBlock handle blockSize
- newBuf <- appendStringBuffers buf' nextBlock
- if len newBuf == len buf
- then return opts
- else do opts' <- loop handle newBuf
- return (opts++opts')
+ (\handle -> do
+ opts <- fmap getOptions' $ lazyGetToks dflags filename handle
+ seqList opts $ return opts)
+
+blockSize :: Int
+-- blockSize = 17 -- for testing :-)
+blockSize = 1024
+
+lazyGetToks :: DynFlags -> FilePath -> Handle -> IO [Located Token]
+lazyGetToks dflags filename handle = do
+ buf <- hGetStringBufferBlock handle blockSize
+ unsafeInterleaveIO $ lazyLexBuf handle (pragState dflags buf loc) False
+ where
+ loc = mkSrcLoc (mkFastString filename) 1 1
+
+ lazyLexBuf :: Handle -> PState -> Bool -> IO [Located Token]
+ lazyLexBuf handle state eof = do
+ case unP (lexer return) state of
+ POk state' t -> do
+ -- pprTrace "lazyLexBuf" (text (show (buffer state'))) (return ())
+ if atEnd (buffer state') && not eof
+ -- if this token reached the end of the buffer, and we haven't
+ -- necessarily read up to the end of the file, then the token might
+ -- be truncated, so read some more of the file and lex it again.
+ then getMore handle state
+ else case t of
+ L _ ITeof -> return [t]
+ _other -> do rest <- lazyLexBuf handle state' eof
+ return (t : rest)
+ _ | not eof -> getMore handle state
+ | otherwise -> return [L (last_loc state) ITeof]
+ -- parser assumes an ITeof sentinel at the end
+
+ getMore :: Handle -> PState -> IO [Located Token]
+ getMore handle state = do
+ -- pprTrace "getMore" (text (show (buffer state))) (return ())
+ nextbuf <- hGetStringBufferBlock handle blockSize
+ if (len nextbuf == 0) then lazyLexBuf handle state True else do
+ newbuf <- appendStringBuffers (buffer state) nextbuf
+ unsafeInterleaveIO $ lazyLexBuf handle state{buffer=newbuf} False
+
+
+getToks :: DynFlags -> FilePath -> StringBuffer -> [Located Token]
+getToks dflags filename buf = lexAll (pragState dflags buf loc)
+ where
+ loc = mkSrcLoc (mkFastString filename) 1 1
+
+ lexAll state = case unP (lexer return) state of
+ POk _ t@(L _ ITeof) -> [t]
+ POk state' t -> t : lexAll state'
+ _ -> [L (last_loc state) ITeof]
+