2 -- The above warning supression flag is a temporary kludge.
3 -- While working on this module you are encouraged to remove it and fix
4 -- any warnings in the module. See
5 -- http://hackage.haskell.org/trac/ghc/wiki/Commentary/CodingStyle#Warnings
8 -- -----------------------------------------------------------------------------
10 -- (c) The University of Glasgow, 1997-2006
12 -- Character encodings
14 -- -----------------------------------------------------------------------------
33 #define COMPILING_FAST_STRING
34 #include "HsVersions.h"
36 import Data.Char ( ord, chr, isDigit, digitToInt, intToDigit,
38 import Numeric ( showIntAtBase )
40 import GHC.Ptr ( Ptr(..) )
43 -- -----------------------------------------------------------------------------
46 -- We can't write the decoder as efficiently as we'd like without
47 -- resorting to unboxed extensions, unfortunately. I tried to write
48 -- an IO version of this function, but GHC can't eliminate boxed
49 -- results from an IO-returning function.
51 -- We assume we can ignore overflow when parsing a multibyte character here.
52 -- To make this safe, we add extra sentinel bytes to unparsed UTF-8 sequences
53 -- before decoding them (see StringBuffer.hs).
55 {-# INLINE utf8DecodeChar# #-}
56 utf8DecodeChar# :: Addr# -> (# Char#, Addr# #)
58 let ch0 = word2Int# (indexWord8OffAddr# a# 0#) in
60 _ | ch0 <=# 0x7F# -> (# chr# ch0, a# `plusAddr#` 1# #)
62 | ch0 >=# 0xC0# && ch0 <=# 0xDF# ->
63 let ch1 = word2Int# (indexWord8OffAddr# a# 1#) in
64 if ch1 <# 0x80# || ch1 >=# 0xC0# then fail 1# else
65 (# chr# (((ch0 -# 0xC0#) `uncheckedIShiftL#` 6#) +#
69 | ch0 >=# 0xE0# && ch0 <=# 0xEF# ->
70 let ch1 = word2Int# (indexWord8OffAddr# a# 1#) in
71 if ch1 <# 0x80# || ch1 >=# 0xC0# then fail 1# else
72 let ch2 = word2Int# (indexWord8OffAddr# a# 2#) in
73 if ch2 <# 0x80# || ch2 >=# 0xC0# then fail 2# else
74 (# chr# (((ch0 -# 0xE0#) `uncheckedIShiftL#` 12#) +#
75 ((ch1 -# 0x80#) `uncheckedIShiftL#` 6#) +#
79 | ch0 >=# 0xF0# && ch0 <=# 0xF8# ->
80 let ch1 = word2Int# (indexWord8OffAddr# a# 1#) in
81 if ch1 <# 0x80# || ch1 >=# 0xC0# then fail 1# else
82 let ch2 = word2Int# (indexWord8OffAddr# a# 2#) in
83 if ch2 <# 0x80# || ch2 >=# 0xC0# then fail 2# else
84 let ch3 = word2Int# (indexWord8OffAddr# a# 3#) in
85 if ch3 <# 0x80# || ch3 >=# 0xC0# then fail 3# else
86 (# chr# (((ch0 -# 0xF0#) `uncheckedIShiftL#` 18#) +#
87 ((ch1 -# 0x80#) `uncheckedIShiftL#` 12#) +#
88 ((ch2 -# 0x80#) `uncheckedIShiftL#` 6#) +#
92 | otherwise -> fail 1#
94 -- all invalid sequences end up here:
95 fail n = (# '\0'#, a# `plusAddr#` n #)
96 -- '\xFFFD' would be the usual replacement character, but
97 -- that's a valid symbol in Haskell, so will result in a
98 -- confusing parse error later on. Instead we use '\0' which
99 -- will signal a lexer error immediately.
101 utf8DecodeChar :: Ptr Word8 -> (Char, Ptr Word8)
102 utf8DecodeChar (Ptr a#) =
103 case utf8DecodeChar# a# of (# c#, b# #) -> ( C# c#, Ptr b# )
105 -- UTF-8 is cleverly designed so that we can always figure out where
106 -- the start of the current character is, given any position in a
107 -- stream. This function finds the start of the previous character,
108 -- assuming there *is* a previous character.
109 utf8PrevChar :: Ptr Word8 -> IO (Ptr Word8)
110 utf8PrevChar p = utf8CharStart (p `plusPtr` (-1))
112 utf8CharStart :: Ptr Word8 -> IO (Ptr Word8)
113 utf8CharStart p = go p
114 where go p = do w <- peek p
115 if w >= 0x80 && w < 0xC0
116 then go (p `plusPtr` (-1))
119 utf8DecodeString :: Ptr Word8 -> Int -> IO [Char]
120 STRICT2(utf8DecodeString)
121 utf8DecodeString (Ptr a#) (I# len#)
124 end# = addr2Int# (a# `plusAddr#` len#)
127 | addr2Int# p# >=# end# = return []
129 case utf8DecodeChar# p# of
134 countUTF8Chars :: Ptr Word8 -> Int -> IO Int
135 countUTF8Chars ptr bytes = go ptr 0
137 end = ptr `plusPtr` bytes
141 | ptr >= end = return n
143 case utf8DecodeChar# (unPtr ptr) of
144 (# c, a #) -> go (Ptr a) (n+1)
148 utf8EncodeChar :: Char -> Ptr Word8 -> IO (Ptr Word8)
149 utf8EncodeChar c ptr =
152 _ | x > 0 && x <= 0x007f -> do
153 poke ptr (fromIntegral x)
154 return (ptr `plusPtr` 1)
155 -- NB. '\0' is encoded as '\xC0\x80', not '\0'. This is so that we
156 -- can have 0-terminated UTF-8 strings (see GHC.Base.unpackCStringUtf8).
158 poke ptr (fromIntegral (0xC0 .|. ((x `shiftR` 6) .&. 0x1F)))
159 pokeElemOff ptr 1 (fromIntegral (0x80 .|. (x .&. 0x3F)))
160 return (ptr `plusPtr` 2)
162 poke ptr (fromIntegral (0xE0 .|. (x `shiftR` 12) .&. 0x0F))
163 pokeElemOff ptr 1 (fromIntegral (0x80 .|. (x `shiftR` 6) .&. 0x3F))
164 pokeElemOff ptr 2 (fromIntegral (0x80 .|. (x .&. 0x3F)))
165 return (ptr `plusPtr` 3)
167 poke ptr (fromIntegral (0xF0 .|. (x `shiftR` 18)))
168 pokeElemOff ptr 1 (fromIntegral (0x80 .|. ((x `shiftR` 12) .&. 0x3F)))
169 pokeElemOff ptr 2 (fromIntegral (0x80 .|. ((x `shiftR` 6) .&. 0x3F)))
170 pokeElemOff ptr 3 (fromIntegral (0x80 .|. (x .&. 0x3F)))
171 return (ptr `plusPtr` 4)
173 utf8EncodeString :: Ptr Word8 -> String -> IO ()
174 utf8EncodeString ptr str = go ptr str
176 go ptr [] = return ()
178 ptr' <- utf8EncodeChar c ptr
181 utf8EncodedLength :: String -> Int
182 utf8EncodedLength str = go 0 str
186 | ord c > 0 && ord c <= 0x007f = go (n+1) cs
187 | ord c <= 0x07ff = go (n+2) cs
188 | ord c <= 0xffff = go (n+3) cs
189 | otherwise = go (n+4) cs
191 -- -----------------------------------------------------------------------------
195 This is the main name-encoding and decoding function. It encodes any
196 string into a string that is acceptable as a C name. This is done
197 right before we emit a symbol name into the compiled C or asm code.
198 Z-encoding of strings is cached in the FastString interface, so we
199 never encode the same string more than once.
201 The basic encoding scheme is this.
203 * Tuples (,,,) are coded as Z3T
205 * Alphabetic characters (upper and lower) and digits
206 all translate to themselves;
207 except 'Z', which translates to 'ZZ'
208 and 'z', which translates to 'zz'
209 We need both so that we can preserve the variable/tycon distinction
211 * Most other printable characters translate to 'zx' or 'Zx' for some
212 alphabetic character x
214 * The others translate as 'znnnU' where 'nnn' is the decimal number
218 --------------------------
230 (# #) Z1H unboxed 1-tuple (note the space)
231 (#,,,,#) Z5H unboxed 5-tuple
232 (NB: There is no Z1T nor Z0H.)
235 type UserString = String -- As the user typed it
236 type EncodedString = String -- Encoded form
239 zEncodeString :: UserString -> EncodedString
240 zEncodeString cs = case maybe_tuple cs of
241 Just n -> n -- Tuples go to Z2T etc
245 go (c:cs) = encode_ch c ++ go cs
247 unencodedChar :: Char -> Bool -- True for chars that don't need encoding
248 unencodedChar 'Z' = False
249 unencodedChar 'z' = False
250 unencodedChar c = c >= 'a' && c <= 'z'
251 || c >= 'A' && c <= 'Z'
252 || c >= '0' && c <= '9'
254 encode_ch :: Char -> EncodedString
255 encode_ch c | unencodedChar c = [c] -- Common case first
258 encode_ch '(' = "ZL" -- Needed for things like (,), and (->)
259 encode_ch ')' = "ZR" -- For symmetry with (
279 encode_ch '\'' = "zq"
280 encode_ch '\\' = "zr"
285 encode_ch c = 'z' : if isDigit (head hex_str) then hex_str
287 where hex_str = showHex (ord c) "U"
288 -- ToDo: we could improve the encoding here in various ways.
289 -- eg. strings of unicode characters come out as 'z1234Uz5678U', we
290 -- could remove the 'U' in the middle (the 'z' works as a separator).
292 showHex = showIntAtBase 16 intToDigit
293 -- needed because prior to GHC 6.2, Numeric.showHex added a "0x" prefix
295 zDecodeString :: EncodedString -> UserString
296 zDecodeString [] = []
297 zDecodeString ('Z' : d : rest)
298 | isDigit d = decode_tuple d rest
299 | otherwise = decode_upper d : zDecodeString rest
300 zDecodeString ('z' : d : rest)
301 | isDigit d = decode_num_esc d rest
302 | otherwise = decode_lower d : zDecodeString rest
303 zDecodeString (c : rest) = c : zDecodeString rest
305 decode_upper, decode_lower :: Char -> Char
307 decode_upper 'L' = '('
308 decode_upper 'R' = ')'
309 decode_upper 'M' = '['
310 decode_upper 'N' = ']'
311 decode_upper 'C' = ':'
312 decode_upper 'Z' = 'Z'
313 decode_upper ch = {-pprTrace "decode_upper" (char ch)-} ch
315 decode_lower 'z' = 'z'
316 decode_lower 'a' = '&'
317 decode_lower 'b' = '|'
318 decode_lower 'c' = '^'
319 decode_lower 'd' = '$'
320 decode_lower 'e' = '='
321 decode_lower 'g' = '>'
322 decode_lower 'h' = '#'
323 decode_lower 'i' = '.'
324 decode_lower 'l' = '<'
325 decode_lower 'm' = '-'
326 decode_lower 'n' = '!'
327 decode_lower 'p' = '+'
328 decode_lower 'q' = '\''
329 decode_lower 'r' = '\\'
330 decode_lower 's' = '/'
331 decode_lower 't' = '*'
332 decode_lower 'u' = '_'
333 decode_lower 'v' = '%'
334 decode_lower ch = {-pprTrace "decode_lower" (char ch)-} ch
336 -- Characters not having a specific code are coded as z224U (in hex)
337 decode_num_esc d rest
338 = go (digitToInt d) rest
340 go n (c : rest) | isHexDigit c = go (16*n + digitToInt c) rest
341 go n ('U' : rest) = chr n : zDecodeString rest
342 go n other = error ("decode_num_esc: " ++ show n ++ ' ':other)
344 decode_tuple :: Char -> EncodedString -> UserString
346 = go (digitToInt d) rest
348 -- NB. recurse back to zDecodeString after decoding the tuple, because
349 -- the tuple might be embedded in a longer name.
350 go n (c : rest) | isDigit c = go (10*n + digitToInt c) rest
351 go 0 ('T':rest) = "()" ++ zDecodeString rest
352 go n ('T':rest) = '(' : replicate (n-1) ',' ++ ")" ++ zDecodeString rest
353 go 1 ('H':rest) = "(# #)" ++ zDecodeString rest
354 go n ('H':rest) = '(' : '#' : replicate (n-1) ',' ++ "#)" ++ zDecodeString rest
355 go n other = error ("decode_tuple: " ++ show n ++ ' ':other)
358 Tuples are encoded as
360 for 3-tuples or unboxed 3-tuples respectively. No other encoding starts
363 * "(# #)" is the tycon for an unboxed 1-tuple (not 0-tuple)
364 There are no unboxed 0-tuples.
366 * "()" is the tycon for a boxed 0-tuple.
367 There are no boxed 1-tuples.
370 maybe_tuple :: UserString -> Maybe EncodedString
372 maybe_tuple "(# #)" = Just("Z1H")
373 maybe_tuple ('(' : '#' : cs) = case count_commas (0::Int) cs of
374 (n, '#' : ')' : cs) -> Just ('Z' : shows (n+1) "H")
376 maybe_tuple "()" = Just("Z0T")
377 maybe_tuple ('(' : cs) = case count_commas (0::Int) cs of
378 (n, ')' : cs) -> Just ('Z' : shows (n+1) "T")
380 maybe_tuple other = Nothing
382 count_commas :: Int -> String -> (Int, String)
383 count_commas n (',' : cs) = count_commas (n+1) cs
384 count_commas n cs = (n,cs)