Home My Page Projects Code Snippets Project Openings diderot
Summary Activity Tracker Tasks SCM

SCM Repository

[diderot] Annotation of /trunk/src/parser/diderot.lex
ViewVC logotype

Annotation of /trunk/src/parser/diderot.lex

Parent Directory Parent Directory | Revision Log Revision Log


Revision 33 - (view) (download)

1 : jhr 14 (* diderot.lex
2 :     *
3 :     * COPYRIGHT (c) 2010 The Diderot Project (http://diderot.cs.uchicago.edu)
4 :     * All rights reserved.
5 :     *)
6 :    
7 :     %name DiderotLex;
8 :    
9 :     %arg (lexErr);
10 :    
11 :     %defs(
12 :    
13 : jhr 30 structure T = DiderotTokens
14 : jhr 14
15 :     (* some type lex_result is necessitated by ml-ulex *)
16 :     type lex_result = T.token
17 :    
18 :     (* the depth int ref will be used for keeping track of comment depth *)
19 :     val depth = ref 0
20 :    
21 :     (* list of string fragments to concatenate *)
22 :     val buf : string list ref = ref []
23 :    
24 :     (* add a string to the buffer *)
25 :     fun addStr s = (buf := s :: !buf)
26 :    
27 :     (* make a string from buf *)
28 :     fun mkString () = let
29 :     val s = String.concat(List.rev(!buf))
30 :     in
31 :     buf := [];
32 :     T.STRING s
33 :     end
34 :    
35 :     (* make a FLOAT token from a substring *)
36 :     fun mkFloat ss = let
37 :     val (isNeg, rest) = (case Substring.getc ss
38 :     of SOME(#"-", r) => (true, r)
39 :     | SOME(#"+", r) => (false, r)
40 :     | _ => (false, ss)
41 :     (* end case *))
42 :     val (whole, rest) = Substring.splitl Char.isDigit rest
43 :     val rest = Substring.triml 1 rest (* remove "." *)
44 :     val (frac, rest) = Substring.splitl Char.isDigit rest
45 :     val exp = if Substring.isEmpty rest
46 :     then 0
47 :     else let
48 :     val rest = Substring.triml 1 rest (* remove "e" or "E" *)
49 :     in
50 :     #1(valOf(Int.scan StringCvt.DEC Substring.getc rest))
51 :     end
52 :     in
53 :     T.FLOAT(FloatLit.float{
54 :     isNeg = isNeg,
55 :     whole = Substring.string whole,
56 :     frac = Substring.string frac,
57 :     exp = exp
58 :     })
59 :     end
60 :    
61 :     (* scan a number from a hexidecimal string *)
62 :     val fromHexString = valOf o (StringCvt.scanString (IntInf.scan StringCvt.HEX))
63 :     (* FIXME: the above code doesn't work in SML/NJ; here is a work around *)
64 :     fun fromHexString s = let
65 :     val SOME(n, _) = IntInf.scan StringCvt.HEX Substring.getc
66 :     (Substring.triml 2 (Substring.full s))
67 :     in
68 :     n
69 :     end
70 :    
71 :     (* eof : unit -> lex_result *)
72 :     (* ml-ulex requires this as well *)
73 :     fun eof () = T.EOF
74 :     );
75 :    
76 :     %states INITIAL STRING COM1 COM2;
77 :    
78 :     %let letter = [a-zA-Z];
79 :     %let dig = [0-9];
80 :     %let num = {dig}+;
81 :     %let hexdigit = [0-9a-fA-F];
82 :     %let hexnum = "0x"{hexdigit}+;
83 :     %let idchar = {letter}|{dig}|"_"|"'";
84 :     %let id = {letter}{idchar}*;
85 :     %let ws = " "|[\t\n\v\f\r];
86 :    
87 :     (***** Keywords and operators *****)
88 :    
89 :     <INITIAL> "(" => (T.LP);
90 :     <INITIAL> ")" => (T.RP);
91 :     <INITIAL> "[" => (T.LB);
92 :     <INITIAL> "]" => (T.RB);
93 :     <INITIAL> "{" => (T.LCB);
94 :     <INITIAL> "}" => (T.RCB);
95 : jhr 26 <INITIAL> "||" => (T.OP_orelse);
96 :     <INITIAL> "&&" => (T.OP_andalso);
97 :     <INITIAL> "<" => (T.OP_lt);
98 :     <INITIAL> "<=" => (T.OP_lte);
99 :     <INITIAL> "==" => (T.OP_eqeq);
100 :     <INITIAL> "!=" => (T.OP_neq);
101 :     <INITIAL> ">=" => (T.OP_gte);
102 :     <INITIAL> ">" => (T.OP_gt);
103 :     <INITIAL> "+" => (T.OP_plus);
104 :     <INITIAL> "-" => (T.OP_minus);
105 :     <INITIAL> "*" => (T.OP_star);
106 :     <INITIAL> "@" => (T.OP_at);
107 : jhr 23 <INITIAL> "," => (T.COMMA);
108 :     <INITIAL> ";" => (T.SEMI);
109 :     <INITIAL> "#" => (T.HASH);
110 : jhr 26 <INITIAL> "!" => (T.BANG);
111 : jhr 14
112 : jhr 33 <INITIAL> {id} => (Keywords.idToken yytext);
113 :    
114 :     <INITIAL> {num} => (T.POSINT(valOf (IntInf.fromString yytext)));
115 :     <INITIAL> "-"{num} => (T.NEGINT(valOf (IntInf.fromString yytext)));
116 :     <INITIAL> "-"?{num}"."{num}([eE][+-]?{num})?
117 :     => (mkFloat yysubstr);
118 :     <INITIAL> {ws} => (skip ());
119 :    
120 : jhr 14 (***** Comments *****)
121 :     <INITIAL> "//"
122 :     => (YYBEGIN COM1; skip());
123 :     <COM1> {eol}
124 :     => (YYBEGIN INITIAL; skip());
125 :    
126 :     <INITIAL> "/*"
127 :     => (YYBEGIN COM2; skip());
128 :     <COM2> "*/"
129 :     => (YYBEGIN INITIAL; skip());
130 :     <COM2> .
131 :     => (skip());

root@smlnj-gforge.cs.uchicago.edu
ViewVC Help
Powered by ViewVC 1.0.0