Home My Page Projects Code Snippets Project Openings diderot
Summary Activity Tracker Tasks SCM

SCM Repository

[diderot] Annotation of /branches/vis12/src/compiler/parser/diderot.lex
ViewVC logotype

Annotation of /branches/vis12/src/compiler/parser/diderot.lex

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1982 - (view) (download)

1 : jhr 14 (* diderot.lex
2 :     *
3 : jhr 435 * COPYRIGHT (c) 2010 The Diderot Project (http://diderot-language.cs.uchicago.edu)
4 : jhr 14 * All rights reserved.
5 :     *)
6 :    
7 :     %name DiderotLex;
8 :    
9 :     %arg (lexErr);
10 :    
11 :     %defs(
12 :    
13 : jhr 30 structure T = DiderotTokens
14 : jhr 14
15 :     (* some type lex_result is necessitated by ml-ulex *)
16 :     type lex_result = T.token
17 :    
18 :     (* the depth int ref will be used for keeping track of comment depth *)
19 :     val depth = ref 0
20 :    
21 :     (* list of string fragments to concatenate *)
22 :     val buf : string list ref = ref []
23 :    
24 :     (* add a string to the buffer *)
25 :     fun addStr s = (buf := s :: !buf)
26 :    
27 :     (* make a string from buf *)
28 :     fun mkString () = let
29 :     val s = String.concat(List.rev(!buf))
30 :     in
31 :     buf := [];
32 :     T.STRING s
33 :     end
34 :    
35 :     (* make a FLOAT token from a substring *)
36 :     fun mkFloat ss = let
37 :     val (isNeg, rest) = (case Substring.getc ss
38 :     of SOME(#"-", r) => (true, r)
39 :     | SOME(#"+", r) => (false, r)
40 :     | _ => (false, ss)
41 :     (* end case *))
42 :     val (whole, rest) = Substring.splitl Char.isDigit rest
43 :     val rest = Substring.triml 1 rest (* remove "." *)
44 :     val (frac, rest) = Substring.splitl Char.isDigit rest
45 :     val exp = if Substring.isEmpty rest
46 :     then 0
47 :     else let
48 :     val rest = Substring.triml 1 rest (* remove "e" or "E" *)
49 :     in
50 :     #1(valOf(Int.scan StringCvt.DEC Substring.getc rest))
51 :     end
52 :     in
53 :     T.FLOAT(FloatLit.float{
54 :     isNeg = isNeg,
55 :     whole = Substring.string whole,
56 :     frac = Substring.string frac,
57 :     exp = exp
58 :     })
59 :     end
60 :    
61 :     (* scan a number from a hexidecimal string *)
62 :     val fromHexString = valOf o (StringCvt.scanString (IntInf.scan StringCvt.HEX))
63 :     (* FIXME: the above code doesn't work in SML/NJ; here is a work around *)
64 :     fun fromHexString s = let
65 :     val SOME(n, _) = IntInf.scan StringCvt.HEX Substring.getc
66 :     (Substring.triml 2 (Substring.full s))
67 :     in
68 :     n
69 :     end
70 :    
71 :     (* eof : unit -> lex_result *)
72 :     (* ml-ulex requires this as well *)
73 :     fun eof () = T.EOF
74 :     );
75 :    
76 :     %states INITIAL STRING COM1 COM2;
77 :    
78 :     %let dig = [0-9];
79 :     %let num = {dig}+;
80 :     %let hexdigit = [0-9a-fA-F];
81 :     %let hexnum = "0x"{hexdigit}+;
82 : jhr 1982 %let greek = [αβγδζηθλμξπρσφψω];
83 :     %let letter = [a-zA-Z]|{greek};
84 : jhr 14 %let idchar = {letter}|{dig}|"_"|"'";
85 :     %let id = {letter}{idchar}*;
86 :     %let ws = " "|[\t\n\v\f\r];
87 : jhr 35 %let esc = "\\"[abfnrtv\\\"]|"\\"{dig}{dig}{dig};
88 :     %let sgood = [\032-\126]&[^\"\\]; (* sgood means "characters good inside strings" *)
89 :     %let eol = "\n";
90 : jhr 14
91 :     (***** Keywords and operators *****)
92 :    
93 : jhr 26 <INITIAL> "||" => (T.OP_orelse);
94 :     <INITIAL> "&&" => (T.OP_andalso);
95 :     <INITIAL> "<" => (T.OP_lt);
96 :     <INITIAL> "<=" => (T.OP_lte);
97 :     <INITIAL> "==" => (T.OP_eqeq);
98 :     <INITIAL> "!=" => (T.OP_neq);
99 :     <INITIAL> ">=" => (T.OP_gte);
100 :     <INITIAL> ">" => (T.OP_gt);
101 :     <INITIAL> "+" => (T.OP_plus);
102 :     <INITIAL> "-" => (T.OP_minus);
103 :     <INITIAL> "*" => (T.OP_star);
104 : jhr 35 <INITIAL> "/" => (T.OP_slash);
105 : jhr 1116 <INITIAL> "^" => (T.OP_exp);
106 : jhr 26 <INITIAL> "@" => (T.OP_at);
107 : jhr 1116 <INITIAL> "⊛" => (T.OP_convolve); (* u229b *)
108 : jhr 247 <INITIAL> "(*)" => (T.OP_convolve);
109 : jhr 1116 <INITIAL> "∇•" => (T.OP_Ddot); (* u2207, u2022 *)
110 :     <INITIAL> "∇⋅" => (T.OP_Ddot); (* u2207, u22c5 *)
111 :     <INITIAL> "∇⊗" => (T.OP_Dotimes); (* u2207, u2297 *)
112 :     <INITIAL> "∇×" => (T.OP_Dtimes); (* u2207, u00d7 *)
113 :     <INITIAL> "∇" => (T.OP_D); (* u2207 *)
114 :     <INITIAL> "•" => (T.OP_dot); (* u2022 *)
115 :     <INITIAL> "⋅" => (T.OP_dot); (* u22c5 *)
116 :     <INITIAL> "×" => (T.OP_cross); (* u00d7 *)
117 :     <INITIAL> "⊗" => (T.OP_outer); (* u2297 *)
118 : jhr 35 <INITIAL> "(" => (T.LP);
119 :     <INITIAL> ")" => (T.RP);
120 :     <INITIAL> "[" => (T.LB);
121 :     <INITIAL> "]" => (T.RB);
122 :     <INITIAL> "{" => (T.LCB);
123 :     <INITIAL> "}" => (T.RCB);
124 : jhr 23 <INITIAL> "," => (T.COMMA);
125 :     <INITIAL> ";" => (T.SEMI);
126 : jhr 381 <INITIAL> ":" => (T.COLON);
127 : jhr 23 <INITIAL> "#" => (T.HASH);
128 : jhr 26 <INITIAL> "!" => (T.BANG);
129 : jhr 35 <INITIAL> "=" => (T.OP_eq);
130 : jhr 1296 <INITIAL> "+=" => (T.OP_pluseq);
131 :     <INITIAL> "-=" => (T.OP_minuseq);
132 :     <INITIAL> "*=" => (T.OP_stareq);
133 :     <INITIAL> "/=" => (T.OP_slasheq);
134 : jhr 40 <INITIAL> "|" => (T.BAR);
135 :     <INITIAL> ".." => (T.DOTDOT);
136 : jhr 14
137 : jhr 1116 <INITIAL> "∞" => (T.FLOAT FloatLit.posInf); (* u221e *)
138 :     <INITIAL> "π" => (T.FLOAT FloatLit.pi); (* u03c0 *)
139 : jhr 250
140 : jhr 33 <INITIAL> {id} => (Keywords.idToken yytext);
141 :    
142 : jhr 40 <INITIAL> {num} => (T.INT(valOf (IntInf.fromString yytext)));
143 :     <INITIAL> {num}"."{num}([eE][+-]?{num})?
144 : jhr 33 => (mkFloat yysubstr);
145 :     <INITIAL> {ws} => (skip ());
146 :    
147 : jhr 50 <INITIAL> "\"" => (YYBEGIN STRING; continue());
148 :    
149 : jhr 35 <INITIAL> . => (lexErr(yypos, ["bad character `", String.toString yytext]);
150 :     continue());
151 :    
152 :     (***** Strings *****)
153 :     <STRING>{esc} => (addStr(valOf(String.fromString yytext)); continue());
154 :     <STRING>{sgood}+ => (addStr yytext; continue());
155 :     <STRING> "\"" => (YYBEGIN INITIAL; mkString());
156 :    
157 :     <STRING> . => (lexErr(yypos, [
158 :     "bad character `", String.toString yytext,
159 :     "' in string literal"
160 :     ]);
161 :     continue());
162 :    
163 : jhr 14 (***** Comments *****)
164 : jhr 35 <INITIAL> "//" => (YYBEGIN COM1; skip());
165 :     <COM1> {eol} => (YYBEGIN INITIAL; skip());
166 :     <COM1> . => (skip());
167 : jhr 14
168 :     <INITIAL> "/*"
169 :     => (YYBEGIN COM2; skip());
170 :     <COM2> "*/"
171 :     => (YYBEGIN INITIAL; skip());
172 :     <COM2> .
173 :     => (skip());

root@smlnj-gforge.cs.uchicago.edu
ViewVC Help
Powered by ViewVC 1.0.0