Home My Page Projects Code Snippets Project Openings diderot
Summary Activity Tracker Tasks SCM

SCM Repository

[diderot] Annotation of /branches/vis15/src/compiler/parser/diderot.lex
ViewVC logotype

Annotation of /branches/vis15/src/compiler/parser/diderot.lex

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3466 - (view) (download)

1 : jhr 3352 (* diderot.lex
2 :     *
3 :     * This code is part of the Diderot Project (http://diderot-language.cs.uchicago.edu)
4 :     *
5 :     * COPYRIGHT (c) 2015 The University of Chicago
6 :     * All rights reserved.
7 :     *)
8 :    
9 :     %name DiderotLex;
10 :    
11 :     %arg (lexErr);
12 :    
13 :     %defs(
14 :    
15 :     structure T = DiderotTokens
16 :    
17 :     (* some type lex_result is necessitated by ml-ulex *)
18 :     type lex_result = T.token
19 :    
20 :     (* the depth int ref will be used for keeping track of comment depth *)
21 :     val depth = ref 0
22 :    
23 :     (* list of string fragments to concatenate *)
24 :     val buf : string list ref = ref []
25 :    
26 :     (* add a string to the buffer *)
27 :     fun addStr s = (buf := s :: !buf)
28 :    
29 :     (* make a string from buf *)
30 :     fun mkString () = let
31 :     val s = String.concat(List.rev(!buf))
32 :     in
33 :     buf := [];
34 :     T.STRING s
35 :     end
36 :    
37 :     (* make a REAL token from a substring *)
38 : jhr 3390 fun mkReal ss = let
39 : jhr 3352 val (isNeg, rest) = (case Substring.getc ss
40 :     of SOME(#"-", r) => (true, r)
41 :     | SOME(#"+", r) => (false, r)
42 :     | _ => (false, ss)
43 :     (* end case *))
44 :     val (whole, rest) = Substring.splitl Char.isDigit rest
45 :     val rest = Substring.triml 1 rest (* remove "." *)
46 :     val (frac, rest) = Substring.splitl Char.isDigit rest
47 :     val exp = if Substring.isEmpty rest
48 :     then 0
49 :     else let
50 :     val rest = Substring.triml 1 rest (* remove "e" or "E" *)
51 :     in
52 :     #1(valOf(Int.scan StringCvt.DEC Substring.getc rest))
53 :     end
54 :     in
55 :     T.REAL(RealLit.real{
56 :     isNeg = isNeg,
57 :     whole = Substring.string whole,
58 :     frac = Substring.string frac,
59 :     exp = exp
60 :     })
61 :     end
62 :    
63 :     (* scan a number from a hexidecimal string *)
64 :     val fromHexString = valOf o (StringCvt.scanString (IntInf.scan StringCvt.HEX))
65 :    
66 :     (* eof : unit -> lex_result *)
67 :     (* ml-ulex requires this as well *)
68 :     fun eof () = T.EOF
69 :     );
70 :    
71 :     %states INITIAL STRING COM1 COM2;
72 :    
73 :     %let dig = [0-9];
74 :     %let num = {dig}+;
75 :     %let hexdigit = [0-9a-fA-F];
76 :     %let hexnum = "0x"{hexdigit}+;
77 :     %let greek = [αβγζηθλμξπρστφψω];
78 :     %let letter = [a-zA-Z]|{greek};
79 :     %let idchar = {letter}|{dig}|"_"|"'";
80 :     %let id = {letter}{idchar}*;
81 :     %let ws = " "|[\t\n\v\f\r];
82 :     %let esc = "\\"[abfnrtv\\\"]|"\\"{dig}{dig}{dig};
83 :     %let sgood = [\032-\126]&[^\"\\]; (* sgood means "characters good inside strings" *)
84 :     %let eol = "\n";
85 :    
86 :     (***** Keywords and operators *****)
87 :    
88 :     <INITIAL> "||" => (T.OP_orelse);
89 :     <INITIAL> "&&" => (T.OP_andalso);
90 :     <INITIAL> "<" => (T.OP_lt);
91 :     <INITIAL> "<=" => (T.OP_lte);
92 :     <INITIAL> "==" => (T.OP_eqeq);
93 :     <INITIAL> "!=" => (T.OP_neq);
94 :     <INITIAL> ">=" => (T.OP_gte);
95 :     <INITIAL> ">" => (T.OP_gt);
96 :     <INITIAL> "+" => (T.OP_plus);
97 :     <INITIAL> "-" => (T.OP_minus);
98 :     <INITIAL> "*" => (T.OP_star);
99 :     <INITIAL> "/" => (T.OP_slash);
100 :     <INITIAL> "%" => (T.OP_mod);
101 :     <INITIAL> "^" => (T.OP_exp);
102 :     <INITIAL> "@" => (T.OP_at);
103 :     <INITIAL> "⊛" => (T.OP_convolve); (* u229b *)
104 :     <INITIAL> "(*)" => (T.OP_convolve);
105 :     <INITIAL> "∇•" => (T.OP_Ddot); (* u2207, u2022 *)
106 :     <INITIAL> "∇⋅" => (T.OP_Ddot); (* u2207, u22c5 *)
107 :     <INITIAL> "∇⊗" => (T.OP_Dotimes); (* u2207, u2297 *)
108 :     <INITIAL> "∇×" => (T.OP_curl); (* u2207, u00d7 *)
109 :     <INITIAL> "∇" => (T.OP_D); (* u2207 *)
110 :     <INITIAL> "•" => (T.OP_dot); (* u2022 *)
111 :     <INITIAL> "⋅" => (T.OP_dot); (* u22c5 *)
112 :     <INITIAL> "×" => (T.OP_cross); (* u00d7 *)
113 :     <INITIAL> "⊗" => (T.OP_outer); (* u2297 *)
114 :     <INITIAL> "(" => (T.LP);
115 :     <INITIAL> ")" => (T.RP);
116 :     <INITIAL> "[" => (T.LB);
117 :     <INITIAL> "]" => (T.RB);
118 :     <INITIAL> "{" => (T.LCB);
119 :     <INITIAL> "}" => (T.RCB);
120 :     <INITIAL> "," => (T.COMMA);
121 :     <INITIAL> ";" => (T.SEMI);
122 :     <INITIAL> ":" => (T.COLON);
123 :     <INITIAL> "#" => (T.HASH);
124 :     <INITIAL> "!" => (T.BANG);
125 :     <INITIAL> "=" => (T.OP_eq);
126 :     <INITIAL> "+=" => (T.OP_pluseq);
127 :     <INITIAL> "-=" => (T.OP_minuseq);
128 :     <INITIAL> "*=" => (T.OP_stareq);
129 :     <INITIAL> "/=" => (T.OP_slasheq);
130 :     <INITIAL> "%=" => (T.OP_modeq);
131 :     <INITIAL> "|" => (T.BAR);
132 :     <INITIAL> "." => (T.DOT);
133 :     <INITIAL> ".." => (T.DOTDOT);
134 :    
135 :     <INITIAL> "∞" => (T.REAL RealLit.posInf); (* u221e *)
136 :     <INITIAL> "π" => (T.REAL RealLit.pi); (* u03c0 *)
137 :    
138 :     <INITIAL> {id} => (Keywords.idToken yytext);
139 :    
140 :     <INITIAL> {num} => (T.INT(valOf (IntInf.fromString yytext)));
141 :     <INITIAL> {num}"."{num}([eE][+-]?{num})?
142 : jhr 3390 => (mkReal yysubstr);
143 : jhr 3352 <INITIAL> {ws} => (skip ());
144 :    
145 :     <INITIAL> "\"" => (YYBEGIN STRING; continue());
146 :    
147 :     <INITIAL> . => (lexErr(yypos, ["bad character `", String.toString yytext]);
148 :     continue());
149 :    
150 :     (***** Strings *****)
151 :     <STRING>{esc} => (addStr(valOf(String.fromString yytext)); continue());
152 :     <STRING>{sgood}+ => (addStr yytext; continue());
153 :     <STRING> "\"" => (YYBEGIN INITIAL; mkString());
154 :    
155 :     <STRING> . => (lexErr(yypos, [
156 :     "bad character `", String.toString yytext,
157 :     "' in string literal"
158 :     ]);
159 :     continue());
160 :    
161 :     (***** Comments *****)
162 :     <INITIAL> "//" => (YYBEGIN COM1; skip());
163 :     <COM1> {eol} => (YYBEGIN INITIAL; skip());
164 :     <COM1> . => (skip());
165 :    
166 :     <INITIAL> "/*"
167 :     => (YYBEGIN COM2; skip());
168 :     <COM2> "*/"
169 :     => (YYBEGIN INITIAL; skip());
170 :     <COM2> .
171 :     => (skip());

root@smlnj-gforge.cs.uchicago.edu
ViewVC Help
Powered by ViewVC 1.0.0