Home My Page Projects Code Snippets Project Openings SML/NJ
Summary Activity Forums Tracker Lists Tasks Docs Surveys News SCM Files

SCM Repository

[smlnj] Annotation of /smlnj-lib/trunk/JSON/json.lex
ViewVC logotype

Annotation of /smlnj-lib/trunk/JSON/json.lex

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3632 - (view) (download)

1 : jhr 3001 (* json.lex
2 :     *
3 :     * COPYRIGHT (c) 2008 The Fellowship of SML/NJ (http://www.smlnj.org)
4 :     * All rights reserved.
5 :     *
6 :     * Lexer for JSON files.
7 : jhr 3018 *
8 :     * TODO:
9 :     * EOF rules for strings
10 :     * newlines in strings
11 :     * error messages for unknown characters
12 : jhr 3001 *)
13 :    
14 :     %name JSONLexer;
15 :    
16 :     %defs (
17 :     structure T = JSONTokens
18 :     type lex_result = T.token
19 :     fun eof () = T.EOF
20 :     fun int s = T.INT(valOf(IntInf.fromString s))
21 :     fun float s = T.FLOAT(valOf(LargeReal.fromString s))
22 : jhr 3003 (* support for incremental construction of strings *)
23 :     val sbuf : string list ref = ref []
24 :     fun addStr s = sbuf := s :: !sbuf
25 : jhr 3004 fun addUChr lit = let
26 :     (* trim the "\u" prefix *)
27 :     val digits = Substring.triml 2 (Substring.full lit)
28 :     val SOME(w, _) = Word.scan StringCvt.HEX Substring.getc digits
29 :     in
30 :     addStr(UTF8.encode w)
31 :     end
32 : jhr 3018 fun finishString () = (T.STRING(String.concat(List.rev(!sbuf))) before sbuf := [])
33 : jhr 3003 );
34 : jhr 3001
35 :     %let digit1_9 = [1-9];
36 :     %let digit = [0-9];
37 :     %let digits = {digit}+;
38 :     %let int = "-"?({digit} | {digit1_9}{digits}+);
39 : jhr 3003 %let frac = "."{digits};
40 : jhr 3001 %let exp = [eE][+-]?{digits};
41 :     %let xdigit = {digit}|[a-fA-F];
42 :    
43 :     %states S;
44 :    
45 : jhr 3018 <INITIAL>[\ \t\n\r]+ => ( skip() );
46 : jhr 3001
47 : jhr 3018 <INITIAL>"{" => ( T.LCB );
48 :     <INITIAL>"}" => ( T.RCB );
49 :     <INITIAL>"[" => ( T.LB );
50 :     <INITIAL>"]" => ( T.RB );
51 :     <INITIAL>"," => ( T.COMMA );
52 :     <INITIAL>":" => ( T.COLON );
53 :     <INITIAL>"null" => ( T.KW_null );
54 :     <INITIAL>"true" => ( T.KW_true );
55 :     <INITIAL>"false" => ( T.KW_false );
56 : jhr 3001
57 : jhr 3018 <INITIAL>{int} => ( T.INT(valOf(IntInf.fromString yytext)) );
58 : jhr 3001
59 : jhr 3018 <INITIAL>{int}{frac} => ( float yytext );
60 :     <INITIAL>{int}{exp} => ( float yytext );
61 :     <INITIAL>{int}{frac}{exp} => ( float yytext );
62 : jhr 3001
63 : jhr 3018 <INITIAL>"\"" => ( YYBEGIN S; continue() );
64 : jhr 3632 <S>"\\\\" => ( addStr "\\"; continue() );
65 :     <S>"\\\"" => ( addStr "\""; continue() );
66 : jhr 3018 <S>"\\/" => ( addStr "/"; continue() );
67 :     <S>"\\b" => ( addStr "\b"; continue() );
68 :     <S>"\\f" => ( addStr "\f"; continue() );
69 :     <S>"\\n" => ( addStr "\n"; continue() );
70 :     <S>"\\r" => ( addStr "\r"; continue() );
71 :     <S>"\\t" => ( addStr "\t"; continue() );
72 :     <S>"\\u"{xdigit}{4} => ( addUChr yytext; continue() );
73 :     <S>[^\\"]+ => ( addStr yytext; continue() );
74 :     <S>"\"" => ( YYBEGIN INITIAL; finishString() );
75 : jhr 3001
76 : jhr 3018 <INITIAL>"/*"(~(.*"*/".*))"*/" => ( skip() );
77 :    
78 :     (* FIXME: add some error reporting *)
79 :     <INITIAL>. => ( skip() );

root@smlnj-gforge.cs.uchicago.edu
ViewVC Help
Powered by ViewVC 1.0.0