Annotation of /smlnj-lib/trunk/JSON/json.lex

Parent Directory | Revision Log

---

Revision 3004 - (view) (download)

1 :	jhr	3001	(* json.lex
2 :			*
3 :			* COPYRIGHT (c) 2008 The Fellowship of SML/NJ (http://www.smlnj.org)
4 :			* All rights reserved.
5 :			*
6 :			* Lexer for JSON files.
7 :			*)
8 :
9 :			%name JSONLexer;
10 :
11 :			%defs (
12 :			structure T = JSONTokens
13 :			type lex_result = T.token
14 :			fun eof () = T.EOF
15 :			fun int s = T.INT(valOf(IntInf.fromString s))
16 :			fun float s = T.FLOAT(valOf(LargeReal.fromString s))
17 :	jhr	3003	(* support for incremental construction of strings *)
18 :			val sbuf : string list ref = ref []
19 :			fun addStr s = sbuf := s :: !sbuf
20 :	jhr	3004	fun addUChr lit = let
21 :			(* trim the "\u" prefix *)
22 :			val digits = Substring.triml 2 (Substring.full lit)
23 :			val SOME(w, _) = Word.scan StringCvt.HEX Substring.getc digits
24 :			in
25 :			addStr(UTF8.encode w)
26 :			end
27 :	jhr	3003	fun finishString () = (String.concat(List.rev(!sbuf)) before sbuf := [])
28 :			);
29 :	jhr	3001
30 :			%let digit1_9 = [1-9];
31 :			%let digit = [0-9];
32 :			%let digits = {digit}+;
33 :			%let int = "-"?({digit} | {digit1_9}{digits}+);
34 :	jhr	3003	%let frac = "."{digits};
35 :	jhr	3001	%let exp = [eE][+-]?{digits};
36 :			%let xdigit = {digit}|[a-fA-F];
37 :
38 :			%states S;
39 :
40 :			[ \t\n\r]+ => (skip() );
41 :
42 :			"{" => ( T.LCB );
43 :			"}" => ( T.RCB );
44 :			"[" => ( T.LB );
45 :			"]" => ( T.RB );
46 :			"," => ( T.COMMA );
47 :			":" => ( T.COLON );
48 :			"null" => ( T.KW_null );
49 :			"true" => ( T.KW_true );
50 :			"false" => ( T.KW_false );
51 :
52 :			{int} => ( T.INT(valOf(IntInf.fromString yytext)) );
53 :
54 :			{int}{frac} => ( float yytext );
55 :			{int}{exp} => ( float yytext );
56 :			{int}{frac}{exp} => ( float yytext );
57 :
58 :			"\"" => ( YYBEGIN S; continue() );
59 :			<S>"\\\"" => ( addStr "\\"; continue() );
60 :			<S>"\\/" => ( addStr "/"; continue() );
61 :			<S>"\\b" => ( addStr "\b"; continue() );
62 :			<S>"\\f" => ( addStr "\f"; continue() );
63 :			<S>"\\n" => ( addStr "\n"; continue() );
64 :			<S>"\\r" => ( addStr "\r"; continue() );
65 :			<S>"\\t" => ( addStr "\t"; continue() );
66 :			<S>"\\u"{xdigit}{4} => ( addUChr yytext; continue() );
67 :			<S>[^\\"]+ => ( addStr yytext; continue() );
68 :			<S>"\"" => ( YYBEGIN INITIAL; finishString() );
69 :
70 :	jhr	3003	"/*"~(.*"*/".*)"*/" => ( skip() );

---

root@smlnj-gforge.cs.uchicago.edu	ViewVC Help
Powered by ViewVC 1.0.0