Annotation of /sml/trunk/src/compiler/Parse/lex/tokentable.sml

Parent Directory | Revision Log

---

Revision 118 - (view) (download)

1 :	monnier	16	(* tokentable.sml
2 :			*
3 :			* COPYRIGHT (c) 1996 Bell Laboratories.
4 :			*
5 :			*)
6 :
7 :			(***************************************************************************
8 :
9 :			TOKEN.SML: hash table for token recognition
10 :
11 :			***************************************************************************)
12 :
13 :			functor TokenTable (Tokens:ML_TOKENS) : sig
14 :
15 :			val checkId : (string * int) -> (Tokens.svalue,int) Tokens.token
16 :			val checkSymId : (string * int) -> (Tokens.svalue,int) Tokens.token
17 :			val checkTyvar : (string * int) -> (Tokens.svalue,int) Tokens.token
18 :
19 :			end = struct
20 :
21 :			exception NotToken
22 :
23 :			structure Tbl = IntStrMap
24 :
25 :			val hashStr = StrgHash.hashString
26 :
27 :			fun mkTable (sz, l) = let
28 :			val t = Tbl.new (128, NotToken)
29 :			fun ins (str, tok) = Tbl.add t (hashStr str, str, tok)
30 :			in
31 :			List.app ins l;
32 :			t
33 :			end
34 :
35 :			val symIdTbl = mkTable (16, [
36 :			("*" , fn yypos => Tokens.ASTERISK(yypos,yypos+1)),
37 :			("|" , fn yypos => Tokens.BAR(yypos,yypos+1)),
38 :			(":" , fn yypos => Tokens.COLON(yypos,yypos+1)),
39 :			(":>" , fn yypos => Tokens.COLONGT(yypos,yypos+1)),
40 :			("=" , fn yypos => Tokens.EQUALOP(yypos,yypos+1)),
41 :			("#" , fn yypos => Tokens.HASH(yypos,yypos+1)),
42 :			("->" , fn yypos => Tokens.ARROW(yypos,yypos+2)),
43 :			("=>" , fn yypos => Tokens.DARROW(yypos,yypos+2))
44 :			])
45 :
46 :			val idTbl = mkTable (64, [
47 :			("and" , fn yypos => Tokens.AND(yypos,yypos+3)),
48 :			("abstype" , fn yypos => Tokens.ABSTYPE(yypos,yypos+7)),
49 :			("as" , fn yypos => Tokens.AS(yypos,yypos+2)),
50 :			("case" , fn yypos => Tokens.CASE(yypos,yypos+4)),
51 :			("datatype" , fn yypos => Tokens.DATATYPE(yypos,yypos+8)),
52 :			("else" , fn yypos => Tokens.ELSE(yypos,yypos+4)),
53 :			("end" , fn yypos => Tokens.END(yypos,yypos+3)),
54 :			("eqtype" , fn yypos => Tokens.EQTYPE(yypos,yypos+6)),
55 :			("exception", fn yypos => Tokens.EXCEPTION(yypos,yypos+9)),
56 :			("do" , fn yypos => Tokens.DO(yypos,yypos+2)),
57 :			("fn" , fn yypos => Tokens.FN(yypos,yypos+2)),
58 :			("fun" , fn yypos => Tokens.FUN(yypos,yypos+3)),
59 :			("functor" , fn yypos => Tokens.FUNCTOR(yypos,yypos+7)),
60 :			("funsig" , fn yypos => Tokens.FUNSIG(yypos,yypos+7)),
61 :			("handle" , fn yypos => Tokens.HANDLE(yypos,yypos+6)),
62 :			("if" , fn yypos => Tokens.IF(yypos,yypos+2)),
63 :			("in" , fn yypos => Tokens.IN(yypos,yypos+2)),
64 :			("include" , fn yypos => Tokens.INCLUDE(yypos,yypos+7)),
65 :			("infix" , fn yypos => Tokens.INFIX(yypos,yypos+5)),
66 :			("infixr" , fn yypos => Tokens.INFIXR(yypos,yypos+6)),
67 :			("lazy" , fn yypos => Tokens.LAZY(yypos,yypos+4)),
68 :			("let" , fn yypos => Tokens.LET(yypos,yypos+3)),
69 :			("local" , fn yypos => Tokens.LOCAL(yypos,yypos+5)),
70 :			("nonfix" , fn yypos => Tokens.NONFIX(yypos,yypos+6)),
71 :			("of" , fn yypos => Tokens.OF(yypos,yypos+2)),
72 :			("op" , fn yypos => Tokens.OP(yypos,yypos+2)),
73 :			("open" , fn yypos => Tokens.OPEN(yypos,yypos+4)),
74 :			("overload" , fn yypos => Tokens.OVERLOAD(yypos,yypos+8)),
75 :			("raise" , fn yypos => Tokens.RAISE(yypos,yypos+5)),
76 :			("rec" , fn yypos => Tokens.REC(yypos,yypos+3)),
77 :			("sharing" , fn yypos => Tokens.SHARING(yypos,yypos+7)),
78 :			("sig" , fn yypos => Tokens.SIG(yypos,yypos+3)),
79 :			("signature", fn yypos => Tokens.SIGNATURE(yypos,yypos+9)),
80 :			("struct" , fn yypos => Tokens.STRUCT(yypos,yypos+6)),
81 :			("structure", fn yypos => Tokens.STRUCTURE(yypos,yypos+9)),
82 :			("then" , fn yypos => Tokens.THEN(yypos,yypos+4)),
83 :			("type" , fn yypos => Tokens.TYPE(yypos,yypos+4)),
84 :			("val" , fn yypos => Tokens.VAL(yypos,yypos+3)),
85 :			("where" , fn yypos => Tokens.WHERE(yypos,yypos+5)),
86 :			("while" , fn yypos => Tokens.WHILE(yypos,yypos+5)),
87 :			("with" , fn yypos => Tokens.WITH(yypos,yypos+4)),
88 :			("withtype" , fn yypos => Tokens.WITHTYPE(yypos,yypos+8)),
89 :			("orelse" , fn yypos => Tokens.ORELSE(yypos,yypos+6)),
90 :			("andalso" , fn yypos => Tokens.ANDALSO(yypos,yypos+7))
91 :			])
92 :
93 :			val overloadHash = hashStr "overload"
94 :			val lazyHash = hashStr "lazy"
95 :
96 :			(* look-up an identifier. If the symbol is found, the corresponding token is
97 :			* generated with the position of its begining. Otherwise it is a regular
98 :			*)
99 :			fun checkId (str, yypos) = let
100 :			val hash = hashStr str
101 :			fun mkId () =
102 :			Tokens.ID(FastSymbol.rawSymbol(hash,str), yypos, yypos+size(str))
103 :			in
104 :			let
105 :			val tokFn = Tbl.map idTbl (hash, str)
106 :			in
107 :			if ((not(!Control.overloadKW))
108 :	monnier	109	andalso (hash = overloadHash) andalso (str = "overload"))
109 :	monnier	16	then mkId()
110 :	monnier	109	else if ((not(!Control.lazysml))
111 :			andalso (hash = lazyHash) andalso (str = "lazy"))
112 :	monnier	16	then mkId()
113 :	monnier	109	else tokFn yypos
114 :	monnier	16	end
115 :			handle NotToken => mkId()
116 :			end
117 :
118 :			fun checkSymId (str, yypos) = let
119 :			val hash = hashStr str
120 :			in
121 :			(Tbl.map symIdTbl (hash, str) yypos)
122 :			handle NotToken =>
123 :			Tokens.ID(FastSymbol.rawSymbol(hash,str), yypos, yypos+size(str))
124 :			end
125 :
126 :			fun checkTyvar (str, yypos) = let
127 :			val hash = hashStr str
128 :			in
129 :			Tokens.TYVAR (FastSymbol.rawSymbol(hash,str),yypos,yypos+size (str))
130 :			end
131 :
132 :			end
133 :
134 :			(*
135 :	monnier	118	* $Log$
136 :	monnier	109	*
137 :	monnier	16	*)

---

root@smlnj-gforge.cs.uchicago.edu	ViewVC Help
Powered by ViewVC 1.0.0