Annotation of /sml/trunk/ckit/src/parser/grammar/tokentable.sml

Parent Directory | Revision Log

---

Revision 809 - (view) (download)

1 :	dbm	597	(* Copyright (c) 1998 by Lucent Technologies *)
2 :
3 :			(***************************************************************************
4 :
5 :			TOKEN.SML: hash table for token recognition
6 :
7 :			***************************************************************************)
8 :
9 :
10 :			signature TOKENTABLE =
11 :			sig
12 :			structure Tokens : C_TOKENS
13 :			val checkToken : (string * int) -> (Tokens.svalue,int)Tokens.token
14 :			end
15 :
16 :			functor TokenTable(structure Tokens : C_TOKENS): TOKENTABLE =
17 :			struct
18 :
19 :			structure Tokens = Tokens
20 :			structure ParseControl = Config.ParseControl
21 :			type item = (int * int) -> (Tokens.svalue, int) Tokens.token
22 :			exception Keyword
23 :			exception LexError
24 :			val keywords : item AtomTable.hash_table = AtomTable.mkTable(64, Keyword)
25 :
26 :	blume	809	local
27 :	dbm	597	val insert = AtomTable.insert keywords
28 :			fun ins (s, item) = insert (Atom.atom s, item)
29 :	blume	802
30 :	blume	809	fun idTok (s, pos, endPos) =
31 :			if TypeDefs.checkTdef(s) = true then
32 :			Tokens.TYPE_NAME(s,pos,endPos)
33 :			else Tokens.ID(s,pos,endPos)
34 :
35 :	blume	802	(* to enter GCC-style 'underscore'-versions of certain keywords *)
36 :			fun insaug (s, item) = let
37 :	blume	809	fun item' (p as (pos, endPos)) =
38 :			case ParseControl.underscoreKeywords of
39 :			NONE => idTok (s, pos, endPos)
40 :			| SOME true => item p
41 :			| SOME false =>
42 :			(ParseControl.violation
43 :			(concat ["gcc-style keywords '__", s, "' or '__",
44 :			s, "__' are not allowed"]);
45 :			raise LexError)
46 :	blume	802	in
47 :			ins ("__" ^ s, item');
48 :			ins ("__" ^ s ^ "__", item')
49 :			end
50 :
51 :			val normaltokens =
52 :			[("auto", Tokens.AUTO),
53 :			("extern", Tokens.EXTERN),
54 :			("register", Tokens.REGISTER),
55 :			("static", Tokens.STATIC),
56 :			("unsigned", Tokens.UNSIGNED),
57 :			("break", Tokens.BREAK),
58 :			("case", Tokens.CASE),
59 :			("continue", Tokens.CONTINUE),
60 :			("default", Tokens.DEFAULT),
61 :			("do", Tokens.DO),
62 :			("else", Tokens.ELSE),
63 :			("for", Tokens.FOR),
64 :			("goto", Tokens.GOTO),
65 :			("if", Tokens.IF),
66 :			("enum", Tokens.ENUM),
67 :			("float", Tokens.FLOAT),
68 :			("double", Tokens.DOUBLE),
69 :			("char", Tokens.CHAR),
70 :			("int", Tokens.INT),
71 :			("long", Tokens.LONG),
72 :			("short", Tokens.SHORT),
73 :			("struct", Tokens.STRUCT),
74 :			("union", Tokens.UNION),
75 :			("void", Tokens.VOID),
76 :			("sizeof", Tokens.SIZEOF),
77 :			("typedef", Tokens.TYPEDEF),
78 :			("return", Tokens.RETURN),
79 :			("switch", Tokens.SWITCH),
80 :			("while", Tokens.WHILE)]
81 :
82 :			(* tokens for which gcc has __* and __*__ versions *)
83 :			val augmentabletokens =
84 :			[("signed", Tokens.SIGNED),
85 :			("const", fn p => if ParseControl.constAllowed
86 :			then (Tokens.CONST p)
87 :			else (ParseControl.violation
88 :			"the keyword 'const' not allowed";
89 :			raise LexError)),
90 :			("volatile", fn p => if ParseControl.volatileAllowed
91 :			then (Tokens.VOLATILE p)
92 :			else (ParseControl.violation
93 :			"the keyword 'volatile' not allowed";
94 :			raise LexError))]
95 :
96 :			(* tokens for D *)
97 :			val dtokens =
98 :			[
99 :			]
100 :	dbm	597
101 :	blume	809	val _ =
102 :			(app ins normaltokens;
103 :			app ins augmentabletokens;
104 :			app insaug augmentabletokens;
105 :			(* enter D keywords only when allowed...
106 :			* (I think the ParseControl test is done at the wrong time here.
107 :			* - Blume) *)
108 :			if ParseControl.Dkeywords then app ins dtokens else ())
109 :	dbm	597	in
110 :	blume	809	fun checkToken (s, pos) = let
111 :			val endPos = pos + size s
112 :			val name = Atom.atom s
113 :			in
114 :			case (AtomTable.find keywords name) of
115 :			SOME tokFn => tokFn(pos, endPos)
116 :			| NONE => idTok (s, pos, endPos)
117 :			end
118 :			end (* local *)
119 :	dbm	597	end

---

root@smlnj-gforge.cs.uchicago.edu	ViewVC Help
Powered by ViewVC 1.0.0