Home My Page Projects Code Snippets Project Openings diderot
Summary Activity Tracker Tasks SCM

SCM Repository

[diderot] View of /trunk/src/parser/diderot.lex
ViewVC logotype

View of /trunk/src/parser/diderot.lex

Parent Directory Parent Directory | Revision Log Revision Log


Revision 26 - (download) (annotate)
Tue Feb 9 00:43:01 2010 UTC (9 years, 8 months ago) by jhr
File size: 3044 byte(s)
  Working on Diderot parser
(* diderot.lex
 *
 * COPYRIGHT (c) 2010 The Diderot Project (http://diderot.cs.uchicago.edu)
 * All rights reserved.
 *)

%name DiderotLex;

%arg (lexErr);

%defs(

    structure T = ManticoreTokens

  (* some type lex_result is necessitated by ml-ulex *)
    type lex_result = T.token

  (* the depth int ref will be used for keeping track of comment depth *)
    val depth = ref 0

  (* list of string fragments to concatenate *)
    val buf : string list ref = ref []

  (* add a string to the buffer *)
    fun addStr s = (buf := s :: !buf)

  (* make a string from buf *)
    fun mkString () = let
	  val s = String.concat(List.rev(!buf))
	  in
	    buf := [];
	    T.STRING s
	  end

  (* make a FLOAT token from a substring *)
    fun mkFloat ss = let
	  val (isNeg, rest) = (case Substring.getc ss
		 of SOME(#"-", r) => (true, r)
		  | SOME(#"+", r) => (false, r)
		  | _ => (false, ss)
		(* end case *))
	  val (whole, rest) = Substring.splitl Char.isDigit rest
	  val rest = Substring.triml 1 rest (* remove "." *)
	  val (frac, rest) = Substring.splitl Char.isDigit rest
	  val exp = if Substring.isEmpty rest
		then 0
		else let
		  val rest = Substring.triml 1 rest (* remove "e" or "E" *)
		  in
		    #1(valOf(Int.scan StringCvt.DEC Substring.getc rest))
		  end
	  in
	    T.FLOAT(FloatLit.float{
		isNeg = isNeg,
		whole = Substring.string whole,
		frac = Substring.string frac,
		exp = exp
	      })
	  end

  (* scan a number from a hexidecimal string *)
    val fromHexString = valOf o (StringCvt.scanString (IntInf.scan StringCvt.HEX))
(* FIXME: the above code doesn't work in SML/NJ; here is a work around *)
fun fromHexString s = let
      val SOME(n, _) = IntInf.scan StringCvt.HEX Substring.getc
	    (Substring.triml 2 (Substring.full s))
      in
	n
      end

  (* eof : unit -> lex_result *)
  (* ml-ulex requires this as well *)
    fun eof () = T.EOF
);

%states INITIAL STRING COM1 COM2;

%let letter = [a-zA-Z];
%let dig = [0-9];
%let num = {dig}+;
%let hexdigit = [0-9a-fA-F];
%let hexnum = "0x"{hexdigit}+;
%let idchar = {letter}|{dig}|"_"|"'";
%let id = {letter}{idchar}*;
%let ws = " "|[\t\n\v\f\r];

(***** Keywords and operators *****)

<INITIAL> "("		=> (T.LP);
<INITIAL> ")"		=> (T.RP);
<INITIAL> "["		=> (T.LB);
<INITIAL> "]"		=> (T.RB);
<INITIAL> "{"		=> (T.LCB);
<INITIAL> "}"		=> (T.RCB);
<INITIAL> "||"		=> (T.OP_orelse);
<INITIAL> "&&"		=> (T.OP_andalso);
<INITIAL> "<"		=> (T.OP_lt);
<INITIAL> "<="		=> (T.OP_lte);
<INITIAL> "=="		=> (T.OP_eqeq);
<INITIAL> "!="		=> (T.OP_neq);
<INITIAL> ">="		=> (T.OP_gte);
<INITIAL> ">"		=> (T.OP_gt);
<INITIAL> "+"		=> (T.OP_plus);
<INITIAL> "-"		=> (T.OP_minus);
<INITIAL> "*"		=> (T.OP_star);
<INITIAL> "@"		=> (T.OP_at);
<INITIAL> ","		=> (T.COMMA);
<INITIAL> ";"		=> (T.SEMI);
<INITIAL> "#"		=> (T.HASH);
<INITIAL> "!"		=> (T.BANG);

(***** Comments *****)
<INITIAL> "//"
	=> (YYBEGIN COM1; skip());
<COM1> {eol}
	=> (YYBEGIN INITIAL; skip());

<INITIAL> "/*"
	=> (YYBEGIN COM2; skip());
<COM2> "*/"
	=> (YYBEGIN INITIAL; skip());
<COM2> .
	=> (skip());

root@smlnj-gforge.cs.uchicago.edu
ViewVC Help
Powered by ViewVC 1.0.0