Home My Page Projects Code Snippets Project Openings SML/NJ
Summary Activity Forums Tracker Lists Tasks Docs Surveys News SCM Files

SCM Repository

[smlnj] Annotation of /sml/trunk/src/cm/compile/compile.sml
ViewVC logotype

Annotation of /sml/trunk/src/cm/compile/compile.sml

Parent Directory Parent Directory | Revision Log Revision Log


Revision 757 - (view) (download)

1 : blume 402 (*
2 :     * Compilation traversals.
3 :     *
4 :     * (C) 1999 Lucent Technologies, Bell Laboratories
5 :     *
6 :     * Author: Matthias Blume (blume@kurims.kyoto-u.ac.jp)
7 :     *)
8 : blume 398 local
9 :     structure GP = GeneralParams
10 :     structure DG = DependencyGraph
11 :     structure GG = GroupGraph
12 :     structure E = GenericVC.Environment
13 : blume 587 structure SE = GenericVC.StaticEnv
14 : blume 398 structure Pid = GenericVC.PersStamps
15 :     structure DE = GenericVC.DynamicEnv
16 :     structure PP = PrettyPrint
17 :     structure EM = GenericVC.ErrorMsg
18 : blume 757 structure SF = GenericVC.SmlFile
19 : blume 398
20 :     type pid = Pid.persstamp
21 :     type statenv = E.staticEnv
22 :     type symenv = E.symenv
23 :     type result = { stat: statenv, sym: symenv }
24 : blume 460 type ed = IInfo.info
25 : blume 398 in
26 :     signature COMPILE = sig
27 : blume 402
28 : blume 400 type bfc
29 :    
30 : blume 398 (* reset internal persistent state *)
31 :     val reset : unit -> unit
32 : blume 399
33 :     (* notify linkage module about recompilation *)
34 : blume 400 type notifier = GP.info -> SmlInfo.info -> unit
35 : blume 399
36 : blume 403 (* type of a function to store away the binfile contents *)
37 :     type bfcReceiver = SmlInfo.info * bfc -> unit
38 :    
39 : blume 398 val getII : SmlInfo.info -> IInfo.info
40 : blume 399
41 : blume 537 val evictStale : unit -> unit
42 : blume 402 val evictAll : unit -> unit
43 : blume 399
44 : blume 400 val newSbnodeTraversal : unit -> GP.info -> DG.sbnode -> ed option
45 :    
46 : blume 403 val newTraversal : notifier * bfcReceiver * GG.group ->
47 : blume 399 { group: GP.info -> result option,
48 :     exports: (GP.info -> result option) SymbolMap.map }
49 : blume 398 end
50 :    
51 : blume 448 functor CompileFn (structure MachDepVC : MACHDEP_VC
52 : blume 588 structure StabModmap : STAB_MODMAP
53 : blume 677 val useStream : TextIO.instream -> unit
54 : blume 666 val compile_there : SrcPath.file -> bool) :>
55 : blume 400 COMPILE where type bfc = MachDepVC.Binfile.bfContent =
56 :     struct
57 : blume 398
58 : blume 400 type notifier = GP.info -> SmlInfo.info -> unit
59 : blume 399
60 : blume 398 structure BF = MachDepVC.Binfile
61 :    
62 :     type bfc = BF.bfContent
63 :    
64 : blume 403 type bfcReceiver = SmlInfo.info * bfc -> unit
65 :    
66 : blume 447 structure FilterMap = MapFn
67 : blume 398 (struct
68 :     type ord_key = pid * SymbolSet.set
69 :     fun compare ((u, f), (u', f')) =
70 :     case Pid.compare (u, u') of
71 :     EQUAL => SymbolSet.compare (f, f')
72 :     | unequal => unequal
73 :     end)
74 :    
75 : blume 403 type bfinfo =
76 :     { cmdata: PidSet.set,
77 :     statenv: unit -> statenv,
78 :     symenv: unit -> symenv,
79 :     statpid: pid,
80 :     sympid: pid }
81 :    
82 : blume 399 type env = { envs: unit -> result, pids: PidSet.set }
83 : blume 460 type envdelta = IInfo.info
84 : blume 398
85 : blume 460 type memo = { ii: IInfo.info, ts: TStamp.t, cmdata: PidSet.set }
86 : blume 398
87 :     (* persistent state! *)
88 :     val filtermap = ref (FilterMap.empty: pid FilterMap.map)
89 :    
90 :     (* more persistent state! *)
91 : blume 402 val globalstate = ref (SmlInfoMap.empty: memo SmlInfoMap.map)
92 : blume 398
93 :     fun reset () =
94 :     (filtermap := FilterMap.empty;
95 : blume 402 globalstate := SmlInfoMap.empty)
96 : blume 398
97 :     fun isValidMemo (memo: memo, provided, smlinfo) =
98 :     not (TStamp.needsUpdate { source = SmlInfo.lastseen smlinfo,
99 :     target = #ts memo })
100 : blume 403 andalso PidSet.equal (provided, #cmdata memo)
101 : blume 398
102 : blume 587 fun memo2ii (memo: memo) = #ii memo
103 : blume 398
104 : blume 460 fun memo2ed memo = memo2ii memo
105 : blume 398
106 : blume 460 fun bfc2memo (bfc, ts) = let
107 : blume 403 val ii = { statenv = fn () => BF.senvOf bfc,
108 :     symenv = fn () => BF.symenvOf bfc,
109 :     statpid = BF.staticPidOf bfc,
110 :     sympid = BF.lambdaPidOf bfc }
111 :     val cmdata = PidSet.addList (PidSet.empty, BF.cmDataOf bfc)
112 :     in
113 : blume 460 { ii = ii, ts = ts, cmdata = cmdata }
114 : blume 403 end
115 :    
116 : blume 398 fun pidset (p1, p2) = PidSet.add (PidSet.singleton p1, p2)
117 :    
118 :     fun nofilter (ed: envdelta) = let
119 : blume 460 val { statenv, symenv, statpid, sympid } = ed
120 : blume 587 val statenv' = Memoize.memoize statenv
121 : blume 398 in
122 : blume 587 { envs = fn () => { stat = statenv' (), sym = symenv () },
123 : blume 398 pids = pidset (statpid, sympid) }
124 :     end
125 :    
126 : blume 735 fun requiredFiltering set se = let
127 :     val dom = SymbolSet.addList (SymbolSet.empty, E.catalogEnv se)
128 :     val filt = SymbolSet.intersection (set, dom)
129 :     in
130 :     if SymbolSet.equal (dom, filt) then NONE
131 :     else SOME filt
132 :     end
133 : blume 398
134 : blume 460 fun filter (ii, s) = let
135 : blume 398 val { statenv, symenv, statpid, sympid } = ii
136 : blume 587 val ste = statenv ()
137 : blume 398 in
138 : blume 735 case requiredFiltering s ste of
139 :     NONE => { envs = fn () => { stat = ste, sym = symenv () },
140 :     pids = pidset (statpid, sympid) }
141 :     | SOME s => let
142 : blume 587 val ste' = E.filterStaticEnv (ste, SymbolSet.listItems s)
143 :     val key = (statpid, s)
144 :     val statpid' =
145 :     case FilterMap.find (!filtermap, key) of
146 :     SOME statpid' => statpid'
147 :     | NONE => let
148 :     val statpid' = GenericVC.Rehash.rehash
149 :     { env = ste', orig_hash = statpid }
150 :     in
151 :     filtermap :=
152 :     FilterMap.insert (!filtermap, key, statpid');
153 :     statpid'
154 :     end
155 :     in
156 :     { envs = fn () => { stat = ste', sym = symenv () },
157 :     pids = pidset (statpid', sympid) }
158 :     end
159 : blume 398 end
160 :    
161 : blume 399 local
162 : blume 398 fun r2e { stat, sym } = E.mkenv { static = stat, symbolic = sym,
163 :     dynamic = DE.empty }
164 :     fun e2r e = { stat = E.staticPart e, sym = E.symbolicPart e }
165 :     in
166 : blume 399 (* This is a bit ugly because somehow we need to mix dummy
167 :     * dynamic envs into the equation just to be able to use
168 :     * concatEnv. But, alas', that's life... *)
169 :     fun rlayer (r, r') = e2r (E.concatEnv (r2e r, r2e r'))
170 :    
171 :     val emptyEnv =
172 :     { envs = fn () => e2r E.emptyEnv, pids = PidSet.empty }
173 : blume 398 end
174 :    
175 :     fun layer ({ envs = e, pids = p }, { envs = e', pids = p' }) =
176 :     { envs = fn () => rlayer (e (), e' ()),
177 :     pids = PidSet.union (p, p') }
178 :    
179 : blume 462 (* I would rather not use an exception here, but short of a better
180 :     * implementation of concurrency I see no choice.
181 :     * The problem is that at each node we sequentiallay wait for the
182 :     * children nodes. But the scheduler might (and probably will)
183 :     * let a child run that we are not currently waiting for, so an
184 :     * error there will not result in "wait" to immediately return
185 :     * as it should for clean error recovery.
186 :     * Using the exception avoids having to implement a
187 :     * "wait for any child -- whichever finishes first" kind of call. *)
188 :     exception Abort
189 : blume 399
190 : blume 462 fun layer'wait u (p, NONE) =
191 :     (ignore (Concur.waitU u p); NONE)
192 :     | layer'wait u (p, SOME e) =
193 :     (case Concur.waitU u p of
194 :     SOME e' => SOME (layer (e', e))
195 :     | NONE => NONE)
196 :    
197 : blume 454 fun mkTraversal (notify, storeBFC, getUrgency) = let
198 : blume 402 val localstate = ref SmlInfoMap.empty
199 : blume 398
200 : blume 537 fun sbnode gp (DG.SB_SNODE n) = snode gp n
201 :     (* The beauty of this scheme is that we don't have
202 :     * to do anything at all for SB_BNODEs: Everything
203 :     * is prepared ready to be used when the library
204 :     * is unpickled: *)
205 : blume 737 | sbnode gp (DG.SB_BNODE (_, ii, _)) = SOME ii
206 : blume 398
207 :     and fsbnode gp (f, n) =
208 :     case (sbnode gp n, f) of
209 :     (NONE, _) => NONE
210 :     | (SOME d, NONE) => SOME (nofilter d)
211 :     | (SOME d, SOME s) => SOME (filter (d, s))
212 :    
213 :     and snode gp (DG.SNODE n) = let
214 : blume 692 val youngest = #youngest gp
215 : blume 398 val { smlinfo = i, localimports = li, globalimports = gi } = n
216 :     val binname = SmlInfo.binname i
217 :    
218 : blume 462 fun fail () =
219 :     if #keep_going (#param gp) then NONE else raise Abort
220 :    
221 : blume 537 fun compile_here (stat, sym, pids, split) = let
222 : blume 677 fun perform_setup _ NONE = ()
223 :     | perform_setup what (SOME code) =
224 :     (Say.vsay ["[setup (", what, "): ", code, "]\n"];
225 :     SafeIO.perform
226 :     { openIt = fn () => TextIO.openString code,
227 :     closeIt = TextIO.closeIn,
228 :     work = useStream,
229 :     cleanup = fn _ => () })
230 : blume 398 fun save bfc = let
231 : blume 757 fun writer s = let
232 :     val sz = BF.write { stream = s, content = bfc,
233 :     nopickle = false }
234 :     fun info ((sel, lab), (l, t)) =
235 :     case sel sz of
236 :     0 => (l, t)
237 :     | n => (lab :: ": " :: Int.toString n ::
238 :     t :: " " :: l,
239 :     ",")
240 :     in
241 :     Say.vsay ("[" ::
242 :     #1 (foldr info
243 :     (["(bytes)]\n"], " ")
244 :     [(#code, "code"),
245 :     (#data, "data"),
246 :     (#env, "env"),
247 :     (#inlinfo, "inlinable")]))
248 :     end
249 : blume 459 fun cleanup _ =
250 : blume 398 OS.FileSys.remove binname handle _ => ()
251 :     in
252 : blume 400 notify gp i;
253 : blume 398 SafeIO.perform { openIt =
254 :     fn () => AutoDir.openBinOut binname,
255 :     closeIt = BinIO.closeOut,
256 :     work = writer,
257 :     cleanup = cleanup }
258 :     handle exn => let
259 :     fun ppb pps =
260 :     (PP.add_newline pps;
261 :     PP.add_string pps (General.exnMessage exn))
262 :     in
263 :     SmlInfo.error gp i EM.WARN
264 :     ("failed to write " ^ binname) ppb
265 :     end;
266 :     TStamp.setTime (binname, SmlInfo.lastseen i)
267 :     end (* save *)
268 :     in
269 :     case SmlInfo.parsetree gp i of
270 : blume 462 NONE => fail ()
271 : blume 398 | SOME (ast, source) => let
272 : blume 592 val ast =
273 :     case #explicit_core_sym (SmlInfo.attribs i) of
274 :     NONE => ast
275 :     | SOME sy => CoreHack.rewrite (ast, sy)
276 : blume 398 val cmData = PidSet.listItems pids
277 : blume 677 val (pre, post) = SmlInfo.setup i
278 :     val toplenv = #get GenericVC.EnvRef.topLevel ()
279 :     before perform_setup "pre" pre
280 : blume 398 (* clear error flag (could still be set from
281 :     * earlier run) *)
282 :     val _ = #anyErrors source := false
283 : blume 537 val bfc = BF.create
284 : blume 587 { splitting = split,
285 : blume 537 cmData = cmData,
286 :     ast = ast,
287 :     source = source,
288 :     senv = stat,
289 : blume 592 symenv = sym }
290 : blume 460 val memo = bfc2memo (bfc, SmlInfo.lastseen i)
291 : blume 398 in
292 : blume 677 perform_setup "post" post;
293 :     #set GenericVC.EnvRef.topLevel toplenv;
294 : blume 398 save bfc;
295 : blume 403 storeBFC (i, bfc);
296 : blume 402 SOME memo
297 : blume 757 end handle (EM.Error | SF.Compile _)
298 :     (* At this point we handle only
299 :     * explicit compiler bugs and ordinary
300 :     * compilation errors because for those
301 :     * there will already have been
302 :     * explanatory messages. Everything
303 :     * else "falls through" and will be
304 :     * treated at top level. *)
305 :     => fail ()
306 : blume 448 end (* compile_here *)
307 : blume 398 fun notlocal () = let
308 : blume 692 val _ = youngest := TStamp.max (!youngest,
309 :     SmlInfo.lastseen i)
310 : blume 454 val urgency = getUrgency i
311 : blume 402 (* Ok, it is not in the local state, so we first have
312 : blume 398 * to traverse all children before we can proceed... *)
313 :     fun loc li_n = Option.map nofilter (snode gp li_n)
314 :     fun glob gi_n = fsbnode gp gi_n
315 : blume 448 val gi_cl =
316 :     map (fn gi_n => Concur.fork (fn () => glob gi_n)) gi
317 :     val li_cl =
318 :     map (fn li_n => Concur.fork (fn () => loc li_n)) li
319 : blume 398 val e =
320 : blume 462 foldl (layer'wait urgency)
321 :     (foldl (layer'wait urgency)
322 : blume 537 (SOME emptyEnv)
323 : blume 462 gi_cl)
324 :     li_cl
325 : blume 398 in
326 :     case e of
327 :     NONE => NONE
328 :     | SOME { envs, pids } => let
329 :     (* We have successfully traversed all
330 :     * children. Now it is time to check the
331 :     * global map... *)
332 :     fun fromfile () = let
333 :     val { stat, sym } = envs ()
334 : blume 537 val { split, extra_compenv, ... } =
335 :     SmlInfo.attribs i
336 :     val stat =
337 :     case extra_compenv of
338 :     NONE => stat
339 :     | SOME s => E.layerStatic (stat, s)
340 : blume 398 fun load () = let
341 :     val ts = TStamp.fmodTime binname
342 :     fun openIt () = BinIO.openIn binname
343 : blume 588 fun reader s = let
344 :     val mm0 = StabModmap.get ()
345 :     val m = GenModIdMap.mkMap' (stat, mm0)
346 :     in
347 : blume 398 (BF.read { stream = s,
348 :     name = binname,
349 : blume 588 modmap = m },
350 : blume 398 ts)
351 : blume 588 end
352 : blume 403
353 : blume 398 in
354 :     SOME (SafeIO.perform
355 :     { openIt = openIt,
356 :     closeIt = BinIO.closeIn,
357 :     work = reader,
358 : blume 459 cleanup = fn _ => () })
359 : blume 398 handle _ => NONE
360 :     end (* load *)
361 : blume 448 fun tryload (what, otherwise) =
362 :     case load () of
363 :     NONE => otherwise ()
364 :     | SOME (bfc, ts) => let
365 : blume 460 val memo = bfc2memo (bfc, ts)
366 : blume 448 in
367 :     if isValidMemo (memo, pids, i) then
368 :     (Say.vsay ["[", binname,
369 :     " ", what, "]\n"];
370 :     storeBFC (i, bfc);
371 :     SOME memo)
372 :     else otherwise ()
373 :     end
374 : blume 632 fun bottleneck () =
375 :     (* Are we the only runable task? *)
376 :     Servers.allIdle () andalso
377 :     Concur.noTasks ()
378 : blume 448 fun compile_again () =
379 : blume 452 (Say.vsay ["[compiling ",
380 :     SmlInfo.descr i, "]\n"];
381 : blume 537 compile_here (stat, sym, pids, split))
382 : blume 632 fun compile_there' p =
383 :     not (bottleneck ()) andalso
384 :     compile_there p
385 : blume 448 fun compile () = let
386 :     val sp = SmlInfo.sourcepath i
387 :     in
388 : blume 692 youngest := TStamp.NOTSTAMP;
389 : blume 632 if compile_there' sp then
390 : blume 450 tryload ("received", compile_again)
391 : blume 448 else compile_again ()
392 :     end
393 : blume 398 in
394 : blume 448 (* If anything goes wrong loading the first
395 :     * time, we go and compile. Compiling
396 :     * may mean compiling externally, and if so,
397 :     * we must load the result of that.
398 :     * If the second load also goes wrong, we
399 :     * compile locally to gather error messages
400 :     * and make everything look "normal". *)
401 :     tryload ("loaded", compile)
402 : blume 398 end (* fromfile *)
403 : blume 402 fun notglobal () =
404 :     case fromfile () of
405 :     NONE => NONE
406 :     | SOME memo =>
407 :     (globalstate :=
408 :     SmlInfoMap.insert (!globalstate, i,
409 :     memo);
410 :     SOME memo)
411 : blume 398 in
412 : blume 402 case SmlInfoMap.find (!globalstate, i) of
413 :     NONE => notglobal ()
414 : blume 398 | SOME memo =>
415 :     if isValidMemo (memo, pids, i) then
416 : blume 402 SOME memo
417 :     else notglobal ()
418 : blume 398 end
419 :     end (* notlocal *)
420 :     in
421 : blume 462 (* Here we just wait (no "waitU") so we don't get
422 :     * priority over threads that may have to clean up after
423 :     * errors. *)
424 : blume 402 case SmlInfoMap.find (!localstate, i) of
425 : blume 448 SOME mopt_c => Option.map memo2ed (Concur.wait mopt_c)
426 : blume 398 | NONE => let
427 : blume 448 val mopt_c = Concur.fork
428 :     (fn () => notlocal () before
429 :     (* "Not local" means that we have not processed
430 :     * this file before. Therefore, we should now
431 :     * remove its parse tree... *)
432 :     SmlInfo.forgetParsetree i)
433 : blume 398 in
434 : blume 402 localstate :=
435 : blume 632 SmlInfoMap.insert (!localstate, i, mopt_c);
436 : blume 448 Option.map memo2ed (Concur.wait mopt_c)
437 : blume 398 end
438 :     end (* snode *)
439 :    
440 : blume 652 fun impexp gp (nth, _, _) = fsbnode gp (nth ())
441 : blume 399 in
442 :     { sbnode = sbnode, impexp = impexp }
443 :     end
444 : blume 398
445 : blume 587 fun newTraversal (_, _, GG.ERRORGROUP) =
446 :     { group = fn _ => NONE, exports = SymbolMap.empty }
447 :     | newTraversal (notify, storeBFC, g as GG.GROUP grec) = let
448 :     val { exports, ... } = grec
449 : blume 652 val um = Memoize.memoize (fn () => Indegree.indegrees g)
450 :     fun getUrgency i = getOpt (SmlInfoMap.find (um (), i), 0)
451 :     (* generate the traversal -- lazily *)
452 :     val impexpth =
453 :     Memoize.memoize
454 :     (fn () =>
455 :     #impexp
456 :     (mkTraversal (notify, storeBFC, getUrgency)))
457 : blume 587 fun group gp = let
458 :     val eo_cl =
459 : blume 652 map (fn x => Concur.fork (fn () => impexpth () gp x))
460 : blume 587 (SymbolMap.listItems exports)
461 :     val eo = foldl (layer'wait 0) (SOME emptyEnv) eo_cl
462 :     in
463 :     case eo of
464 :     NONE => (Servers.reset false; NONE)
465 :     | SOME e => SOME (#envs e ())
466 :     end handle Abort => (Servers.reset false; NONE)
467 :     fun mkExport ie gp =
468 : blume 652 case impexpth () gp ie handle Abort => NONE of
469 : blume 587 NONE => (Servers.reset false; NONE)
470 :     | SOME e => SOME (#envs e ())
471 : blume 399 in
472 : blume 587 { group = group,
473 :     exports = SymbolMap.map mkExport exports }
474 :     end
475 : blume 398
476 : blume 400 fun newSbnodeTraversal () = let
477 : blume 537 val { sbnode, ... } =
478 :     mkTraversal (fn _ => fn _ => (), fn _ => (), fn _ => 0)
479 : blume 461 fun sbn_trav gp g = let
480 : blume 462 val r = sbnode gp g handle Abort => NONE
481 : blume 461 in
482 :     if isSome r then () else Servers.reset false;
483 :     r
484 :     end
485 : blume 398 in
486 : blume 461 sbn_trav
487 : blume 398 end
488 :    
489 : blume 537 fun evictStale () =
490 :     globalstate :=
491 :     SmlInfoMap.filteri (SmlInfo.isKnown o #1) (!globalstate)
492 : blume 400
493 : blume 403 fun evictAll () = globalstate := SmlInfoMap.empty
494 : blume 402
495 : blume 403 fun getII i = memo2ii (valOf (SmlInfoMap.find (!globalstate, i)))
496 : blume 398 end
497 :     end

root@smlnj-gforge.cs.uchicago.edu
ViewVC Help
Powered by ViewVC 1.0.0