1 // Copyright 2006-2007 all rights reserved; see LICENSE file for BSD-style license
3 package edu.berkeley.sbp.meta;
4 import edu.berkeley.sbp.util.*;
5 import edu.berkeley.sbp.*;
6 import edu.berkeley.sbp.chr.*;
7 import edu.berkeley.sbp.misc.*;
9 import java.lang.annotation.*;
10 import java.lang.reflect.*;
14 * The inner classes of this class represent nodes in the Abstract
15 * Syntax Tree of a grammar.
17 public class GrammarAST {
19 public static interface ImportResolver {
20 public InputStream getImportStream(String importname);
24 * Returns a Union representing the metagrammar (<tt>meta.g</tt>); the Tree produced by
25 * parsing with this Union should be provided to <tt>buildFromAST</tt>
27 public static Union getMetaGrammar() {
28 return buildFromAST(MetaGrammar.meta, "s", null);
32 * Create a grammar from a parse tree and binding resolver
34 * @param t a tree produced by parsing a grammar using the metagrammar
35 * @param s the name of the "start symbol"
36 * @param gbr a GrammarBindingResolver that resolves grammatical reductions into tree-node-heads
38 public static Union buildFromAST(Tree grammarAST, String startingNonterminal, ImportResolver resolver) {
39 return new GrammarAST(resolver, "").buildGrammar(grammarAST, startingNonterminal);
42 private static Object illegalTag = ""; // this is the tag that should never appear in the non-dropped output FIXME
44 // Instance //////////////////////////////////////////////////////////////////////////////
46 private final String prefix;
47 private final ImportResolver resolver;
49 public GrammarAST(ImportResolver resolver, String prefix) {
51 this.resolver = resolver;
54 // Methods //////////////////////////////////////////////////////////////////////////////
56 private Union buildGrammar(Tree t, String rootNonTerminal) {
58 if (o instanceof Union) return (Union)o;
59 return ((GrammarAST.GrammarNode)o).build(rootNonTerminal);
62 private Object[] walkChildren(Tree t) {
63 Object[] ret = new Object[t.size()];
64 for(int i=0; i<ret.length; i++)
65 ret[i] = walk(t.child(i));
66 return Reflection.lub(ret);
68 private static String stringifyChildren(Tree t) {
69 StringBuffer sb = new StringBuffer();
70 for(int i=0; i<t.size(); i++) {
71 sb.append(t.child(i).head());
72 sb.append(stringifyChildren(t.child(i)));
76 private static String unescape(Tree t) {
77 StringBuffer sb = new StringBuffer();
78 for(int i=0; i<t.size(); i++)
79 sb.append(t.child(i).head()+stringifyChildren(t.child(i)));
83 private ElementNode walkElement(Tree t) { return (ElementNode)walk(t); }
84 private String walkString(Tree t) { return (String)walk(t); }
85 private Seq walkSeq(Tree t) { return (Seq)walk(t); }
86 private Object walk(Tree t) {
87 String head = (String)t.head();
88 while(head.indexOf('.') > 0)
89 head = head.substring(head.indexOf('.')+1);
90 if (head==null) throw new RuntimeException("head is null: " + t);
91 if (head.equals("|")) return walkChildren(t);
92 if (head.equals("RHS")) return walkChildren(t);
93 if (head.equals("Grammar")) return new GrammarNode(walkChildren(t));
94 if (head.equals("(")) return new UnionNode((Seq[][])walkChildren(t.child(0)));
95 if (head.equals("Word")) return stringifyChildren(t);
96 if (head.equals("Elements")) return new Seq((ElementNode[])Reflection.rebuild(walkChildren(t), ElementNode[].class));
97 if (head.equals("NonTerminalReference")) return new ReferenceNode(stringifyChildren(t.child(0)));
98 if (head.equals(")")) return new ReferenceNode(stringifyChildren(t.child(0)), true);
99 if (head.equals("::")) return walkSeq(t.child(1)).tag(walkString(t.child(0)));
100 if (head.equals("...")) return new DropNode(new RepeatNode(new TildeNode(new AtomNode()), null, true, true, false));
102 if (head.equals("++")) return new RepeatNode(walkElement(t.child(0)), null, false, true, true);
103 if (head.equals("+")) return new RepeatNode(walkElement(t.child(0)), null, false, true, false);
104 if (head.equals("++/")) return new RepeatNode(walkElement(t.child(0)), walkElement(t.child(1)), false, true, true);
105 if (head.equals("+/")) return new RepeatNode(walkElement(t.child(0)), walkElement(t.child(1)), false, true, false);
106 if (head.equals("**")) return new RepeatNode(walkElement(t.child(0)), null, true, true, true);
107 if (head.equals("*")) return new RepeatNode(walkElement(t.child(0)), null, true, true, false);
108 if (head.equals("**/")) return new RepeatNode(walkElement(t.child(0)), walkElement(t.child(1)), true, true, true);
109 if (head.equals("*/")) return new RepeatNode(walkElement(t.child(0)), walkElement(t.child(1)), true, true, false);
110 if (head.equals("?")) return new RepeatNode(walkElement(t.child(0)), null, true, false, false);
112 if (head.equals("!")) return new DropNode(walkElement(t.child(0)));
113 if (head.equals("^")) return new LiteralNode(walkString(t.child(0)), true);
114 if (head.equals("`")) return new BacktickNode(walkElement(t.child(0)));
115 if (head.equals("Quoted")) return stringifyChildren(t);
116 if (head.equals("Literal")) return new LiteralNode(walkString(t.child(0)));
117 if (head.equals("->")) return walkSeq(t.child(0)).follow(walkElement(t.child(1)));
118 if (head.equals("DropNT")) return new NonTerminalNode(walkString(t.child(0)), (Seq[][])walkChildren(t.child(1)), false, null, true);
119 if (head.equals("=")) return new NonTerminalNode(walkString(t.child(0)), (Seq[][])walk(t.child(2)),
120 true, t.size()==2 ? null : walkString(t.child(1)), false);
121 if (head.equals("&")) return walkSeq(t.child(0)).and(walkSeq(t.child(1)));
122 if (head.equals("&~")) return walkSeq(t.child(0)).andnot(walkSeq(t.child(1)));
123 if (head.equals("/")) return (walkSeq(t.child(0))).separate(walkElement(t.child(1)));
124 if (head.equals("()")) return new LiteralNode("");
125 if (head.equals("[")) return new AtomNode((char[][])Reflection.rebuild(walkChildren(t), char[][].class));
126 if (head.equals("\\{")) return new DropNode(new AtomNode(new char[] { CharAtom.left, CharAtom.left }));
127 if (head.equals("\\}")) return new DropNode(new AtomNode(new char[] { CharAtom.right, CharAtom.right }));
128 if (head.equals(">>")) return new DropNode(new AtomNode(new char[] { CharAtom.left, CharAtom.left }));
129 if (head.equals("<<")) return new DropNode(new AtomNode(new char[] { CharAtom.right, CharAtom.right }));
130 if (head.equals("~")) return new TildeNode(walkElement(t.child(0)));
131 if (head.equals("~~")) return new Seq(new RepeatNode(new TildeNode(new AtomNode()), null, true, true, false)).andnot(walkSeq(t.child(0)));
132 if (head.equals("Range")) {
133 if (t.size()==2 && ">".equals(t.child(0).head())) return new char[] { CharAtom.left, CharAtom.left };
134 if (t.size()==2 && "<".equals(t.child(0).head())) return new char[] { CharAtom.right, CharAtom.right };
135 if (t.size()==1) return new char[] { unescape(t).charAt(0), unescape(t).charAt(0) };
136 return new char[] { unescape(t).charAt(0), unescape(t).charAt(1) };
138 if (head.equals("\"\"")) return "";
139 if (head.equals("\n")) return "\n";
140 if (head.equals("\t")) return "\t";
141 if (head.equals("\r")) return "\r";
142 if (head.equals("SubGrammar")) return GrammarAST.buildFromAST(t.child(0), "s", resolver);
143 if (head.equals("NonTerminal"))
144 return new NonTerminalNode(walkString(t.child(0)),
145 (Seq[][])walkChildren(t.child(1)), false, null, false);
146 if (head.equals("Colons")) {
147 String tag = walkString(t.child(0));
148 Seq[][] seqs = (Seq[][])walk(t.child(1));
149 for(Seq[] seq : seqs)
150 for(int i=0; i<seq.length; i++)
151 seq[i] = seq[i].tag(tag);
152 return new NonTerminalNode(tag, seqs, false, null, false);
154 if (head.equals("#import")) {
155 if (resolver != null) {
156 String fileName = (String)stringifyChildren(t.child(0));
158 String newPrefix = t.size()<2 ? "" : (walkString(t.child(1))+".");
159 InputStream fis = resolver.getImportStream(fileName);
161 throw new RuntimeException("unable to find #include file \""+fileName+"\"");
162 Tree tr = new CharParser(getMetaGrammar()).parse(fis).expand1();
163 return (GrammarNode)new GrammarAST(resolver, newPrefix).walk(tr);
164 } catch (Exception e) {
165 throw new RuntimeException("while parsing " + fileName, e);
168 throw new RuntimeException("no resolver given");
171 throw new RuntimeException("unknown head: \"" + head + "\" => " + (head.equals("...")));
175 // Nodes //////////////////////////////////////////////////////////////////////////////
177 /** Root node of a grammar's AST; a set of named nonterminals */
178 private class GrammarNode extends HashMap<String,NonTerminalNode> {
179 public GrammarNode(NonTerminalNode[] nonterminals) {
180 for(NonTerminalNode nt : nonterminals) {
181 if (nt==null) continue;
182 if (this.get(nt.name)!=null)
183 throw new RuntimeException("duplicate definition of nonterminal \""+nt.name+"\"");
184 this.put(nt.name, nt);
187 public GrammarNode(Object[] nt) { add(nt); }
188 private void add(Object o) {
190 else if (o instanceof Object[]) for(Object o2 : (Object[])o) add(o2);
191 else if (o instanceof NonTerminalNode) {
192 NonTerminalNode nt = (NonTerminalNode)o;
193 if (this.get(nt.name)!=null)
194 throw new RuntimeException("duplicate definition of nonterminal \""+nt.name+"\"");
195 this.put(nt.name, nt);
197 else if (o instanceof GrammarNode)
198 for(NonTerminalNode n : ((GrammarNode)o).values())
201 public String toString() {
203 for(NonTerminalNode nt : values()) ret += nt + ", ";
206 public Union build(String rootNonterminal) {
207 Context cx = new Context(this);
209 for(GrammarAST.NonTerminalNode nt : values())
210 if (nt.name.equals(rootNonterminal))
211 return (Union)cx.get(nt.name);
216 /** a node in the AST which is resolved into an Element */
217 private abstract class ElementNode {
218 public boolean isLifted() { return false; }
219 public boolean isDropped(Context cx) { return false; }
220 public Atom toAtom(Context cx) { throw new Error("can't convert a " + this.getClass().getName() + " to an atom: " + this); }
221 public abstract Element build(Context cx, NonTerminalNode cnt, boolean dropall);
224 private class UnionNode extends ElementNode {
225 public Seq[][] sequences;
226 public String sep = null;
228 public UnionNode(Seq seq) { this(new Seq[][] { new Seq[] { seq } }); }
229 public UnionNode(Seq[][] sequences) { this(sequences, false, null); }
230 public UnionNode(Seq[][] sequences, boolean rep, String sep) {
231 this.sequences = sequences;
235 public boolean isDropped(Context cx) {
236 for(Seq[] seqs : sequences)
238 if (!seq.isDropped(cx))
242 public Atom toAtom(Context cx) {
244 for(Seq[] ss : sequences)
246 ret = ret==null ? s.toAtom(cx) : (Atom)ret.union(s.toAtom(cx));
249 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) {
250 return buildIntoPreallocatedUnion(cx, cnt, dropall, new Union(null, false)); }
251 public Element buildIntoPreallocatedUnion(Context cx, NonTerminalNode cnt, boolean dropall, Union u) {
254 urep = new Union(null, false);
255 urep.add(Sequence.create(cnt.name, new Element[0]));
257 ? Sequence.create(new Element[] { u }, 0)
258 : Sequence.create(new Element[] { cx.get(sep), u }, 1));
260 HashSet<Sequence> bad2 = new HashSet<Sequence>();
261 for(int i=0; i<sequences.length; i++) {
262 Seq[] group = sequences[i];
263 Union u2 = new Union(null, false);
264 if (sequences.length==1) u2 = u;
265 for(int j=0; j<group.length; j++)
267 group[j].build(cx, u2, cnt, dropall);
269 Union u3 = new Union(null, false);
270 group[j].build(cx, u3, cnt, dropall);
271 Sequence s = Sequence.create(cnt.name,
272 new Element[] { u3, urep },
273 new boolean[] { false, false },
274 new boolean[] { false, true});
277 if (sequences.length==1) break;
278 Sequence seq = Sequence.create(u2);
279 for(Sequence s : bad2) seq = seq.andnot(s);
281 bad2.add(Sequence.create(u2));
287 private class NonTerminalNode extends UnionNode {
288 public boolean alwaysDrop;
289 public String name = null;
290 public boolean isDropped(Context cx) { return alwaysDrop; }
291 public NonTerminalNode(String name, Seq[][] sequences, boolean rep, String sep, boolean alwaysDrop) {
292 super(sequences, rep, sep==null?null:(prefix + sep));
293 this.name = prefix + name;
294 this.alwaysDrop = alwaysDrop;
296 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) { return cx.get(name); }
300 public boolean alwaysDrop = false;
301 public boolean isDropped(Context cx) {
302 if (alwaysDrop) return true;
303 if (tag!=null) return false;
304 for(int i=0; i<elements.length; i++)
305 if (!elements[i].isDropped(cx) || ((elements[i] instanceof LiteralNode) && ((LiteralNode)elements[i]).caret))
309 HashSet<Seq> and = new HashSet<Seq>();
310 HashSet<Seq> not = new HashSet<Seq>();
311 ElementNode[] elements;
314 public Seq(ElementNode e) { this(new ElementNode[] { e }); }
315 public Seq(ElementNode[] elements) { this(elements, true); }
316 public Seq(ElementNode[] el, boolean check) {
317 this.elements = new ElementNode[el.length];
318 System.arraycopy(el, 0, elements, 0, el.length);
319 for(int i=0; i<elements.length; i++) {
320 if (elements[i]==null)
321 throw new RuntimeException();
323 // FIXME: this whole mechanism is sketchy
325 for(int i=0; i<elements.length; i++) {
326 if ((elements[i] instanceof ReferenceNode) && ((ReferenceNode)elements[i]).parenthesized) {
327 ReferenceNode rn = (ReferenceNode)elements[i];
328 ElementNode replace = null;
329 for(int j=0; j<elements.length; j++) {
330 if (!(elements[j] instanceof ReferenceNode)) continue;
331 ReferenceNode rn2 = (ReferenceNode)elements[j];
332 if (rn2.nonTerminal.equals(rn.nonTerminal) && !rn2.parenthesized) {
333 if (replace == null) {
334 replace = new UnionNode(new Seq(rn2).andnot(new Seq(elements, false)));
336 elements[j] = replace;
342 public Atom toAtom(Context cx) {
343 if (elements.length != 1)
344 throw new Error("you attempted to use ->, **, ++, or a similar character-class"+
345 " operator on a [potentially] multicharacter production");
346 return elements[0].toAtom(cx);
348 public Seq tag(String tag) { this.tag = tag; return this; }
349 public Seq follow(ElementNode follow) { this.follow = follow; return this; }
350 public Seq and(Seq s) { and.add(s); s.alwaysDrop = true; return this; }
351 public Seq andnot(Seq s) { not.add(s); s.alwaysDrop = true; return this; }
352 public Seq separate(ElementNode sep) {
353 ElementNode[] elements = new ElementNode[this.elements.length * 2 - 1];
354 for(int i=0; i<this.elements.length; i++) {
355 elements[i*2] = this.elements[i];
356 if (i<this.elements.length-1)
357 elements[i*2+1] = new DropNode(sep);
359 this.elements = elements;
362 public Sequence build(Context cx, Union u, NonTerminalNode cnt, boolean dropall) {
363 Sequence ret = build0(cx, cnt, dropall);
364 for(Seq s : and) ret = ret.and(s.build(cx, null, cnt, true));
365 for(Seq s : not) ret = ret.andnot(s.build(cx, null, cnt, true));
366 if (u!=null) u.add(ret);
369 public Sequence build0(Context cx, NonTerminalNode cnt, boolean dropall) {
370 boolean[] drops = new boolean[elements.length];
371 Element[] els = new Element[elements.length];
372 dropall |= isDropped(cx);
373 for(int i=0; i<elements.length; i++) {
374 if (dropall) drops[i] = true;
375 else drops[i] = elements[i].isDropped(cx);
376 if (elements[i] instanceof LiteralNode && ((LiteralNode)elements[i]).caret) {
377 if (tag != null) throw new RuntimeException("cannot have multiple tags in a sequence: " + this);
378 tag = ((LiteralNode)elements[i]).getLiteralTag();
383 boolean multiNonDrop = false;
384 for(int i=0; i<drops.length; i++)
386 if (idx==-1) idx = i;
387 else multiNonDrop = true;
388 for(int i=0; i<elements.length; i++) {
389 if (!multiNonDrop && i==idx && tag!=null && elements[i] instanceof RepeatNode) {
390 els[i] = ((RepeatNode)elements[i]).build(cx, cnt, dropall, tag);
393 els[i] = elements[i].build(cx, cnt, dropall);
395 if (tag==null && multiNonDrop)
396 throw new RuntimeException("multiple non-dropped elements in sequence: " + Sequence.create("", els));
397 boolean[] lifts = new boolean[elements.length];
398 for(int i=0; i<elements.length; i++)
399 lifts[i] = elements[i].isLifted();
403 ? Sequence.create(illegalTag, els)
404 : Sequence.create(tag, els, drops, lifts);
405 else if (tag==null) ret = Sequence.create(els, idx);
406 else ret = Sequence.create(tag, els, drops, lifts);
409 ret = Sequence.create(tag, els, drops, lifts);
410 if (this.follow != null)
411 ret = ret.followedBy(this.follow.toAtom(cx));
416 private class ReferenceNode extends ElementNode {
417 public String nonTerminal;
418 public boolean parenthesized;
419 public ReferenceNode() { }
420 public ReferenceNode(String nonTerminal) { this(nonTerminal, false); }
421 public ReferenceNode(String nonTerminal, boolean parenthesized) {
422 this.nonTerminal = nonTerminal.indexOf('.')==-1 ? (prefix + nonTerminal) : nonTerminal;
423 this.parenthesized = parenthesized;
425 public NonTerminalNode resolve(Context cx) {
426 NonTerminalNode ret = cx.grammar.get(nonTerminal);
427 if (ret==null) throw new RuntimeException("undefined nonterminal: " + nonTerminal);
430 public Atom toAtom(Context cx) {
431 ElementNode ret = cx.grammar.get(nonTerminal);
432 if (ret == null) throw new RuntimeException("unknown nonterminal \""+nonTerminal+"\"");
433 return ret.toAtom(cx);
435 public boolean isDropped(Context cx) { return resolve(cx).isDropped(cx); }
436 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) {
437 Element ret = cx.get(nonTerminal);
438 if (ret == null) throw new RuntimeException("unknown nonterminal \""+nonTerminal+"\"");
443 private class LiteralNode extends ElementNode {
444 private String string;
445 private final String thePrefix = prefix;
446 private boolean caret;
447 public LiteralNode(String string) { this(string, false); }
448 public LiteralNode(String string, boolean caret) {
449 this.string = string;
452 public String getLiteralTag() { return caret ? thePrefix+string : null; }
453 public String toString() { return "\""+string+"\""; }
454 public boolean isDropped(Context cx) { return true; }
455 public Atom toAtom(Context cx) {
456 if (string.length()!=1) return super.toAtom(cx);
457 Range.Set set = new Range.Set();
458 set.add(string.charAt(0), string.charAt(0));
459 return CharAtom.set(set);
461 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) { return CharAtom.string(string); }
464 private class AtomNode extends ElementNode {
466 public AtomNode() { this(new char[0][]); }
467 public AtomNode(char[][] ranges) { this.ranges = ranges; }
468 public AtomNode(char[] range) { this.ranges = new char[][] { range }; }
469 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) { return toAtom(cx); }
470 public Atom toAtom(Context cx) {
471 Range.Set set = new Range.Set();
472 for(char[] r : ranges) set.add(r[0], r[1]);
473 return CharAtom.set(set);
477 private class RepeatNode extends ElementNode {
478 public ElementNode e, sep;
479 public final boolean zero, many, max;
480 public RepeatNode(ElementNode e, ElementNode sep, boolean zero, boolean many, boolean max) {
481 this.e = e; this.sep = sep; this.zero = zero; this.many = many; this.max = max;
483 public Atom toAtom(Context cx) { return sep==null ? e.toAtom(cx) : super.toAtom(cx); }
484 public boolean isDropped(Context cx) { return e.isDropped(cx); }
485 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) {
486 Element ret = build(cx, cnt, dropall, illegalTag);
487 String must = "must be tagged unless they appear within a dropped expression or their contents are dropped: ";
488 if (!dropall && !isDropped(cx) && !e.isDropped(cx))
489 if (!many) throw new RuntimeException("options (?) " + must + ret);
490 else if (zero) throw new RuntimeException("zero-or-more repetitions (*) " + must + ret);
491 else throw new RuntimeException("one-or-more repetitions (+) " + must + ret);
494 public Element build(Context cx, NonTerminalNode cnt, boolean dropall, Object repeatTag) {
496 ? Repeat.repeat(e.build(cx, null, dropall), zero, many, sep==null ? null : sep.build(cx, null, dropall), repeatTag)
498 ? Repeat.repeatMaximal(e.toAtom(cx), zero, many, repeatTag)
499 : Repeat.repeatMaximal(e.build(cx, null, dropall), zero, many, sep.toAtom(cx), repeatTag);
503 private abstract class ElementNodeWrapper extends ElementNode {
504 protected ElementNode _e;
505 public ElementNodeWrapper(ElementNode e) { this._e = e; }
506 public boolean isDropped(Context cx) { return _e.isDropped(cx); }
507 public Atom toAtom(Context cx) { return _e.toAtom(cx); }
508 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) { return _e.build(cx, cnt, dropall); }
511 private class BacktickNode extends ElementNodeWrapper {
512 public BacktickNode(ElementNode e) { super(e); }
513 public boolean isLifted() { return true; }
516 private class TildeNode extends ElementNodeWrapper {
517 public TildeNode(ElementNode e) { super(e); }
518 public Atom toAtom(Context cx) { return (Atom)((Topology<Character>)_e.toAtom(cx).complement()); }
519 public Element build(Context cx, NonTerminalNode cnt, boolean dropall) { return toAtom(cx); }
522 private class DropNode extends ElementNodeWrapper {
523 public DropNode(ElementNode e) { super(e); }
524 public boolean isDropped(Context cx) { return true; }
527 //////////////////////////////////////////////////////////////////////////////
529 public class Context {
530 public HashMap<String,Union> map = new HashMap<String,Union>();
531 public GrammarNode grammar;
532 public Context(Tree t) { }
533 public Context(GrammarNode g) { this.grammar = g; }
534 public Union build() {
536 for(NonTerminalNode nt : grammar.values()) {
537 Union u = get(nt.name);
538 if ("s".equals(nt.name))
543 public Union peek(String name) { return map.get(name); }
544 public void put(String name, Union u) { map.put(name, u); }
545 public Union get(String name) {
546 Union ret = map.get(name);
547 if (ret != null) return ret;
548 NonTerminalNode nt = grammar.get(name);
550 throw new Error("warning could not find " + name);
552 ret = new Union(name, false);
554 nt.buildIntoPreallocatedUnion(this, nt, nt.isDropped(this), ret);