1 // You may modify, copy, and redistribute this code under
2 // the terms of the GNU General Public License version 2.
11 * A VERY crude, inefficient Java preprocessor
13 * //#define FOO bar baz -- replace all instances of token FOO with "bar baz"
14 * //#replace foo/bar baz/bop -- DUPLICATE everything between here and //#end,
15 * replacing foo with bar and baz with bop in the *second* copy
16 * //#switch(EXPR) -- switch on strings
20 * //#ifdef FOO -- includes contents if FOO passed as define to preprocessor
26 * Replacements are done on a token basis. Tokens are defined as a
27 * sequence of characters which all belong to a single class. The
28 * two character classes are:
31 * - all other non-whitespace characters
33 * Preprocessor makes use of several optional system properties:
35 * - ibex.tool.preprocessor.define
36 * - ibex.tool.preprocessor.inputdir
37 * - ibex.tool.preprocessor.outputdir
39 * @author adam@ibex.org, crawshaw@ibex.org
41 public class Preprocessor {
43 public static String replaceAll(String source, String regexp, String replaceWith) {
44 return source.replaceAll(regexp, replaceWith);
47 RE re = new RE(regexp, 0, RESyntax.RE_SYNTAX_PERL5);
48 return (String)re.substituteAll(source, replaceWith);
49 } catch (Exception e) {
56 public static void main(String[] args) throws Exception {
57 List defs = new ArrayList();
59 String define = System.getProperty("ibex.tool.preprocessor.define");
61 StringTokenizer st = new StringTokenizer(define.toUpperCase(), ",");
62 while (st.hasMoreTokens()) defs.add(st.nextToken().trim());
65 String inputdir = System.getProperty("ibex.tool.preprocessor.inputdir");
66 if (inputdir == null) inputdir = "src/";
68 String outputdir = System.getProperty("ibex.tool.preprocessor.outputdir");
69 if (outputdir == null) outputdir = "build/java/";
71 if (args.length == 0) {
72 BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
73 BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(System.out));
75 // process stdin to stdout
76 Preprocessor cc = new Preprocessor(br, bw, defs);
77 Vector err = cc.process();
81 boolean errors = false;
82 for (int i=0; i < err.size(); i++) {
83 if (err.get(i) instanceof Error) errors = true; System.err.println(err.get(i)); }
84 if (errors) throw new Exception();
86 for(int i=0; i<args.length; i++) {
87 if (!args[i].endsWith(".java")) continue;
88 File source = new File(args[i]);
89 File target = new File(replaceAll(args[i], inputdir, outputdir));
90 if (target.exists() && target.lastModified() > source.lastModified()) continue;
91 System.err.println("preprocessing " + args[i]);
92 new File(target.getParent()).mkdirs();
93 BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(source)));
94 BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(target)));
95 Preprocessor cc = new Preprocessor(br, bw, defs);
96 Vector err = cc.process();
98 boolean errors = false;
99 for (int j=0; j < err.size(); j++) {
100 if (err.get(j) instanceof Error) errors = true; System.err.println(err.get(j)); }
101 if (errors) System.exit(-1);
108 private LineNumberReader in;
109 private PrintWriter out;
111 private Hashtable replace = new Hashtable();
112 private Hashtable repeatreplace = null;
113 private Vector sinceLastRepeat = null;
114 private Vector err = new Vector();
117 private int enumSwitch = 0; // number appended to variable used in switch implementation
120 public Preprocessor(Reader reader, Writer writer, List d) {
126 public void setReader(Reader reader) { r = reader; if (r != null) in = new LineNumberReader(r); }
127 public Reader getReader() { return r; }
129 public void setWriter(Writer writer) { w = writer; if (w != null) out = new PrintWriter(w); }
130 public Writer getWriter() { return w; }
133 /** process data from reader, write to writer, return vector of errors */
134 public Vector process() throws IOException {
139 while((s = in.readLine()) != null) {
140 if (sinceLastRepeat != null) sinceLastRepeat.addElement(s);
141 String trimmed = s.trim();
143 if (trimmed.startsWith("//#define ")) {
144 if (trimmed.length() == 9 || trimmed.charAt(9) != ' ') {
145 err.add(new Error("#define badly formed, ignored")); continue PROCESS;
147 int keyStart = indexOfNotWS(trimmed, 9);
148 if (keyStart == -1) {
149 err.add(new Error("#define requires KEY")); continue PROCESS;
151 int keyEnd = indexOfWS(trimmed, keyStart);
153 int macroStart = trimmed.indexOf('(');
154 int macroEnd = trimmed.indexOf(')');
155 if (macroStart > keyEnd) {
156 // no macro is defined, just make sure something dumb like KEYNA)ME hasn't been done
157 if (macroEnd < keyEnd) { err.add(new Error("#define key contains invalid char: ')'")); continue PROCESS; }
158 macroStart = macroEnd = -1;
161 if (macroStart == 0) {
162 err.add(new Error("#define macro requires name")); continue PROCESS;
163 } else if (macroStart > 0) {
164 if (macroStart > macroEnd) { err.add(new Error("#define macro badly formed")); continue PROCESS; }
165 if (macroStart+1 == macroEnd) { err.add(new Error("#define macro requires property name")); continue PROCESS; }
167 JSFunctionMacro fm = new JSFunctionMacro();
168 String key = trimmed.substring(keyStart, macroStart);
169 String unbound = trimmed.substring(macroStart +1, macroEnd);
170 int unboundDiv = unbound.indexOf(',');
171 if (unboundDiv == -1) {
172 fm.unbound1 = unbound;
174 fm.unbound1 = unbound.substring(0, unboundDiv);
175 fm.unbound2 = unbound.substring(unboundDiv +1);
176 if (fm.unbound1.length() == 0) { err.add(new Error("#define macro property 1 requires name")); continue PROCESS; }
177 if (fm.unbound2.length() == 0) { err.add(new Error("#define macro property 1 requires name")); continue PROCESS; }
179 fm.expression = trimmed.substring(keyEnd).trim();
180 replace.put(key, fm);
182 String key = trimmed.substring(keyStart, keyEnd);
183 String val = trimmed.substring(keyEnd).trim();
184 replace.put(key, val);
186 out.print("\n"); // preserve line numbers
188 } else if (trimmed.startsWith("//#repeat ")) {
189 trimmed = trimmed.substring(9);
190 while(trimmed.charAt(trimmed.length() - 1) == '\\') {
191 String s2 = in.readLine().trim();
192 if (s2.startsWith("//")) s2 = s2.substring(2).trim();
193 trimmed = trimmed.substring(0, trimmed.length() - 1) + " " + s2;
194 out.print("\n"); // preserve line numbers
196 StringTokenizer st = new StringTokenizer(trimmed, " ");
197 repeatreplace = (Hashtable)replace.clone();
198 while (st.hasMoreTokens()) {
199 String tok = st.nextToken().trim();
200 String key = tok.substring(0, tok.indexOf('/'));
201 String val = tok.substring(tok.indexOf('/') + 1);
202 repeatreplace.put(key, val);
204 sinceLastRepeat = new Vector();
205 out.print("\n"); // preserve line numbers
207 } else if (trimmed.startsWith("//#end")) {
208 if (sinceLastRepeat == null) { err.add(new Warning("#end orphaned")); continue PROCESS; }
209 Hashtable save = replace;
210 replace = repeatreplace;
212 for(int i=0; i<sinceLastRepeat.size() - 1; i++) out.print(processLine((String)sinceLastRepeat.elementAt(i), true));
213 sinceLastRepeat = null;
215 } else if (trimmed.startsWith("//#ifdef")) {
216 String expr = trimmed.substring(8).trim().toUpperCase();
217 out.println(trimmed);
218 boolean useCode = defs.contains(expr);
219 for (trimmed = in.readLine().trim(); !trimmed.startsWith("//#endif"); trimmed = in.readLine().trim()) {
220 if (trimmed.startsWith("//#else")) useCode = !useCode;
221 else if (!useCode) out.print("// ");
222 out.print(processLine(trimmed, false));
224 out.println("//#endif "+expr);
226 } else if (trimmed.startsWith("//#switch")) {
227 int expStart = trimmed.indexOf('(') +1;
228 if (expStart < 1) { err.add(new Error("expected ( in #switch")); continue PROCESS; }
229 int expEnd = trimmed.lastIndexOf(')');
230 if (expEnd == -1) { err.add(new Error("expected ) in #switch")); continue PROCESS; }
231 if (expEnd - expStart <= 1) { err.add(new Error("badly formed #switch statement")); continue PROCESS; }
232 String expr = trimmed.substring(expStart, expEnd);
234 out.print("final String ccSwitch"+enumSwitch+" = (String)("+expr+"); ");
235 out.print("SUCCESS:do { switch(ccSwitch"+enumSwitch+".length()) {\n");
237 Hashtable[] byLength = new Hashtable[255];
239 String Default = null;
240 for(trimmed = in.readLine().trim(); !trimmed.startsWith("//#end"); trimmed = in.readLine().trim()) {
241 if (trimmed.startsWith("default:")) {
242 Default = processLine(trimmed.substring(8), false);
245 if (trimmed.startsWith("case ")) {
247 int strStart = trimmed.indexOf('\"') +1;
248 if (strStart < 1) { err.add(new Error("expected opening of String literal")); continue PROCESS; }
249 int strEnd = trimmed.indexOf('\"', strStart);
250 if (strEnd == -1) { err.add(new Error("expected closing of String literal")); continue PROCESS; }
251 key = trimmed.substring(strStart, strEnd);
253 Hashtable thisCase = (Hashtable)byLength[key.length()];
254 if (thisCase == null) byLength[key.length()] = thisCase = new Hashtable();
255 thisCase.put(key, "");
257 // find end of case definition
258 int caseEnd = trimmed.indexOf(':', strEnd) +1;
259 if (caseEnd < 1) { err.add(new Error("expected :")); continue PROCESS; }
260 trimmed = trimmed.substring(caseEnd);
264 Hashtable hash = byLength[key.length()];
265 hash.put(key, (String)hash.get(key) + replaceAll(processLine(trimmed, false), "//[^\"]*$", "").trim() + "\n");
267 out.print(processLine(trimmed, false));
271 for(int i=0; i<255; i++) {
272 if (byLength[i] == null) continue;
273 out.print("case " + i + ": { switch(ccSwitch"+enumSwitch+".charAt(0)) { ");
274 buildTrie("", byLength[i]);
275 out.print("}; break; } ");
277 out.print("} /* switch */ ");
278 if (Default != null) out.print(" " + Default);
279 out.print(" } while(false); /* OUTER */\n");
283 out.print(processLine(s, false));
290 private static class MyVec {
291 private Object[] store;
292 private int size = 0;
294 public MyVec() { this(10); }
295 public MyVec(int i) { store = new Object[i]; }
296 public MyVec(int i, Object[] store) { size = i; this.store = store; }
298 private void grow() { grow(store.length * 2); }
299 private void grow(int newsize) {
300 Object[] newstore = new Object[newsize];
301 System.arraycopy(store, 0, newstore, 0, size);
305 public void removeAllElements() {
306 for(int i=0; i<size; i++) store[i] = null;
310 public void toArray(Object[] o) {
311 for(int i=0; i<size; i++)
315 public int indexOf(Object o) {
316 for(int i=0; i<size; i++)
317 if (o == null ? store[i] == null : store[i].equals(o)) return i;
322 public void addElement(Object o) {
323 if (size >= store.length - 1) grow();
327 public Object peek() {
328 return lastElement();
331 public Object elementAt(int i) {
335 public Object lastElement() {
336 if (size == 0) return null;
337 return store[size - 1];
340 public void push(Object o) { addElement(o); }
341 public Object pop() {
342 Object ret = lastElement();
343 if (size > 0) store[size--] = null;
347 public int size() { return size; }
349 public void setSize(int newSize) {
350 if (newSize < 0) throw new RuntimeException("tried to set size to negative value");
351 if (newSize > store.length) grow(newSize * 2);
353 for(int i=newSize; i<size; i++)
358 public void copyInto(Object[] out) {
359 for(int i=0; i<size; i++)
363 public void fromArray(Object[] in) {
365 for(int i=0; i<size; i++)
369 public void removeElementAt(int i) {
370 if (i >= size || i < 0) throw new RuntimeException("tried to remove an element outside the vector's limits");
371 for(int j=i; j<size - 1; j++)
372 store[j] = store[j + 1];
376 public void setElementAt(Object o, int i) {
377 if (i >= size) setSize(i);
381 public void removeElement(Object o) {
382 int idx = indexOf(o);
383 if (idx != -1) removeElementAt(idx);
386 public void insertElementAt(Object o, int at) {
387 if (size == store.length) grow();
388 for(int i=size; i>at; i--)
389 store[i] = store[i-1];
394 public interface CompareFunc {
395 public int compare(Object a, Object b);
398 public void sort(CompareFunc c) {
399 sort(this, null, c, 0, size-1);
402 public static void sort(MyVec a, MyVec b, CompareFunc c) {
403 if (b != null && a.size != b.size) throw new IllegalArgumentException("MyVec a and b must be of equal size");
404 sort(a, b, c, 0, a.size-1);
407 private static final void sort(MyVec a, MyVec b, CompareFunc c, int start, int end) {
408 Object tmpa, tmpb = null;
409 if(start >= end) return;
411 for(int i=start+1;i<=end;i++) {
413 if (b != null) tmpb = b.store[i];
415 for(j=i-1;j>=start;j--) {
416 if(c.compare(a.store[j],tmpa) <= 0) break;
417 a.store[j+1] = a.store[j];
418 if (b != null) b.store[j+1] = b.store[j];
421 if (b != null) b.store[j+1] = tmpb;
426 Object pivot = a.store[end];
431 while(c.compare(a.store[++lo],pivot) < 0) { }
432 while((hi > lo) && c.compare(a.store[--hi],pivot) > 0) { }
434 if (b != null) swap(b, lo,hi);
438 if (b != null) swap(b, lo,end);
440 sort(a, b, c, start, lo-1);
441 sort(a, b, c, lo+1, end);
444 private static final void swap(MyVec vec, int a, int b) {
446 Object tmp = vec.store[a];
447 vec.store[a] = vec.store[b];
454 private void buildTrie(String prefix, Hashtable cases) {
455 Enumeration caseKeys = cases.keys();
456 MyVec keys = new MyVec();
457 while(caseKeys.hasMoreElements()) keys.addElement(caseKeys.nextElement());
458 keys.sort(new MyVec.CompareFunc() { public int compare(Object a, Object b) {
459 return ((String)a).compareTo((String)b);
462 for(int i=0; i<keys.size(); i++) {
463 if (!((String)keys.elementAt(i)).startsWith(prefix)) continue;
464 String prefixPlusOne = ((String)keys.elementAt(i)).substring(0, prefix.length() + 1);
465 if (i<keys.size()-1 && prefixPlusOne.equals((((String)keys.elementAt(i + 1)).substring(0, prefix.length() + 1)))) {
466 out.print("case \'" + prefixPlusOne.charAt(prefixPlusOne.length() - 1) + "\': { ");
467 out.print("switch(ccSwitch"+enumSwitch+".charAt(" + (prefix.length()+1) + ")) { ");
468 buildTrie(prefixPlusOne, cases);
469 out.print("} break; } ");
470 while(i<keys.size() && prefixPlusOne.equals(((String)keys.elementAt(i)).substring(0, prefix.length() + 1))) i++;
471 if (i<keys.size()) { i--; continue; }
473 out.print("case \'" + prefixPlusOne.charAt(prefixPlusOne.length() - 1) + "\': ");
474 String code = (String)cases.get(keys.elementAt(i));
475 code = code.substring(0, code.length());
476 String key = (String)keys.elementAt(i);
477 out.print("if (\""+key+"\".equals(ccSwitch"+enumSwitch+")) { if (true) do { " + code + " } while(false); break SUCCESS; } break; ");
482 private String processLine(String s, boolean deleteLineEndings) throws IOException {
483 if (deleteLineEndings && s.indexOf("//") != -1) s = s.substring(0, s.indexOf("//"));
485 for(int i=0; i<s.length(); i++) {
486 char c = s.charAt(i);
487 if (!Character.isLetter(c) && !Character.isDigit(c) && c != '_') {
492 for(j = i; j < s.length(); j++) {
494 if (!Character.isLetter(c) && !Character.isDigit(c) && c != '_') break;
496 String tok = s.substring(i, j);
497 Object val = replace.get(tok);
501 } else if (val instanceof JSFunctionMacro) {
502 if (s.charAt(j) != '(') { ret += tok; i = j - 1; continue; }
503 ret += ((JSFunctionMacro)val).process(s.substring(j+1, s.indexOf(')', j)));
504 i = s.indexOf(')', j);
510 if (!deleteLineEndings) ret += "\n";
514 private static int indexOfWS(String s) { return indexOfWS(s, 0); }
515 private static int indexOfWS(String s, int beginIndex) {
516 if (s == null || beginIndex >= s.length()) return -1;
517 for (; beginIndex < s.length(); beginIndex++) {
518 if (s.charAt(beginIndex) == ' ') return beginIndex;
523 private static int indexOfNotWS(String s) { return indexOfWS(s, 0); }
524 private static int indexOfNotWS(String s, int beginIndex) {
525 if (s == null || beginIndex >= s.length()) return -1;
526 for (; beginIndex < s.length(); beginIndex++) {
527 if (s.charAt(beginIndex) != ' ') return beginIndex;
532 public class Warning {
533 protected String msg;
536 public Warning() { msg = ""; }
537 public Warning(String m) { msg = m; if (in != null) line = in.getLineNumber(); }
539 public String toString() { return "WARNING Line "+line+": "+msg; }
542 public class Error extends Warning {
543 public Error() { super(); }
544 public Error(String m) { super(m); }
545 public String toString() { return "ERROR Line "+line+": "+msg; }
548 public static class JSFunctionMacro {
549 public String unbound1 = null;
550 public String unbound2 = null;
551 public String expression = null;
552 public String process(String args) {
553 String bound1 = null;
554 String bound2 = null;
555 if (unbound2 == null) {
557 return replaceAll(expression, unbound1, bound1);
559 bound1 = args.substring(0, args.indexOf(','));
560 bound2 = args.substring(args.indexOf(',') + 1);
561 return replaceAll(replaceAll(expression, unbound1, bound1), unbound2, bound2);