mildew.parser source code

1 /**
2 This module implements the Parser struct, which generates Nodes from tokens. The resulting syntax tree
3 is processed by the Compiler.
4 
5 ────────────────────────────────────────────────────────────────────────────────
6 
7 Copyright (C) 2021 pillager86.rf.gd
8 
9 This program is free software: you can redistribute it and/or modify it under 
10 the terms of the GNU General Public License as published by the Free Software 
11 Foundation, either version 3 of the License, or (at your option) any later 
12 version.
13 
14 This program is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
16 PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17 
18 You should have received a copy of the GNU General Public License along with 
19 this program.  If not, see <https://www.gnu.org/licenses/>.
20 */
21 module mildew.parser;
22 
23 import std.conv: to, parse;
24 debug
25 {
26     import std.stdio;
27 }
28 
29 import mildew.exceptions: ScriptCompileException;
30 import mildew.lexer: Token;
31 import mildew.nodes;
32 import mildew.types.any: ScriptAny;
33 import mildew.util.stack;
34 
35 private int unaryOpPrecedence(Token opToken, bool isPost = false)
36 {
37     if(opToken.isKeyword("typeof"))
38     {
39         if(!isPost)
40             return 17;
41     }
42 
43     // see grammar.txt for explanation of magic constants
44     switch(opToken.type)
45     {
46         // TODO handle ++, -- postfix
47         case Token.Type.BIT_NOT: 
48         case Token.Type.NOT:
49         case Token.Type.PLUS:
50         case Token.Type.DASH:
51             if(!isPost)
52                 return 17;
53             else
54                 return 0;
55         case Token.Type.INC:
56         case Token.Type.DEC:
57             if(isPost)
58                 return 18;
59             else
60                 return 17;
61         default: 
62             return 0;
63     }
64 }
65 
66 private int binaryOpPrecedence(Token opToken)
67 {
68     // TODO handle keywords in as 12 here
69     if(opToken.isKeyword("instanceof"))
70         return 12;
71 
72     // see grammar.txt for explanation of magic constants
73     switch(opToken.type)
74     {
75         case Token.Type.LBRACKET:
76         case Token.Type.DOT: 
77         case Token.Type.LPAREN:
78             return 20;
79         case Token.Type.POW:
80             return 16;
81         case Token.Type.STAR: 
82         case Token.Type.FSLASH: 
83         case Token.Type.PERCENT:
84             return 15;
85         case Token.Type.PLUS: 
86         case Token.Type.DASH:
87             return 14;
88         case Token.Type.BIT_LSHIFT: 
89         case Token.Type.BIT_RSHIFT: 
90         case Token.Type.BIT_URSHIFT:
91             return 13;
92         case Token.Type.LT: 
93         case Token.Type.LE: 
94         case Token.Type.GT: 
95         case Token.Type.GE:
96             return 12;
97         case Token.Type.EQUALS: 
98         case Token.Type.NEQUALS: 
99         case Token.Type.STRICT_EQUALS: 
100         case Token.Type.STRICT_NEQUALS:
101             return 11;
102         case Token.Type.BIT_AND:
103             return 10;
104         case Token.Type.BIT_XOR:
105             return 9;
106         case Token.Type.BIT_OR:
107             return 8;
108         case Token.Type.AND:
109             return 7;
110         case Token.Type.OR:
111             return 6;
112         case Token.Type.QUESTION:
113             return 4;
114         case Token.Type.ASSIGN:
115         case Token.Type.POW_ASSIGN:
116         case Token.Type.STAR_ASSIGN:
117         case Token.Type.FSLASH_ASSIGN:
118         case Token.Type.PERCENT_ASSIGN:
119         case Token.Type.PLUS_ASSIGN:
120         case Token.Type.DASH_ASSIGN:
121         case Token.Type.BAND_ASSIGN:
122         case Token.Type.BXOR_ASSIGN:
123         case Token.Type.BOR_ASSIGN:
124         case Token.Type.BLS_ASSIGN:
125         case Token.Type.BRS_ASSIGN:
126         case Token.Type.BURS_ASSIGN:
127             return 3;
128         default:
129             return 0;
130     }
131     // TODO null coalesce 5,yield 2, comma 1?
132 }
133 
134 private bool isBinaryOpLeftAssociative(in Token opToken)
135 {
136     switch(opToken.type)
137     {
138         case Token.Type.LBRACKET:
139         case Token.Type.DOT: 
140         case Token.Type.LPAREN:
141             return true;
142         case Token.Type.POW:
143             return false;
144         case Token.Type.STAR: 
145         case Token.Type.FSLASH: 
146         case Token.Type.PERCENT:
147             return true;
148         case Token.Type.PLUS: 
149         case Token.Type.DASH:
150             return true;
151         case Token.Type.BIT_LSHIFT: 
152         case Token.Type.BIT_RSHIFT: 
153         case Token.Type.BIT_URSHIFT:
154             return true;
155         case Token.Type.LT: 
156         case Token.Type.LE: 
157         case Token.Type.GT: 
158         case Token.Type.GE:
159             return true;
160         case Token.Type.EQUALS: 
161         case Token.Type.NEQUALS: 
162         case Token.Type.STRICT_EQUALS: 
163         case Token.Type.STRICT_NEQUALS:
164             return true;
165         case Token.Type.BIT_AND:
166             return true;
167         case Token.Type.BIT_XOR:
168             return true;
169         case Token.Type.BIT_OR:
170             return true;
171         case Token.Type.AND:
172             return true;
173         case Token.Type.OR:
174             return true;
175         case Token.Type.QUESTION:
176             return false;
177         case Token.Type.ASSIGN:
178         case Token.Type.POW_ASSIGN:
179         case Token.Type.STAR_ASSIGN:
180         case Token.Type.FSLASH_ASSIGN:
181         case Token.Type.PERCENT_ASSIGN:
182         case Token.Type.PLUS_ASSIGN:
183         case Token.Type.DASH_ASSIGN:
184         case Token.Type.BAND_ASSIGN:
185         case Token.Type.BXOR_ASSIGN:
186         case Token.Type.BOR_ASSIGN:
187         case Token.Type.BLS_ASSIGN:
188         case Token.Type.BRS_ASSIGN:
189         case Token.Type.BURS_ASSIGN:
190             return false;
191         default:
192             return false;
193     }   
194 }
195 
196 private bool tokenBeginsLoops(const Token tok)
197 {
198     return tok.type == Token.Type.LABEL 
199         || tok.isKeyword("while")
200         || tok.isKeyword("do")
201         || tok.isKeyword("for");
202 }
203 
204 /**
205  * The parser is used by the interpreter to generate a syntax tree out of tokens.
206  */
207 struct Parser
208 {
209     /**
210      * The constructor takes all tokens so that in the future, looking ahead for specific tokens
211      * can allow support for lambdas and other complex language features.
212      */
213     this(Token[] tokens)
214     {
215         _tokens = tokens;
216         nextToken(); // prime
217     }
218 
219     /**
220      * The main starting point. Also the "program" grammar rule. This method generates a block statement
221      * node where the interpreter iterates through each statement and executes it.
222      */
223     BlockStatementNode parseProgram()
224     {
225         immutable lineNo = _currentToken.position.line;
226         _functionContextStack.push(FunctionContext(FunctionContextType.NORMAL, 0, 0, []));
227         auto statements = parseStatements(Token.Type.EOF);
228         _functionContextStack.pop();
229         assert(_functionContextStack.size == 0, "Sanity check failed: _functionContextStack");
230         return new BlockStatementNode(lineNo, statements);
231     }
232 
233 package:
234 
235     /// parse a single expression. See https://eli.thegreenplace.net/2012/08/02/parsing-expressions-by-precedence-climbing
236     /// for algorithm.
237     ExpressionNode parseExpression(int minPrec = 1)
238     {      
239         ExpressionNode primaryLeft = null;
240 
241         immutable unOpPrec = _currentToken.unaryOpPrecedence;
242         if(unOpPrec > minPrec)
243         {
244             auto opToken = _currentToken;
245             nextToken();
246             primaryLeft = parsePrimaryExpression();
247             primaryLeft = new UnaryOpNode(opToken, primaryLeft);
248         }
249         else
250         {
251             primaryLeft = parsePrimaryExpression();
252         }
253 
254         while(_currentToken.binaryOpPrecedence >= minPrec || _currentToken.unaryOpPrecedence(true) >= minPrec)
255         {
256             if(_currentToken.unaryOpPrecedence(true) >= minPrec)
257             {
258                 // writeln("We must handle postfix " ~ _currentToken.symbol ~ " for " ~ primaryLeft.toString);
259                 primaryLeft = new PostfixOpNode(_currentToken, primaryLeft);
260                 nextToken();
261             }
262             else 
263             {
264                 auto opToken = _currentToken;
265                 immutable prec = opToken.binaryOpPrecedence;
266                 immutable isLeftAssoc = opToken.isBinaryOpLeftAssociative;
267                 immutable nextMinPrec = isLeftAssoc? prec + 1 : prec;
268                 nextToken();
269                 if(opToken.type == Token.Type.QUESTION)
270                 {
271                     // primaryLeft is our condition node
272                     auto onTrue = parseExpression();
273                     if(_currentToken.type != Token.Type.COLON)
274                         throw new ScriptCompileException("Expected ':' in terniary operator expression", _currentToken);
275                     nextToken();
276                     auto onFalse = parseExpression();
277                     primaryLeft = new TerniaryOpNode(primaryLeft, onTrue, onFalse);
278                 }
279                 else if(opToken.type == Token.Type.DOT)
280                 {
281                     auto right = parsePrimaryExpression();
282                     if(!cast(VarAccessNode)right)
283                         throw new ScriptCompileException("Right hand side of `.` operator must be identifier", opToken);
284                     if(cast(VarAccessNode)right is null)
285                         throw new ScriptCompileException("Object members must be valid identifiers", _currentToken);
286                     if(unOpPrec != 0 && prec > unOpPrec)
287                     {
288                         auto uon = cast(UnaryOpNode)primaryLeft;
289                         primaryLeft = new UnaryOpNode(uon.opToken, 
290                                 new MemberAccessNode(uon.operandNode, opToken, right));
291                     }
292                     else
293                         primaryLeft = new MemberAccessNode(primaryLeft, opToken, right);
294                 }
295                 else if(opToken.type == Token.Type.LBRACKET)
296                 {
297                     auto index = parseExpression();
298                     if(_currentToken.type != Token.Type.RBRACKET)
299                         throw new ScriptCompileException("Missing ']'", _currentToken);
300                     nextToken();
301                     if(unOpPrec != 0 && prec > unOpPrec)
302                     {
303                         auto uon = cast(UnaryOpNode)primaryLeft;
304                         primaryLeft = new UnaryOpNode(uon.opToken, new ArrayIndexNode(uon.operandNode, index));
305                     }
306                     else
307                         primaryLeft = new ArrayIndexNode(primaryLeft, index);
308                 }
309                 else if(opToken.type == Token.Type.LPAREN)
310                 {
311                     auto params = parseCommaSeparatedExpressions(Token.Type.RPAREN);
312                     nextToken();
313                     if(unOpPrec != 0 && prec > unOpPrec)
314                     {
315                         auto uon = cast(UnaryOpNode)primaryLeft;
316                         primaryLeft = new UnaryOpNode(uon.opToken, new FunctionCallNode(uon.operandNode, params));
317                     }
318                     else
319                         primaryLeft = new FunctionCallNode(primaryLeft, params);
320                 }
321                 else 
322                 {
323                     ExpressionNode primaryRight = parseExpression(nextMinPrec);
324                     // catch invalid assignments
325                     if(opToken.isAssignmentOperator)
326                     {
327                         if(!(cast(VarAccessNode)primaryLeft 
328                           || cast(MemberAccessNode)primaryLeft 
329                           || cast(ArrayIndexNode)primaryLeft))
330                         {
331                             throw new ScriptCompileException("Invalid left hand operand for assignment "
332                                     ~ primaryLeft.toString(), opToken);
333                         }
334                     }
335                     primaryLeft = new BinaryOpNode(opToken, primaryLeft, primaryRight);
336                 }
337             }
338         }
339 
340         return primaryLeft;
341     }
342 
343 private:
344 
345     // very weird way of doing things but it works, may need rework to allow undefined to be a case
346     ScriptAny evaluateCTFE(ExpressionNode expr)
347     {
348         import mildew.environment: Environment;
349         import mildew.compiler: Compiler;
350         import mildew.vm.chunk: Chunk;
351         import mildew.vm.virtualmachine: VirtualMachine;
352         auto ret = new ReturnStatementNode(0, expr);
353         auto compiler = new Compiler();
354         auto chunk = compiler.compile([ret]);
355         auto vm = new VirtualMachine(new Environment(null, "<ctfe>"));
356         try 
357         {
358             return vm.run(chunk);
359         }
360         catch(Exception ex)
361         {
362             return ScriptAny.UNDEFINED;
363         }
364     }
365 
366     /// parses a single statement
367     StatementNode parseStatement()
368     {
369         StatementNode statement;
370         immutable lineNumber = _currentToken.position.line;
371         // check for var declaration
372         if(_currentToken.isKeyword("var") || _currentToken.isKeyword("let") || _currentToken.isKeyword("const"))
373         {
374             statement = parseVarDeclarationStatement();
375         }
376         // check for {} block
377         else if(_currentToken.type == Token.Type.LBRACE)
378         {
379             // TODO: peek two tokens ahead for a ':' to indicate whether or not this is an object literal expression
380             nextToken();
381             auto statements = parseStatements(Token.Type.RBRACE);
382             nextToken();
383             statement = new BlockStatementNode(lineNumber, statements);
384         }
385         // check for if statement
386         else if(_currentToken.isKeyword("if"))
387         {
388             statement = parseIfStatement();
389         }
390         // check for switch
391         else if(_currentToken.isKeyword("switch"))
392         {
393             statement = parseSwitchStatement();
394         }
395         // check for loops
396         else if(_currentToken.tokenBeginsLoops())
397         {
398             statement = parseLoopStatement();
399         }
400         // break statement?
401         else if(_currentToken.isKeyword("break"))
402         {
403             if(_functionContextStack.top.loopStack == 0 && _functionContextStack.top.switchStack == 0)
404                 throw new ScriptCompileException("Break statements only allowed in loops or switch-case bodies", 
405                     _currentToken);
406             nextToken();
407             string label = "";
408             if(_currentToken.type == Token.Type.IDENTIFIER)
409             {
410                 label = _currentToken.text;
411                 bool valid = false;
412                 // label must exist on stack
413                 for(size_t i = _functionContextStack.top.labelStack.length; i > 0; --i)
414                 {
415                     if(_functionContextStack.top.labelStack[i-1] == label)
416                     {
417                         valid = true;
418                         break;
419                     }
420                 }
421                 if(!valid)
422                     throw new ScriptCompileException("Break label does not refer to valid label", _currentToken);
423                 nextToken();
424             }
425             if(_currentToken.type != Token.Type.SEMICOLON)
426                 throw new ScriptCompileException("Expected ';' after break", _currentToken);
427             nextToken();
428             statement = new BreakStatementNode(lineNumber, label);
429         }
430         // continue statement
431         else if(_currentToken.isKeyword("continue"))
432         {
433             if(_functionContextStack.top.loopStack == 0)
434                 throw new ScriptCompileException("Continue statements only allowed in loops", _currentToken);
435             nextToken();
436             string label = "";
437             if(_currentToken.type == Token.Type.IDENTIFIER)
438             {
439                 label = _currentToken.text;
440                 bool valid = false;
441                 for(size_t i = _functionContextStack.top.labelStack.length; i > 0; --i)
442                 {
443                     if(_functionContextStack.top.labelStack[i-1] == label)
444                     {
445                         valid = true;
446                         break;
447                     }
448                 }
449                 if(!valid)
450                     throw new ScriptCompileException("Continue label does not refer to valid label", _currentToken);
451                 nextToken();
452             }
453             if(_currentToken.type != Token.Type.SEMICOLON)
454                 throw new ScriptCompileException("Expected ';' after continue", _currentToken);
455             nextToken();
456             statement = new ContinueStatementNode(lineNumber, label);
457         }
458         // return statement with optional expression
459         else if(_currentToken.isKeyword("return"))
460         {
461             nextToken();
462             ExpressionNode expression = null;
463             if(_currentToken.type != Token.Type.SEMICOLON)
464                 expression = parseExpression();
465             if(_currentToken.type != Token.Type.SEMICOLON)
466                 throw new ScriptCompileException("Expected ';' after return", _currentToken);
467             nextToken();
468             statement = new ReturnStatementNode(lineNumber, expression);
469         }
470         else if(_currentToken.isKeyword("function"))
471         {
472             statement = parseFunctionDeclarationStatement();
473         }
474         else if(_currentToken.isKeyword("throw"))
475         {
476             nextToken();
477             auto expr = parseExpression();
478             if(_currentToken.type != Token.Type.SEMICOLON)
479                 throw new ScriptCompileException("Expected ';' after throw expression", _currentToken);
480             nextToken();
481             statement = new ThrowStatementNode(lineNumber, expr);
482         }
483         else if(_currentToken.isKeyword("try"))
484         {
485             statement = parseTryCatchBlockStatement();
486         }
487         else if(_currentToken.isKeyword("delete"))
488         {
489             immutable delToken = _currentToken;
490             nextToken();
491             auto tok = _currentToken;
492             auto expression = parseExpression();
493             if(cast(MemberAccessNode)expression is null && cast(ArrayIndexNode)expression is null)
494                 throw new ScriptCompileException("Invalid operand for delete operation", tok);
495             statement = new DeleteStatementNode(lineNumber, delToken, expression);
496         }
497         else if(_currentToken.isKeyword("class"))
498         {
499             statement = parseClassDeclaration();
500         }
501         else // for now has to be one expression followed by semicolon or EOF
502         {
503             if(_currentToken.type == Token.Type.SEMICOLON)
504             {
505                 // empty statement
506                 statement = new ExpressionStatementNode(lineNumber, null);
507                 nextToken();
508             }
509             else 
510             {
511                 auto expression = parseExpression();
512                 if(_currentToken.type != Token.Type.SEMICOLON && _currentToken.type != Token.Type.EOF)
513                     throw new ScriptCompileException("Expected semicolon after expression", _currentToken);
514                 nextToken(); // eat semicolon
515                 statement = new ExpressionStatementNode(lineNumber, expression);
516             }
517         }
518         return statement;
519     }
520 
521     ExpressionNode parsePrimaryExpression()
522     {
523         import std.conv: to, ConvException;
524 
525         ExpressionNode left = null;
526         switch(_currentToken.type)
527         {
528             case Token.Type.LPAREN: {
529                 // first check if this is a lambda
530                 auto lookahead = peekTokens(3);
531                 if((lookahead[1].type == Token.Type.COMMA ||
532                    lookahead[1].type == Token.Type.ARROW ||
533                    lookahead[2].type == Token.Type.ARROW) && lookahead[0].type != Token.Type.LPAREN)
534                 {
535                     left = parseLambda(true);
536                 }
537                 else
538                 {
539                     nextToken();
540                     left = parseExpression();
541                     if(_currentToken.type != Token.Type.RPAREN)
542                         throw new ScriptCompileException("Missing ')' in primary expression", _currentToken);
543                     nextToken();
544                 }
545                 break;
546             }
547             case Token.Type.LBRACE:
548                 left = parseObjectLiteral();
549                 break;
550             case Token.Type.DOUBLE:
551                 if(_currentToken.literalFlag == Token.LiteralFlag.NONE)
552                     left = new LiteralNode(_currentToken, ScriptAny(to!double(_currentToken.text)));
553                 else
554                     throw new ScriptCompileException("Malformed floating point token detected", _currentToken);
555                 nextToken();
556                 break;
557             case Token.Type.INTEGER:
558                 try 
559                 {
560                     if(_currentToken.literalFlag == Token.LiteralFlag.NONE)
561                         left = new LiteralNode(_currentToken, ScriptAny(to!long(_currentToken.text)));
562                     else if(_currentToken.literalFlag == Token.LiteralFlag.HEXADECIMAL)
563                         left = new LiteralNode(_currentToken, ScriptAny(_currentToken.text[2..$].to!long(16)));
564                     else if(_currentToken.literalFlag == Token.LiteralFlag.OCTAL)
565                         left = new LiteralNode(_currentToken, ScriptAny(_currentToken.text[2..$].to!long(8)));
566                     else if(_currentToken.literalFlag == Token.LiteralFlag.BINARY)
567                         left = new LiteralNode(_currentToken, ScriptAny(_currentToken.text[2..$].to!long(2)));
568                 }
569                 catch(ConvException ex)
570                 {
571                     throw new ScriptCompileException("Integer literal is too long", _currentToken);
572                 }
573                 nextToken();
574                 break;
575             case Token.Type.STRING:
576                 if(_currentToken.literalFlag != Token.LiteralFlag.TEMPLATE_STRING)
577                     left = new LiteralNode(_currentToken, ScriptAny(_currentToken.text));
578                 else
579                     left = parseTemplateStringNode();
580                 nextToken();
581                 break;
582             case Token.Type.REGEX:
583                 left = new LiteralNode(_currentToken, ScriptAny(_currentToken.text));
584                 nextToken();
585                 break;
586             case Token.Type.KEYWORD:
587                 if(_currentToken.text == "true" || _currentToken.text == "false")
588                 {
589                     left = new LiteralNode(_currentToken, ScriptAny(to!bool(_currentToken.text)));
590                     nextToken();
591                 }
592                 else if(_currentToken.text == "null")
593                 {
594                     left = new LiteralNode(_currentToken, ScriptAny(null));
595                     nextToken();
596                 }
597                 else if(_currentToken.text == "undefined")
598                 {
599                     left = new LiteralNode(_currentToken, ScriptAny.UNDEFINED);
600                     nextToken();
601                 }
602                 else if(_currentToken.text == "function") // function literal
603                 {
604                     bool isGenerator = false;
605                     nextToken();
606                     if(_currentToken.type == Token.Type.STAR)
607                     {
608                         isGenerator = true;
609                         nextToken();
610                     }
611 
612                     string optionalName = "";
613                     if(_currentToken.type == Token.Type.IDENTIFIER)
614                     {
615                         optionalName = _currentToken.text;
616                         nextToken();
617                     }
618 
619                     if(_currentToken.type != Token.Type.LPAREN)
620                         throw new ScriptCompileException("Argument list expected after anonymous function", 
621                             _currentToken);
622                     nextToken();
623                     string[] argNames = [];
624                     while(_currentToken.type != Token.Type.RPAREN)
625                     {
626                         if(_currentToken.type != Token.Type.IDENTIFIER)
627                             throw new ScriptCompileException("Argument list must be valid identifier", _currentToken);
628                         argNames ~= _currentToken.text;
629                         nextToken();
630                         if(_currentToken.type == Token.Type.COMMA)
631                             nextToken();
632                         else if(_currentToken.type !=  Token.Type.RPAREN)
633                             throw new ScriptCompileException("Missing ')' after argument list", _currentToken);
634                     }
635                     nextToken(); // eat the )
636                     if(_currentToken.type != Token.Type.LBRACE)
637                         throw new ScriptCompileException("Expected '{' before anonymous function body", _currentToken);
638                     nextToken(); // eat the {
639                     _functionContextStack.push(FunctionContext(
640                         isGenerator? FunctionContextType.GENERATOR : FunctionContextType.NORMAL, 
641                         0, 0, []));
642                     auto statements = parseStatements(Token.Type.RBRACE);
643                     _functionContextStack.pop();
644                     nextToken();
645                     // auto func = new ScriptFunction(name, argNames, statements, null);
646                     left = new FunctionLiteralNode(argNames, statements, optionalName, false, isGenerator);
647                 }
648                 else if(_currentToken.text == "class")
649                 {
650                     left = parseClassExpression();
651                 }
652                 else if(_currentToken.text == "new")
653                 {
654                     immutable newToken = _currentToken;
655                     nextToken();
656                     auto expression = parseExpression();
657                     auto fcn = cast(FunctionCallNode)expression;
658                     if(fcn is null)
659                     {
660                         // if this isn't a function call, turn it into one
661                         fcn = new FunctionCallNode(expression, [], true);
662                     }
663                     fcn.returnThis = true;
664                     left = new NewExpressionNode(fcn);                    
665                 }
666                 else if(_currentToken.text == "super")
667                 {
668                     immutable stoken = _currentToken;
669                     if(_baseClassStack.length < 1)
670                         throw new ScriptCompileException("Super expression only allowed in derived classes", stoken);
671                     left = new SuperNode(stoken, _baseClassStack[$-1]);
672                     nextToken();
673                 }
674                 else if(_currentToken.text == "yield")
675                 {
676                     // check context stack
677                     if(_functionContextStack.top.fct != FunctionContextType.GENERATOR)
678                         throw new ScriptCompileException("Yield may only be used in Generator functions", 
679                             _currentToken);
680                     
681                     immutable ytoken = _currentToken;
682                     nextToken();
683                     ExpressionNode expr;
684                     if(_currentToken.type != Token.Type.RBRACE && _currentToken.type != Token.Type.SEMICOLON)
685                         expr = parseExpression();
686                     left = new YieldNode(ytoken, expr);
687                 }
688                 else
689                     throw new ScriptCompileException("Unexpected keyword in primary expression", _currentToken);
690                 break;
691                 // TODO function
692             case Token.Type.IDENTIFIER: {
693                 immutable lookahead = peekToken();
694                 if(lookahead.type == Token.Type.ARROW)
695                 {
696                     left = parseLambda(false);
697                 }
698                 else
699                 {
700                     left = new VarAccessNode(_currentToken);
701                     nextToken();
702                 }
703                 break;
704             }
705             case Token.Type.LBRACKET: // an array
706             {
707                 nextToken(); // eat the [
708                 auto values = parseCommaSeparatedExpressions(Token.Type.RBRACKET);
709                 nextToken(); // eat the ]
710                 left = new ArrayLiteralNode(values);
711                 break;
712             }
713             default:
714                 throw new ScriptCompileException("Unexpected token in primary expression", _currentToken);
715         }
716         return left;
717     }
718     
719     // TODO optimize this to use indexing instead of string building.
720     TemplateStringNode parseTemplateStringNode()
721     {
722         import mildew.lexer: Lexer;
723 
724         enum TSState { LIT, EXPR }
725 
726         size_t textIndex;
727         string currentExpr = "";
728         string currentLiteral = "";
729         ExpressionNode[] nodes;
730         TSState state = TSState.LIT;
731 
732         string peekTwo(in string text, size_t i)
733         {
734             immutable char first = i >= text.length ? '\0' : text[i];
735             immutable char second = i+1 >= text.length ? '\0' : text[i+1];
736             return cast(string)[first,second];
737         }
738 
739         while(textIndex < _currentToken.text.length)
740         {
741             if(state == TSState.LIT)
742             {
743                 if(peekTwo(_currentToken.text, textIndex) == "${")
744                 {
745                     currentExpr = "";
746                     textIndex += 2;
747                     state = TSState.EXPR;
748                     if(currentLiteral.length > 0)
749                         nodes ~= new LiteralNode(_currentToken, ScriptAny(currentLiteral));
750                 }
751                 else
752                 {
753                     currentLiteral ~= _currentToken.text[textIndex++];
754                 }
755             }
756             else
757             {
758                 if(_currentToken.text[textIndex] == '}')
759                 {
760                     currentLiteral = "";
761                     textIndex++;
762                     state = TSState.LIT;
763                     if(currentExpr.length > 0)
764                     {
765                         auto lexer = Lexer(currentExpr);
766                         auto tokens = lexer.tokenize();
767                         auto parser = Parser(tokens);
768                         nodes ~= parser.parseExpression();
769                         if(parser._currentToken.type != Token.Type.EOF)
770                         {
771                             parser._currentToken.position = _currentToken.position;
772                             throw new ScriptCompileException("Unexpected token in template expression", 
773                                     parser._currentToken);
774                         }
775                     }
776                 }
777                 else
778                 {
779                     currentExpr ~= _currentToken.text[textIndex++];
780                 }
781             }
782         }
783         if(state == TSState.EXPR)
784             throw new ScriptCompileException("Unclosed template expression", _currentToken);
785         if(currentLiteral.length > 0)
786             nodes ~= new LiteralNode(_currentToken, ScriptAny(currentLiteral));
787         return new TemplateStringNode(nodes);
788     }
789 
790     /// after class ? extend base this can begin
791     ClassDefinition parseClassDefinition(Token classToken, string className, ExpressionNode baseClass)
792     {
793         if(_currentToken.type != Token.Type.LBRACE)
794             throw new ScriptCompileException("Expected '{' after class", _currentToken);
795         nextToken();
796         FunctionLiteralNode constructor;
797         string[] methodNames;
798         FunctionLiteralNode[] methods;
799         string[] getMethodNames;
800         FunctionLiteralNode[] getMethods;
801         string[] setMethodNames;
802         FunctionLiteralNode[] setMethods;
803         string[] staticMethodNames;
804         FunctionLiteralNode[] staticMethods;
805         enum PropertyType { NONE, GET, SET, STATIC }
806         while(_currentToken.type != Token.Type.RBRACE && _currentToken.type != Token.Type.EOF)
807         {
808             PropertyType ptype = PropertyType.NONE;
809             string currentMethodName = "";
810             // could be a get or set
811             if(_currentToken.isIdentifier("get"))
812             {
813                 ptype = PropertyType.GET;
814                 nextToken();
815             }
816             else if(_currentToken.isIdentifier("set"))
817             {
818                 ptype = PropertyType.SET;
819                 nextToken();
820             }
821             else if(_currentToken.isIdentifier("static"))
822             {
823                 ptype = PropertyType.STATIC;
824                 nextToken();
825             }
826             // then an identifier
827             if(_currentToken.type != Token.Type.IDENTIFIER)
828                 throw new ScriptCompileException("Method names must be valid identifiers", _currentToken);
829             currentMethodName = _currentToken.text;
830             nextToken();
831             // then a (
832             if(_currentToken.type != Token.Type.LPAREN)
833                 throw new ScriptCompileException("Expected '(' after method name", _currentToken);
834             nextToken();
835             string[] argNames;
836             while(_currentToken.type != Token.Type.RPAREN)
837             {
838                 if(_currentToken.type != Token.Type.IDENTIFIER)
839                     throw new ScriptCompileException("Method arguments must be valid identifiers", _currentToken);
840                 argNames ~= _currentToken.text;
841                 nextToken();
842                 if(_currentToken.type == Token.Type.COMMA)
843                     nextToken();
844                 else if(_currentToken.type != Token.Type.RPAREN)
845                     throw new ScriptCompileException("Method arguments must be separated by ','", _currentToken);
846             }
847             nextToken(); // eat the )
848             // then a {
849             if(_currentToken.type != Token.Type.LBRACE)
850                 throw new ScriptCompileException("Method bodies must begin with '{'", _currentToken);
851             nextToken();
852             if(currentMethodName != "constructor")
853                 _functionContextStack.push(FunctionContext(FunctionContextType.METHOD, 0, 0, []));
854             else
855                 _functionContextStack.push(FunctionContext(FunctionContextType.CONSTRUCTOR, 0, 0, []));
856             auto statements = parseStatements(Token.Type.RBRACE);
857             _functionContextStack.pop();
858             nextToken(); // eat }
859             // now we have a method but if this is the constructor
860             if(currentMethodName == "constructor")
861             {
862                 if(ptype != PropertyType.NONE)
863                     throw new ScriptCompileException("Get and set not allowed for constructor", classToken);
864                 if(constructor !is null)
865                     throw new ScriptCompileException("Classes may only have one constructor", classToken);
866                 // if this is extending a class it MUST have ONE super call
867                 if(baseClass !is null)
868                 {
869                     ulong numSupers = 0;
870                     foreach(stmt ; statements)
871                     {
872                         if(auto exprStmt = cast(ExpressionStatementNode)stmt)
873                         {
874                             if(auto fcn = cast(FunctionCallNode)exprStmt.expressionNode)
875                             {
876                                 if(auto supernode = cast(SuperNode)fcn.functionToCall)
877                                     numSupers++;
878                             }
879                         }
880                     }
881                     if(numSupers != 1)
882                         throw new ScriptCompileException("Derived class constructors must have one super call", 
883                                 classToken);
884                 }
885                 constructor = new FunctionLiteralNode(argNames, statements, className, true);
886             }
887             else // it's a normal method or getter/setter
888             {
889                 if(ptype == PropertyType.NONE)
890                 {
891                     auto trueName = currentMethodName;
892                     if(className != "<anonymous class>" && className != "")
893                         trueName = className ~ ".prototype." ~ currentMethodName;
894                     methods ~= new FunctionLiteralNode(argNames, statements, trueName);
895                     methodNames ~= currentMethodName;
896                 }
897                 else if(ptype == PropertyType.GET)
898                 {
899                     getMethods ~= new FunctionLiteralNode(argNames, statements, currentMethodName);
900                     getMethodNames ~= currentMethodName;                    
901                 }
902                 else if(ptype == PropertyType.SET)
903                 {
904                     setMethods ~= new FunctionLiteralNode(argNames, statements, currentMethodName);
905                     setMethodNames ~= currentMethodName;                    
906                 }
907                 else if(ptype == PropertyType.STATIC)
908                 {
909                     auto trueName = currentMethodName;
910                     if(className != "<anonymous class>" && className != "")
911                         trueName = className ~ "." ~ currentMethodName;
912                     staticMethods ~= new FunctionLiteralNode(argNames, statements, trueName);
913                     staticMethodNames ~= currentMethodName;
914                 }
915             }
916         }
917         nextToken(); // eat the class body }
918 
919         // check for duplicate methods
920         bool[string] mnameMap;
921         foreach(mname ; methodNames)
922         {
923             if(mname in mnameMap)
924                 throw new ScriptCompileException("Duplicate methods are not allowed", classToken);
925             mnameMap[mname] = true;
926         }
927 
928         if(constructor is null)
929         {
930             // probably should enforce required super when base class exists
931             constructor = new FunctionLiteralNode([], [], className, true);
932         }
933 
934         if(baseClass !is null)
935             _baseClassStack = _baseClassStack[0..$-1];
936        	return new ClassDefinition(className, constructor, methodNames, methods, getMethodNames, getMethods, 
937 	   		setMethodNames, setMethods, staticMethodNames, staticMethods, baseClass);
938     }
939 
940     ClassLiteralNode parseClassExpression()
941     {
942         immutable classToken = _currentToken;
943         nextToken();
944         string className = "<anonymous class>";
945         if(_currentToken.type == Token.Type.IDENTIFIER)
946         {
947             className = _currentToken.text;
948             nextToken();
949         }
950         ExpressionNode baseClass = null;
951         if(_currentToken.isKeyword("extends"))
952         {
953             nextToken();
954             baseClass = parseExpression(); // let's hope this is an expression that results in a ScriptFunction value
955             _baseClassStack ~= baseClass;
956         }
957         auto classDef = parseClassDefinition(classToken, className, baseClass);
958         return new ClassLiteralNode(classToken, classDef);
959     }
960 
961     /// parses multiple statements until reaching stop
962     StatementNode[] parseStatements(in Token.Type stop)
963     {
964         StatementNode[] statements;
965         while(_currentToken.type != stop && _currentToken.type != Token.Type.EOF)
966         {
967             statements ~= parseStatement();
968             // each statement parse should eat the semicolon or } so there's nothing to do here
969         }
970         return statements;
971     }
972 
973     VarDeclarationStatementNode parseVarDeclarationStatement(bool consumeSemicolon = true)
974     {
975         auto specifier = _currentToken;
976         nextToken();
977         // auto expressions = parseCommaSeparatedExpressions(Token.Type.SEMICOLON);
978         ExpressionNode[] expressions;
979         while(_currentToken.type != Token.Type.SEMICOLON && _currentToken.type != Token.Type.EOF 
980           && !_currentToken.isIdentifier("of") && !_currentToken.isKeyword("in"))
981         {
982             if(_currentToken.type == Token.Type.IDENTIFIER)
983             {
984                 auto varName = _currentToken.text;
985                 nextToken();
986                 if(_currentToken.type == Token.Type.ASSIGN)
987                 {
988                     nextToken();
989                     auto assignment = parseExpression();
990                     expressions ~= new BinaryOpNode(Token.createFakeToken(Token.Type.ASSIGN, "="), 
991                         new VarAccessNode(Token.createFakeToken(Token.Type.IDENTIFIER, varName)),
992                         assignment);
993                 }
994                 else
995                 {
996                     expressions ~= new VarAccessNode(Token.createFakeToken(Token.Type.IDENTIFIER, varName));
997                 }
998             }
999             else if(_currentToken.type == Token.Type.LBRACE || _currentToken.type == Token.Type.LBRACKET)
1000             {
1001                 immutable isObj = _currentToken.type == Token.Type.LBRACE;
1002                 immutable endTokenType = isObj ? Token.Type.RBRACE : Token.Type.RBRACKET;
1003                 nextToken();
1004                 string[] names;
1005                 string remainderName;
1006                 while(_currentToken.type != endTokenType && _currentToken.type != Token.Type.EOF)
1007                 {
1008                     if(_currentToken.type == Token.Type.TDOT)
1009                     {
1010                         nextToken();
1011                         if(_currentToken.type != Token.Type.IDENTIFIER)
1012                             throw new ScriptCompileException("Remainder must be identifier", _currentToken);
1013                         if(remainderName != "")
1014                             throw new ScriptCompileException("Only one remainder allowed", _currentToken);
1015                         remainderName = _currentToken.text;
1016                     }
1017                     else if(_currentToken.type == Token.Type.IDENTIFIER)
1018                     {
1019                         auto currentName = _currentToken.text;
1020                         names ~= currentName;
1021                     }
1022                     nextToken();
1023                     if(_currentToken.type == Token.Type.COMMA)
1024                         nextToken(); // eat the ,
1025                     else if(_currentToken.type != endTokenType && _currentToken.type != Token.Type.EOF)
1026                         throw new ScriptCompileException("Destructuring variable names must be separated by comma",
1027                             _currentToken);
1028                 }
1029                 if(names.length == 0 && remainderName == "")
1030                     throw new ScriptCompileException("Destructuring declaration cannot be empty", _currentToken);
1031                 nextToken(); // eat the } or ]
1032                 if(_currentToken.type != Token.Type.ASSIGN)
1033                     throw new ScriptCompileException("Destructuring declarations must be assignments", _currentToken);
1034                 immutable assignToken = _currentToken;
1035                 nextToken(); // eat the =
1036                 auto assignment = parseExpression();
1037                 expressions ~= new BinaryOpNode(assignToken, new DestructureTargetNode(names, remainderName, isObj),
1038                     assignment);
1039             }
1040 
1041             if(_currentToken.type == Token.Type.COMMA)
1042             {
1043                 nextToken();
1044             }
1045             else if(_currentToken.type != Token.Type.SEMICOLON && _currentToken.type != Token.Type.EOF
1046                 && !_currentToken.isIdentifier("of") && !_currentToken.isKeyword("in"))
1047             {
1048                 throw new ScriptCompileException("Expected ',' between variable declarations (or missing ';')", 
1049                     _currentToken);
1050             }
1051         }
1052         // make sure all expressions are valid BinaryOpNodes or VarAccessNodes
1053         foreach(expression; expressions)
1054         {
1055             if(auto node = cast(BinaryOpNode)expression)
1056             {
1057                 if(!(cast(VarAccessNode)node.leftNode || cast(DestructureTargetNode)node.leftNode))
1058                     throw new ScriptCompileException("Invalid assignment node", _currentToken);
1059                 if(node.opToken.type != Token.Type.ASSIGN)
1060                     throw new ScriptCompileException("Invalid assignment statement", node.opToken);
1061             }
1062             else if(!cast(VarAccessNode)expression)
1063             {
1064                 throw new ScriptCompileException("Invalid variable name in declaration", _currentToken);
1065             }
1066         }
1067         if(consumeSemicolon)
1068             nextToken(); // eat semicolon
1069         return new VarDeclarationStatementNode(specifier, expressions);
1070     }
1071 
1072     IfStatementNode parseIfStatement()
1073     {
1074         immutable lineNumber = _currentToken.position.line;
1075         nextToken();
1076         if(_currentToken.type != Token.Type.LPAREN)
1077             throw new ScriptCompileException("Expected '(' after if keyword", _currentToken);
1078         nextToken();
1079         auto condition = parseExpression();
1080         if(_currentToken.type != Token.Type.RPAREN)
1081             throw new ScriptCompileException("Expected ')' after if condition", _currentToken);
1082         nextToken();
1083         auto ifTrueStatement = parseStatement();
1084         StatementNode elseStatement = null;
1085         if(_currentToken.isKeyword("else"))
1086         {
1087             nextToken();
1088             elseStatement = parseStatement();
1089         }
1090         return new IfStatementNode(lineNumber, condition, ifTrueStatement, elseStatement);
1091     }
1092 
1093     StatementNode parseLoopStatement()
1094     {
1095         string label = "";
1096         if(_currentToken.type == Token.Type.LABEL)
1097         {
1098             label = _currentToken.text;
1099             _functionContextStack.top.labelStack ~= label;
1100             nextToken();
1101         }
1102         StatementNode statement;
1103         if(_currentToken.isKeyword("while"))
1104         {
1105             ++_functionContextStack.top.loopStack;
1106             statement = parseWhileStatement(label);
1107             --_functionContextStack.top.loopStack;
1108         }
1109         // check for do-while statement TODO check for label
1110         else if(_currentToken.isKeyword("do"))
1111         {
1112             ++_functionContextStack.top.loopStack;
1113             statement = parseDoWhileStatement(label);
1114             --_functionContextStack.top.loopStack;
1115         }
1116         // check for for loop TODO check label
1117         else if(_currentToken.isKeyword("for"))
1118         {
1119             ++_functionContextStack.top.loopStack;
1120             statement = parseForStatement(label);
1121             --_functionContextStack.top.loopStack;
1122         }
1123         if(label != "")
1124         {
1125             _functionContextStack.top.labelStack = _functionContextStack.top.labelStack[0..$-1];
1126         }
1127         return statement;
1128     }
1129 
1130     WhileStatementNode parseWhileStatement(string label = "")
1131     {
1132         immutable lineNumber = _currentToken.position.line;
1133         nextToken();
1134         if(_currentToken.type != Token.Type.LPAREN)
1135             throw new ScriptCompileException("Expected '(' after while keyword", _currentToken);
1136         nextToken();
1137         auto condition = parseExpression();
1138         if(_currentToken.type != Token.Type.RPAREN)
1139             throw new ScriptCompileException("Expected ')' after while condition", _currentToken);
1140         nextToken();
1141         auto loopBody = parseStatement();
1142         return new WhileStatementNode(lineNumber, condition, loopBody, label);
1143     }
1144 
1145     DoWhileStatementNode parseDoWhileStatement(string label = "")
1146     {
1147         immutable lineNumber = _currentToken.position.line;
1148         nextToken();
1149         auto loopBody = parseStatement();
1150         if(!_currentToken.isKeyword("while"))
1151             throw new ScriptCompileException("Expected while keyword after do statement", _currentToken);
1152         nextToken();
1153         if(_currentToken.type != Token.Type.LPAREN)
1154             throw new ScriptCompileException("Expected '(' before do-while condition", _currentToken);
1155         nextToken();
1156         auto condition = parseExpression();
1157         if(_currentToken.type != Token.Type.RPAREN)
1158             throw new ScriptCompileException("Expected ')' after do-while condition", _currentToken);
1159         nextToken();
1160         if(_currentToken.type != Token.Type.SEMICOLON)
1161             throw new ScriptCompileException("Expected ';' after do-while statement", _currentToken);
1162         nextToken();
1163         return new DoWhileStatementNode(lineNumber, loopBody, condition, label);
1164     }
1165 
1166     StatementNode parseForStatement(string label = "")
1167     {
1168         immutable lineNumber = _currentToken.position.line;
1169         nextToken();
1170         if(_currentToken.type != Token.Type.LPAREN)
1171             throw new ScriptCompileException("Expected '(' after for keyword", _currentToken);
1172         nextToken();
1173         VarDeclarationStatementNode decl = null;
1174         if(_currentToken.type != Token.Type.SEMICOLON)
1175             decl = parseVarDeclarationStatement(false);
1176         if(_currentToken.isKeyword("in") || _currentToken.isIdentifier("of"))
1177         {
1178             immutable ofInToken = _currentToken;
1179             // first we need to validate the VarDeclarationStatementNode to make sure it only consists
1180             // of let or const and VarAccessNodes
1181             if(decl is null)
1182                 throw new ScriptCompileException("Invalid for in statement", _currentToken);
1183             Token qualifier;
1184             VarAccessNode[] vans;
1185             if(decl.qualifier.text != "const" && decl.qualifier.text != "let")
1186                 throw new ScriptCompileException("Global variable declaration invalid in for in statement",
1187                     decl.qualifier);
1188             int vanCount = 0;
1189             foreach(va ; decl.varAccessOrAssignmentNodes)
1190             {
1191                 auto valid = cast(VarAccessNode)va;
1192                 if(valid is null)
1193                     throw new ScriptCompileException("Invalid variable declaration in for in statement", 
1194                         _currentToken);
1195                 vans ~= valid;
1196                 ++vanCount;
1197             }
1198             if(vanCount > 2)
1199                 throw new ScriptCompileException("For in loops may only have one or two variable declarations", 
1200                         _currentToken);
1201             nextToken();
1202             auto objToIterateExpr = parseExpression();
1203             if(_currentToken.type != Token.Type.RPAREN)
1204                 throw new ScriptCompileException("Expected ')' after array or object", _currentToken);
1205             nextToken();
1206             auto bodyStatement = parseStatement();
1207             return new ForOfStatementNode(lineNumber, qualifier, ofInToken, vans, objToIterateExpr, 
1208                     bodyStatement, label);
1209         }
1210         else if(_currentToken.type == Token.Type.SEMICOLON)
1211         {
1212             nextToken();
1213             ExpressionNode condition = null;
1214             if(_currentToken.type != Token.Type.SEMICOLON)
1215             {
1216                 condition = parseExpression();
1217                 if(_currentToken.type != Token.Type.SEMICOLON)
1218                     throw new ScriptCompileException("Expected ';' after for condition", _currentToken);
1219             }
1220             else
1221             {
1222                 condition = new LiteralNode(_currentToken, ScriptAny(true));
1223             }
1224             nextToken();
1225             ExpressionNode increment = null;
1226             if(_currentToken.type != Token.Type.RPAREN)
1227             {
1228                 increment = parseExpression();
1229             }
1230             else
1231             {
1232                 increment = new LiteralNode(_currentToken, ScriptAny(true));
1233             }
1234             if(_currentToken.type != Token.Type.RPAREN)
1235                 throw new ScriptCompileException("Expected ')' before for loop body", _currentToken);
1236             nextToken();
1237             auto bodyNode = parseStatement();
1238             return new ForStatementNode(lineNumber, decl, condition, increment, bodyNode, label);
1239         }
1240         else
1241             throw new ScriptCompileException("Invalid for statement", _currentToken);
1242     }
1243 
1244     FunctionDeclarationStatementNode parseFunctionDeclarationStatement()
1245     {
1246         import std.algorithm: uniq, count;
1247         immutable lineNumber = _currentToken.position.line;
1248         bool isGenerator = false;
1249         nextToken(); // eat the function keyword
1250 
1251         if(_currentToken.type == Token.Type.STAR)
1252         {
1253             isGenerator = true;
1254             nextToken();
1255         }
1256 
1257         if(_currentToken.type != Token.Type.IDENTIFIER)
1258             throw new ScriptCompileException("Expected identifier after function keyword", _currentToken);
1259         string name = _currentToken.text;
1260         nextToken();
1261 
1262         if(_currentToken.type == Token.Type.STAR)
1263         {
1264             if(isGenerator)
1265                 throw new ScriptCompileException("Only one asterisk allowed to described Generators", _currentToken);
1266             isGenerator = true;
1267             nextToken();
1268         }
1269         
1270         if(_currentToken.type != Token.Type.LPAREN)
1271             throw new ScriptCompileException("Expected '(' after function name", _currentToken);
1272         nextToken();
1273         string[] argNames = [];
1274         while(_currentToken.type != Token.Type.RPAREN)
1275         {
1276             if(_currentToken.type != Token.Type.IDENTIFIER)
1277                 throw new ScriptCompileException("Function argument names must be valid identifiers", _currentToken);
1278             argNames ~= _currentToken.text;
1279             nextToken();
1280             if(_currentToken.type == Token.Type.COMMA)
1281                 nextToken();
1282             else if(_currentToken.type != Token.Type.RPAREN)
1283                 throw new ScriptCompileException("Function argument names must be separated by comma", _currentToken);
1284         }
1285         nextToken(); // eat the )
1286 
1287         // make sure there are no duplicate parameter names
1288         if(argNames.uniq.count != argNames.length)
1289             throw new ScriptCompileException("Function argument names must be unique", _currentToken);
1290 
1291         if(_currentToken.type != Token.Type.LBRACE)
1292             throw new ScriptCompileException("Function definition must begin with '{'", _currentToken);
1293         nextToken();
1294         _functionContextStack.push(FunctionContext(
1295             isGenerator? FunctionContextType.GENERATOR: FunctionContextType.NORMAL, 
1296             0, 0, []));
1297         auto statements = parseStatements(Token.Type.RBRACE);
1298         _functionContextStack.pop();
1299         nextToken(); // eat the }
1300         return new FunctionDeclarationStatementNode(lineNumber, name, argNames, statements, isGenerator);
1301     }
1302 
1303     TryCatchBlockStatementNode parseTryCatchBlockStatement()
1304     {
1305         immutable lineNumber = _currentToken.position.line;
1306         auto tryToken = _currentToken;
1307         nextToken(); // eat the 'try'
1308         auto tryBlock = parseStatement();
1309         StatementNode catchBlock = null;
1310         StatementNode finallyBlock = null;
1311         auto name = "";
1312         if(_currentToken.isKeyword("catch"))
1313         {
1314             nextToken(); // eat the catch
1315             if(_currentToken.type == Token.Type.LPAREN)
1316             {
1317                 nextToken(); // eat (
1318                 if(_currentToken.type != Token.Type.IDENTIFIER)
1319                     throw new ScriptCompileException("Name of exception required after '('", _currentToken);
1320                 name = _currentToken.text;
1321                 nextToken();
1322                 if(_currentToken.type != Token.Type.RPAREN)
1323                     throw new ScriptCompileException("')' required after exception name", _currentToken);
1324                 nextToken();
1325             }
1326             catchBlock = parseStatement();
1327         }
1328         if(_currentToken.isKeyword("finally"))
1329         {
1330             nextToken();
1331             finallyBlock = parseStatement();
1332         }
1333         // can't be missing both catch and finally
1334         if(catchBlock is null && finallyBlock is null)
1335             throw new ScriptCompileException("Try-catch blocks must have catch and/or finally block", tryToken);
1336         return new TryCatchBlockStatementNode(lineNumber, tryBlock, name, catchBlock, finallyBlock);
1337     }
1338 
1339     ObjectLiteralNode parseObjectLiteral()
1340     {
1341         nextToken(); // eat the {
1342         string[] keys = [];
1343         ExpressionNode[] valueExpressions = [];
1344         while(_currentToken.type != Token.Type.RBRACE)
1345         {
1346             // first must be an identifier token or string literal token
1347             immutable idToken = _currentToken;
1348             if(_currentToken.type != Token.Type.IDENTIFIER && _currentToken.type != Token.Type.STRING
1349                 && _currentToken.type != Token.Type.LABEL)
1350                 throw new ScriptCompileException("Invalid key for object literal", _currentToken);
1351             keys ~= _currentToken.text;
1352 
1353             nextToken();
1354             // next must be a :
1355             if(idToken.type != Token.Type.LABEL)
1356             {
1357                 if(_currentToken.type != Token.Type.COLON)
1358                     throw new ScriptCompileException("Expected ':' after key in object literal", _currentToken);
1359                 nextToken();
1360             }
1361             // next can be any valid expression
1362             valueExpressions ~= parseExpression();
1363             // if next is not a comma it must be a closing brace to exit
1364             if(_currentToken.type == Token.Type.COMMA)
1365                 nextToken();
1366             else if(_currentToken.type != Token.Type.RBRACE)
1367                 throw new ScriptCompileException("Key value pairs must be separated by ','", _currentToken);
1368         }
1369         nextToken(); // eat the }
1370         if(keys.length != valueExpressions.length)
1371             throw new ScriptCompileException("Number of keys must match values in object literal", _currentToken);
1372         return new ObjectLiteralNode(keys, valueExpressions);
1373     }
1374 
1375     ClassDeclarationStatementNode parseClassDeclaration()
1376     {
1377         immutable lineNumber = _currentToken.position.line;
1378         immutable classToken = _currentToken;
1379         nextToken();
1380         if(_currentToken.type != Token.Type.IDENTIFIER)
1381             throw new ScriptCompileException("Class name must be valid identifier", _currentToken);
1382         auto className = _currentToken.text;
1383         nextToken();
1384         ExpressionNode baseClass = null;
1385         if(_currentToken.isKeyword("extends"))
1386         {
1387             nextToken();
1388             baseClass = parseExpression(); // let's hope this is an expression that results in a ScriptFunction value
1389             _baseClassStack ~= baseClass;
1390         }
1391         auto classDef = parseClassDefinition(classToken, className, baseClass);
1392         return new ClassDeclarationStatementNode(lineNumber, classToken, classDef);
1393     }
1394 
1395     SwitchStatementNode parseSwitchStatement() 
1396     {
1397         ++_functionContextStack.top.switchStack;
1398         immutable lineNumber = _currentToken.position.line;
1399         immutable switchToken = _currentToken;
1400         nextToken();
1401         if(_currentToken.type != Token.Type.LPAREN)
1402             throw new ScriptCompileException("Expected '(' after switch keyword", _currentToken);
1403         nextToken();
1404         auto expression = parseExpression();
1405         if(_currentToken.type != Token.Type.RPAREN)
1406             throw new ScriptCompileException("Expected ')' after switch expression", _currentToken);
1407         nextToken();
1408         if(_currentToken.type != Token.Type.LBRACE)
1409             throw new ScriptCompileException("Expected '{' to begin switch body", _currentToken);
1410         nextToken();
1411         bool caseStarted = false;
1412         size_t statementCounter = 0;
1413         StatementNode[] statementNodes;
1414         size_t defaultStatementID = size_t.max;
1415         size_t[ScriptAny] jumpTable;
1416         while(_currentToken.type != Token.Type.RBRACE)
1417         {
1418             if(_currentToken.isKeyword("case"))
1419             {
1420                 nextToken();
1421                 caseStarted = true;
1422                 auto caseExpression = parseExpression();
1423                 // it has to be evaluatable at compile time
1424                 /*auto vr = caseExpression.accept(interpreter).get!(Interpreter.VisitResult);
1425                 if(vr.exception !is null || vr.result == ScriptAny.UNDEFINED)
1426                     throw new ScriptCompileException("Case expression must be determined at compile time", switchToken);
1427                 */
1428                 auto result = evaluateCTFE(caseExpression);
1429                 if(result == ScriptAny.UNDEFINED)
1430                     throw new ScriptCompileException(
1431                         "Case expression must be determined at compile time and cannot be undefined", 
1432                         switchToken);
1433                 if(_currentToken.type != Token.Type.COLON)
1434                     throw new ScriptCompileException("Expected ':' after case expression", _currentToken);
1435                 nextToken();
1436                 if(result in jumpTable)
1437                     throw new ScriptCompileException("Duplicate case entries not allowed", switchToken);
1438                 jumpTable[result] = statementCounter;
1439             }
1440             else if(_currentToken.isKeyword("default"))
1441             {
1442                 caseStarted = true;
1443                 nextToken();
1444                 if(_currentToken.type != Token.Type.COLON)
1445                     throw new ScriptCompileException("':' expected after default keyword", _currentToken);
1446                 nextToken();
1447                 defaultStatementID = statementCounter;
1448             }
1449             else
1450             {
1451                 if(!caseStarted)
1452                     throw new ScriptCompileException("Case condition required before any statements", _currentToken);
1453                 statementNodes ~= parseStatement();
1454                 ++statementCounter;
1455             }
1456         }
1457         nextToken(); // consume }
1458         --_functionContextStack.top.switchStack;
1459         return new SwitchStatementNode(lineNumber, expression, new SwitchBody(statementNodes, defaultStatementID, 
1460             jumpTable));
1461     }
1462 
1463     LambdaNode parseLambda(bool hasParentheses)
1464     {
1465         string[] argList;
1466         if(hasParentheses)
1467         {
1468             nextToken(); // eat the (
1469             while(_currentToken.type != Token.Type.RPAREN)
1470             {
1471                 if(_currentToken.type != Token.Type.IDENTIFIER)
1472                     throw new ScriptCompileException("Lambda argument names must be valid identifiers", _currentToken);
1473                 argList ~= _currentToken.text;
1474                 nextToken();
1475                 if(_currentToken.type == Token.Type.COMMA)
1476                     nextToken();
1477                 else if(_currentToken.type != Token.Type.RPAREN)
1478                     throw new ScriptCompileException("Lambda argument names must be separated by comma", _currentToken);
1479             }
1480             nextToken(); // eat the )
1481         }
1482         else
1483         {
1484             if(_currentToken.type != Token.Type.IDENTIFIER)
1485                 throw new ScriptCompileException("Lambda argument name must be valid identifier", _currentToken);
1486             argList ~= _currentToken.text;
1487             nextToken();
1488         }
1489         // make sure arrow
1490         if(_currentToken.type != Token.Type.ARROW)
1491             throw new ScriptCompileException("Arrow expected after lambda argument list", _currentToken);
1492         auto arrow = _currentToken;
1493         nextToken();
1494         // either a single expression, or body marked by {
1495         if(_currentToken.type == Token.Type.LBRACE)
1496         {
1497             nextToken(); // eat the {
1498             auto stmts = parseStatements(Token.Type.RBRACE);
1499             nextToken(); // eat the }
1500             return new LambdaNode(arrow, argList, stmts);
1501         }
1502         else
1503         {
1504             auto expr = parseExpression();
1505             return new LambdaNode(arrow, argList, expr);
1506         }
1507     }
1508 
1509     ExpressionNode[] parseCommaSeparatedExpressions(in Token.Type stop)
1510     {
1511         ExpressionNode[] expressions;
1512 
1513         while(_currentToken.type != stop && _currentToken.type != Token.Type.EOF && !_currentToken.isIdentifier("of")
1514           && !_currentToken.isKeyword("in"))
1515         {
1516             auto expression = parseExpression();
1517             expressions ~= expression;
1518             if(_currentToken.type == Token.Type.COMMA)
1519                 nextToken();
1520             else if(_currentToken.type != stop 
1521               && !_currentToken.isIdentifier("of")
1522               && !_currentToken.isKeyword("in"))
1523                 throw new ScriptCompileException("Comma separated list items must be separated by ','" 
1524                     ~ " (or missing '" ~ Token.createFakeToken(stop, "").symbol ~ "')", 
1525                     _currentToken);
1526         }
1527 
1528         return expressions;
1529     }
1530 
1531     void nextToken()
1532     {
1533         if(_tokenIndex >= _tokens.length)
1534             _currentToken = Token(Token.Type.EOF);
1535         else
1536             _currentToken = _tokens[_tokenIndex++];
1537     }
1538 
1539     void putbackToken()
1540     {
1541         if(_tokenIndex > 0)
1542             _currentToken = _tokens[--_tokenIndex];
1543     }
1544 
1545     Token peekToken()
1546     {
1547         return peekTokens(1)[0];
1548     }
1549 
1550     Token[] peekTokens(int numToPeek)
1551     {
1552         Token[] list;
1553         for(size_t i = _tokenIndex; i < _tokenIndex+numToPeek; ++i)
1554         {
1555             if(i < _tokens.length)
1556                 list ~= _tokens[i];
1557             else
1558                 list ~= Token.createFakeToken(Token.Type.EOF, "");
1559         }
1560         return list;
1561     }
1562 
1563     enum FunctionContextType {NORMAL, CONSTRUCTOR, METHOD, GENERATOR}
1564 
1565     struct FunctionContext
1566     {
1567         FunctionContextType fct;
1568         int loopStack;
1569         int switchStack;
1570         string[] labelStack;
1571     }
1572 
1573     Token[] _tokens;
1574     size_t _tokenIndex = 0;
1575     Token _currentToken;
1576     Stack!FunctionContext _functionContextStack;
1577     ExpressionNode[] _baseClassStack; // in case we have nested class declarations
1578 }