some refactor

include/pocketpy/compiler.h

@@ -56,8 +56,8 @@ class Compiler {
     void consume_end_stmt();
 
     /*************************************************/
-    void EXPR(bool push_stack=true);
-    void EXPR_TUPLE(bool push_stack=true);
+    void EXPR();
+    void EXPR_TUPLE();
     Expr_ EXPR_VARS();  // special case for `for loop` and `comp`
 
     template <typename T, typename... Args>
@@ -91,7 +91,6 @@ class Compiler {
     void exprBytes();
     void exprFString();
     void exprLambda();
-    void exprTuple();
     void exprOr();
     void exprAnd();
     void exprTernary();
@@ -104,6 +103,8 @@ class Compiler {
     void exprCall();
     void exprName();
     void exprAttrib();
+    void exprSlice0();
+    void exprSlice1();
     void exprSubscr();
     void exprLiteral0();
 

include/pocketpy/lexer.h

@@ -71,8 +71,7 @@ struct Token{
 
 // https://docs.python.org/3/reference/expressions.html#operator-precedence
 enum Precedence {
-  PREC_NONE,
-  PREC_TUPLE,         // ,
+  PREC_LOWEST,
   PREC_LAMBDA,        // lambda
   PREC_TERNARY,       // ?:
   PREC_LOGICAL_OR,    // or
@@ -93,7 +92,7 @@ enum Precedence {
   PREC_UNARY,         // - not ~
   PREC_EXPONENT,      // **
   PREC_PRIMARY,       // f() x[] a.b 1:2
-  PREC_HIGHEST,pyth
+  PREC_HIGHEST,
 };
 
 enum StringType { NORMAL_STRING, RAW_STRING, F_STRING, NORMAL_BYTES };

python/builtins.py

@@ -266,9 +266,6 @@ def long(*args, **kwargs):
 
 
 # builtin exceptions
-class SyntaxError(Exception): pass
-class IndentationError(SyntaxError): pass
-
 class StackOverflowError(Exception): pass
 class IOError(Exception): pass
 class NotImplementedError(Exception): pass

src/compiler.cpp

@@ -72,7 +72,7 @@ namespace pkpy{
         count += 1;
 // http://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/
 #define PK_METHOD(name) &Compiler::name
-#define PK_NO_INFIX nullptr, PREC_NONE
+#define PK_NO_INFIX nullptr, PREC_LOWEST
         for(TokenIndex i=0; i<kTokenCount; i++) rules[i] = { nullptr, PK_NO_INFIX };
         rules[TK(".")] =        { nullptr,                  PK_METHOD(exprAttrib),         PREC_PRIMARY };
         rules[TK("(")] =        { PK_METHOD(exprGroup),     PK_METHOD(exprCall),           PREC_PRIMARY };
@@ -101,7 +101,6 @@ namespace pkpy{
         rules[TK("^")] =        { nullptr,               PK_METHOD(exprBinaryOp),       PREC_BITWISE_XOR };
         rules[TK("@")] =        { nullptr,               PK_METHOD(exprBinaryOp),       PREC_FACTOR };
         rules[TK("if")] =       { nullptr,               PK_METHOD(exprTernary),        PREC_TERNARY };
-        rules[TK(",")] =        { nullptr,               PK_METHOD(exprTuple),          PREC_TUPLE };
         rules[TK("not in")] =   { nullptr,               PK_METHOD(exprBinaryOp),       PREC_COMPARISION };
         rules[TK("is not")] =   { nullptr,               PK_METHOD(exprBinaryOp),       PREC_COMPARISION };
         rules[TK("and") ] =     { nullptr,               PK_METHOD(exprAnd),            PREC_LOGICAL_AND };
@@ -119,6 +118,8 @@ namespace pkpy{
         rules[TK("@long")] =    { PK_METHOD(exprLong),      PK_NO_INFIX };
         rules[TK("@imag")] =    { PK_METHOD(exprImag),      PK_NO_INFIX };
         rules[TK("@bytes")] =   { PK_METHOD(exprBytes),     PK_NO_INFIX };
+        rules[TK(":")] =        { PK_METHOD(exprSlice0),    PK_METHOD(exprSlice1),      PREC_PRIMARY };
+        
 #undef PK_METHOD
 #undef PK_NO_INFIX
     }
@@ -164,12 +165,23 @@ namespace pkpy{
         if (!match_end_stmt()) SyntaxError("expected statement end");
     }
 
-    void Compiler::EXPR(bool push_stack) {
-        parse_expression(PREC_TUPLE+1, push_stack);
+    void Compiler::EXPR() {
+        parse_expression(PREC_LOWEST+1);
     }
 
-    void Compiler::EXPR_TUPLE(bool push_stack) {
-        parse_expression(PREC_TUPLE, push_stack);
+    void Compiler::EXPR_TUPLE() {
+        EXPR();
+        if(!match(TK(","))) return;
+        // tuple expression
+        std::vector<Expr_> items;
+        items.push_back(ctx()->s_expr.popx());
+        do {
+            if(curr().brackets_level) match_newlines_repl();
+            if(!is_expression()) break;
+            EXPR(); items.push_back(ctx()->s_expr.popx());
+            if(curr().brackets_level) match_newlines_repl();
+        } while(match(TK(",")));
+        ctx()->s_expr.push(make_expr<TupleExpr>(std::move(items)));
     }
 
     // special case for `for loop` and `comp`
@@ -217,21 +229,6 @@ namespace pkpy{
         ctx()->s_expr.push(std::move(e));
     }
 
-    void Compiler::exprTuple(){
-        std::vector<Expr_> items;
-        items.push_back(ctx()->s_expr.popx());
-        do {
-            if(curr().brackets_level) match_newlines_repl();
-            if(!is_expression()) break;
-            EXPR();
-            items.push_back(ctx()->s_expr.popx());
-            if(curr().brackets_level) match_newlines_repl();
-        } while(match(TK(",")));
-        ctx()->s_expr.push(make_expr<TupleExpr>(
-            std::move(items)
-        ));
-    }
-
     void Compiler::exprOr(){
         auto e = make_expr<OrExpr>();
         e->lhs = ctx()->s_expr.popx();
@@ -418,72 +415,42 @@ namespace pkpy{
             make_expr<AttribExpr>(ctx()->s_expr.popx(), StrName::get(prev().sv()))
         );
     }
+
+    void Compiler::exprSlice0() {
+        auto slice = make_expr<SliceExpr>();
+        if(is_expression()){    // :<expr>
+            EXPR();
+            slice->stop = ctx()->s_expr.popx();
+            // try optional step
+            if(match(TK(":"))){
+                EXPR();
+                slice->step = ctx()->s_expr.popx();
+            }
+        }
+        ctx()->s_expr.push(std::move(slice));
+    }
+
+    void Compiler::exprSlice1() {
+        auto slice = make_expr<SliceExpr>();
+        slice->start = ctx()->s_expr.popx();
+        if(is_expression()){    // <expr>:<expr>
+            EXPR();
+            slice->stop = ctx()->s_expr.popx();
+            // try optional step
+            if(match(TK(":"))){
+                EXPR();
+                slice->step = ctx()->s_expr.popx();
+            }
+        }
+        ctx()->s_expr.push(std::move(slice));
+    }
     
     void Compiler::exprSubscr() {
         auto e = make_expr<SubscrExpr>();
         e->a = ctx()->s_expr.popx();        // a[...]
-        auto slice = make_expr<SliceExpr>();
-        bool is_slice = false;
-        // a[<0> <state:1> : state<3> : state<5>]
-        int state = 0;
-        do{
-            match_newlines_repl();
-            switch(state){
-                case 0:
-                    if(match(TK(":"))){
-                        is_slice=true;
-                        state=2;
-                        break;
-                    }
-                    if(match(TK("]"))) SyntaxError();
-                    EXPR_TUPLE();
-                    slice->start = ctx()->s_expr.popx();
-                    state=1;
-                    break;
-                case 1:
-                    if(match(TK(":"))){
-                        is_slice=true;
-                        state=2;
-                        break;
-                    }
-                    if(match(TK("]"))) goto __SUBSCR_END;
-                    SyntaxError("expected ':' or ']'");
-                    break;
-                case 2:
-                    if(match(TK(":"))){
-                        state=4;
-                        break;
-                    }
-                    if(match(TK("]"))) goto __SUBSCR_END;
-                    EXPR_TUPLE();
-                    slice->stop = ctx()->s_expr.popx();
-                    state=3;
-                    break;
-                case 3:
-                    if(match(TK(":"))){
-                        state=4;
-                        break;
-                    }
-                    if(match(TK("]"))) goto __SUBSCR_END;
-                    SyntaxError("expected ':' or ']'");
-                    break;
-                case 4:
-                    if(match(TK("]"))) goto __SUBSCR_END;
-                    EXPR_TUPLE();
-                    slice->step = ctx()->s_expr.popx();
-                    state=5;
-                    break;
-                case 5: consume(TK("]")); goto __SUBSCR_END;
-            }
-            match_newlines_repl();
-        }while(true);
-__SUBSCR_END:
-        if(is_slice){
-            e->b = std::move(slice);
-        }else{
-            PK_ASSERT(state == 1)
-            e->b = std::move(slice->start);
-        }
+        EXPR_TUPLE();                       // a[<expr>]
+        e->b = ctx()->s_expr.popx();
+        consume(TK("]"));
         ctx()->s_expr.push(std::move(e));
     }
 
@@ -596,15 +563,18 @@ __EAT_DOTS_END:
 
     bool Compiler::is_expression(){
         PrattCallback prefix = rules[curr().type].prefix;
-        return prefix != nullptr;
+        // slice expression is restricted to be used in subscript
+        return prefix != nullptr && curr().type != TK(":");
     }
 
     void Compiler::parse_expression(int precedence, bool push_stack) {
         PrattCallback prefix = rules[curr().type].prefix;
-        if (prefix == nullptr) SyntaxError(Str("expected an expression, got ") + TK_STR(curr().type));
+        if (prefix == nullptr || curr().type == TK(":")){
+            SyntaxError(Str("expected an expression, got ") + TK_STR(curr().type));
+        }
         advance();
         (this->*prefix)();
-        while (rules[curr().type].precedence >= precedence) {
+        while (rules[curr().type].precedence >= precedence && curr().type != TK(":")) {
             TokenIndex op = curr().type;
             advance();
             PrattCallback infix = rules[op].infix;
@@ -615,7 +585,8 @@ __EAT_DOTS_END:
     }
 
     void Compiler::compile_if_stmt() {
-        EXPR(false);   // condition
+        EXPR();   // condition
+        ctx()->emit_expr();
         int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line);
         compile_block_body();
         if (match(TK("elif"))) {
@@ -635,7 +606,8 @@ __EAT_DOTS_END:
 
     void Compiler::compile_while_loop() {
         CodeBlock* block = ctx()->enter_block(CodeBlockType::WHILE_LOOP);
-        EXPR(false);   // condition
+        EXPR();   // condition
+        ctx()->emit_expr();
         int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line);
         compile_block_body();
         ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE);
@@ -651,7 +623,7 @@ __EAT_DOTS_END:
     void Compiler::compile_for_loop() {
         Expr_ vars = EXPR_VARS();
         consume(TK("in"));
-        EXPR_TUPLE(false);
+        EXPR_TUPLE(); ctx()->emit_expr();
         ctx()->emit_(OP_GET_ITER, BC_NOARG, BC_KEEPLINE);
         CodeBlock* block = ctx()->enter_block(CodeBlockType::FOR_LOOP);
         ctx()->emit_(OP_FOR_ITER, BC_NOARG, BC_KEEPLINE);
@@ -681,7 +653,8 @@ __EAT_DOTS_END:
                 StrName as_name;
                 consume(TK("except"));
                 if(is_expression()){
-                    EXPR(false);      // push assumed type on to the stack
+                    EXPR();      // push assumed type on to the stack
+                    ctx()->emit_expr();
                     ctx()->emit_(OP_EXCEPTION_MATCH, BC_NOARG, prev().line);
                     if(match(TK("as"))){
                         consume(TK("@id"));
@@ -802,7 +775,7 @@ __EAT_DOTS_END:
                 break;
             case TK("yield"): 
                 if (contexts.size() <= 1) SyntaxError("'yield' outside function");
-                EXPR_TUPLE(false);
+                EXPR_TUPLE(); ctx()->emit_expr();
                 // if yield present, mark the function as generator
                 ctx()->co->is_generator = true;
                 ctx()->emit_(OP_YIELD_VALUE, BC_NOARG, kw_line);
@@ -810,7 +783,7 @@ __EAT_DOTS_END:
                 break;
             case TK("yield from"):
                 if (contexts.size() <= 1) SyntaxError("'yield from' outside function");
-                EXPR_TUPLE(false);
+                EXPR_TUPLE(); ctx()->emit_expr();
                 // if yield from present, mark the function as generator
                 ctx()->co->is_generator = true;
                 ctx()->emit_(OP_GET_ITER, BC_NOARG, kw_line);
@@ -826,7 +799,7 @@ __EAT_DOTS_END:
                 if(match_end_stmt()){
                     ctx()->emit_(OP_RETURN_VALUE, 1, kw_line);
                 }else{
-                    EXPR_TUPLE(false);
+                    EXPR_TUPLE(); ctx()->emit_expr();
                     // check if it is a generator
                     if(ctx()->co->is_generator) SyntaxError("'return' with argument inside generator function");
                     consume_end_stmt();
@@ -878,11 +851,13 @@ __EAT_DOTS_END:
                 break;
             }
             case TK("assert"):{
-                EXPR(false);    // condition
+                EXPR();    // condition
+                ctx()->emit_expr();
                 int index = ctx()->emit_(OP_POP_JUMP_IF_TRUE, BC_NOARG, kw_line);
                 int has_msg = 0;
                 if(match(TK(","))){
-                    EXPR(false);    // message
+                    EXPR();    // message
+                    ctx()->emit_expr();
                     has_msg = 1;
                 }
                 ctx()->emit_(OP_RAISE_ASSERT, has_msg, kw_line);
@@ -898,7 +873,7 @@ __EAT_DOTS_END:
                 consume_end_stmt();
                 break;
             case TK("raise"): {
-                EXPR(false);
+                EXPR(); ctx()->emit_expr();
                 ctx()->emit_(OP_RAISE, BC_NOARG, kw_line);
                 consume_end_stmt();
             } break;
@@ -910,7 +885,8 @@ __EAT_DOTS_END:
                 consume_end_stmt();
             } break;
             case TK("with"): {
-                EXPR(false);    // [ <expr> ]
+                EXPR();    // [ <expr> ]
+                ctx()->emit_expr();
                 ctx()->enter_block(CodeBlockType::CONTEXT_MANAGER);
                 Expr_ as_name;
                 if(match(TK("as"))){
@@ -1197,7 +1173,7 @@ __EAT_DOTS_END:
         match_newlines();   // skip possible leading '\n'
 
         if(mode()==EVAL_MODE) {
-            EXPR_TUPLE(false);
+            EXPR_TUPLE(); ctx()->emit_expr();
             consume(TK("@eof"));
             ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE);
             pop_context();

src/vm.cpp

@@ -709,6 +709,10 @@ void VM::init_builtin_types(){
     if(tp_staticmethod != _new_type_object("staticmethod")) exit(-3);
     if(tp_classmethod != _new_type_object("classmethod")) exit(-3);
 
+    // SyntaxError and IndentationError must be created here
+    Type tp_syntax_error = _new_type_object("SyntaxError", tp_exception, true);
+    Type tp_indentation_error = _new_type_object("IndentationError", tp_syntax_error, true);
+
     this->None = heap._new<Dummy>(_new_type_object("NoneType"));
     this->NotImplemented = heap._new<Dummy>(_new_type_object("NotImplementedType"));
     this->Ellipsis = heap._new<Dummy>(_new_type_object("ellipsis"));
@@ -735,6 +739,8 @@ void VM::init_builtin_types(){
     builtins->attr().set("NotImplemented", NotImplemented);
     builtins->attr().set("slice", _t(tp_slice));
     builtins->attr().set("Exception", _t(tp_exception));
+    builtins->attr().set("SyntaxError", _t(tp_syntax_error));
+    builtins->attr().set("IndentationError", _t(tp_indentation_error));
 
     post_init();
     this->_main = new_module("__main__");
@@ -1164,7 +1170,7 @@ PyObject* VM::bind(PyObject* obj, const char* sig, const char* docstring, Native
     CodeObject_ co;
     try{
         // fn(a, b, *c, d=1) -> None
-        co = compile("def " + Str(sig) + " : pass", "<bind>", EXEC_MODE);
+        co = compile(fmt("def ", sig, " : pass"), "<bind>", EXEC_MODE);
     }catch(const Exception&){
         throw std::runtime_error("invalid signature: " + std::string(sig));
     }