some fix

build_g.sh

@@ -2,6 +2,6 @@ python prebuild.py
 
 SRC=$(find src/ -name "*.cpp")
 
-FLAGS="-std=c++17 -Og -stdlib=libc++ -Iinclude -frtti -Wfatal-errors -g -DDEBUG"
+FLAGS="-std=c++17 -O0 -stdlib=libc++ -Iinclude -frtti -Wfatal-errors -g -DDEBUG"
 
 clang++ $FLAGS -o main src2/main.cpp $SRC

include/pocketpy/common/config.h

@@ -37,6 +37,7 @@
 #define PK_DEBUG_MEMORY_POOL        0
 #define PK_DEBUG_NO_AUTO_GC         0
 #define PK_DEBUG_GC_STATS           0
+#define PK_DEBUG_COMPILER           0
 
 #ifndef PK_DEBUG_PRECOMPILED_EXEC
 #define PK_DEBUG_PRECOMPILED_EXEC   0

include/pocketpy/common/vector.hpp

@@ -165,7 +165,10 @@ struct vector {
 
     int capacity() const { return _capacity; }
 
-    T& back() { return _data[_size - 1]; }
+    T& back() {
+        assert(_size > 0);
+        return _data[_size - 1];
+    }
 
     T* begin() const { return _data; }
 
@@ -173,9 +176,15 @@ struct vector {
 
     T* data() const { return _data; }
 
-    T& operator[] (int i) { return _data[i]; }
+    T& operator[] (int i) {
+        assert(i >= 0 && i < _size);
+        return _data[i];
+    }
 
-    const T& operator[] (int i) const { return _data[i]; }
+    const T& operator[] (int i) const {
+        assert(i >= 0 && i < _size);
+        return _data[i];
+    }
 
     void clear() {
         std::destroy(begin(), end());

include/pocketpy/compiler/compiler.hpp

@@ -24,23 +24,33 @@ struct Compiler {
     VM* vm;
     bool unknown_global_scope;  // for eval/exec() call
     // for parsing token stream
-    int i = 0;
+    int __i = 0;
 
     const Token& tk(int i) const noexcept{ return lexer.nexts[i]; }
-    const Token& prev() const noexcept{ return tk(i - 1); }
-    const Token& curr() const noexcept{ return tk(i); }
-    const Token& next() const noexcept{ return tk(i + 1); }
+    const Token& prev() const noexcept{ return tk(__i - 1); }
+    const Token& curr() const noexcept{ return tk(__i); }
+    const Token& next() const noexcept{ return tk(__i + 1); }
 
     const Token& err() const noexcept{
-        if(i >= lexer.nexts.size()) return prev();
+        if(__i >= lexer.nexts.size()) return prev();
         return curr();
     }
 
-    void advance(int delta = 1) noexcept{ i += delta; }
+    void advance(int delta = 1) noexcept{
+        __i += delta;
+#if PK_DEBUG_COMPILER
+        if(__i>=0 && __i<lexer.nexts.size()){
+            printf("%s:%d %s %s\n",
+                lexer.src->filename.c_str(),
+                curr().line,
+                TK_STR(curr().type),
+                curr().str().escape().c_str()
+            );
+        }
+#endif
+    }
 
     CodeEmitContext* ctx() noexcept{ return &contexts.back(); }
-    vector<Expr*>& s_expr() noexcept{ return ctx()->s_expr; }
-
     CompileMode mode() const noexcept{ return lexer.src->mode; }
 
     NameScope name_scope() const noexcept;

include/pocketpy/compiler/expr.hpp

@@ -60,7 +60,7 @@ struct CodeEmitContext{
     VM* vm;
     FuncDecl_ func;  // optional
     CodeObject_ co;  // 1 CodeEmitContext <=> 1 CodeObject_
-    vector<Expr*> s_expr;
+    vector<Expr*> _s_expr;
     int level;
     vector<StrName> global_names;
 
@@ -75,8 +75,6 @@ struct CodeEmitContext{
     int get_loop() const noexcept;
     CodeBlock* enter_block(CodeBlockType type) noexcept;
     void exit_block() noexcept;
-    void emit_expr(bool emit = true) noexcept;  // clear the expression stack and generate bytecode
-    void emit_decorators(int count) noexcept;
     int emit_(Opcode opcode, uint16_t arg, int line, bool is_virtual = false) noexcept;
     void revert_last_emit_() noexcept;
     int emit_int(i64 value, int line) noexcept;
@@ -88,6 +86,59 @@ struct CodeEmitContext{
     int add_func_decl(FuncDecl_ decl) noexcept;
     void emit_store_name(NameScope scope, StrName name, int line) noexcept;
     void try_merge_for_iter_store(int) noexcept;
+    // emit top -> pop -> delete
+    void s_emit_top() noexcept{
+        s_debug_info("s_emit_top");
+        Expr* e = _s_expr.popx_back();
+        e->emit_(this);
+        delete_expr(e);
+    }
+    // push
+    void s_push(Expr* expr) noexcept{
+        s_debug_info("s_push");
+        _s_expr.push_back(expr);
+    }
+    // top
+    Expr* s_top() noexcept{
+        return _s_expr.back();
+    }
+    // size
+    int s_size() const noexcept{
+        return _s_expr.size();
+    }
+    // pop -> delete
+    void s_pop() noexcept{
+        s_debug_info("s_pop");
+        Expr* e = _s_expr.popx_back();
+        delete_expr(e);
+    }
+    // pop move
+    Expr* s_popx() noexcept{
+        s_debug_info("s_popx");
+        return _s_expr.popx_back();
+    }
+    // clean
+    void s_clean() noexcept{
+        s_debug_info("s_clean");
+        for(Expr* e: _s_expr) delete_expr(e);
+        _s_expr.clear();
+    }
+    // emit decorators
+    void s_emit_decorators(int count) noexcept;
+    // debug stack
+#if PK_DEBUG_COMPILER
+    void s_debug_info(const char* op) noexcept{
+        SStream ss;
+        for(int i=0; i<s_size(); i++) {
+            Expr* e = _s_expr[i];
+            ss << typeid(*e).name();
+            if(i != s_size() - 1) ss << ", ";
+        }
+        printf("[%s] %s\n", ss.str().c_str(), op);
+    }
+#else
+    void s_debug_info(const char*) noexcept{}
+#endif
 };
 
 struct NameExpr : Expr {

include/pocketpy/compiler/lexer.hpp

@@ -103,6 +103,7 @@ struct Lexer {
     vector<Token> nexts;
     small_vector_2<int, 8> indents;
     int brackets_level = 0;
+    bool used = false;
 
     char peekchar() const noexcept { return *curr_char; }
 

include/pocketpy/objects/error.hpp

@@ -33,14 +33,14 @@ struct Exception {
     PyObject* _self;  // weak reference
 
     struct Frame {
-        SourceData* src;        // weak ref
+        std::shared_ptr<SourceData> src;        // weak ref
         int lineno;
         const char* cursor;
         std::string name;
 
         Str snapshot() const { return src->snapshot(lineno, cursor, name); }
 
-        Frame(SourceData* src, int lineno, const char* cursor, std::string_view name) :
+        Frame(std::shared_ptr<SourceData> src, int lineno, const char* cursor, std::string_view name) :
             src(src), lineno(lineno), cursor(cursor), name(name) {}
     };
 
@@ -79,7 +79,7 @@ struct TopLevelException : std::exception {
 
 struct Error{
     const char* type;
-    SourceData* src;
+    std::shared_ptr<SourceData> src;
     int lineno;
     const char* cursor;
     char msg[100];

scripts/run_tests.py

@@ -39,13 +39,7 @@ def test_dir(path):
 print('CPython:', str(sys.version).replace('\n', ''))
 print('System:', '64-bit' if sys.maxsize > 2**32 else '32-bit')
 
-if len(sys.argv) == 2:
-    assert 'benchmark' in sys.argv[1]
-    test_dir('benchmarks/')
-else:
-    test_dir('tests/')
-
-    # test interactive mode
+def test_repl():
     print("[REPL Test Enabled]")
     if sys.platform in ['linux', 'darwin']:
         cmd = './main'
@@ -74,4 +68,13 @@ exit()
         assert 'ans_1: 3' in res.stdout, res.stdout
         assert 'ans_2: abc' in res.stdout, res.stdout
 
+
+if len(sys.argv) == 2:
+    assert 'benchmark' in sys.argv[1]
+    test_dir('benchmarks/')
+else:
+    test_dir('tests/')
+    # test_repl()
+
+
 print("ALL TESTS PASSED")

src/common/str.cpp

@@ -391,9 +391,10 @@ StrName StrName::get(std::string_view s) {
 
 Str SStream::str() {
     // after this call, the buffer is no longer valid
-    buffer.reserve(buffer.size() + 1);  // allocate one more byte for '\0'
-    buffer[buffer.size()] = '\0';       // set '\0'
-    return Str(buffer.detach());
+    buffer.push_back('\0');
+    auto detached = buffer.detach();
+    detached.second--;  // remove the last '\0'
+    return Str(detached);
 }
 
 SStream& SStream::operator<< (const Str& s) {

src/compiler/compiler.cpp

@@ -6,7 +6,7 @@ namespace pkpy {
 
 #define consume(expected) if(!match(expected)) return SyntaxError("expected '%s', got '%s'", TK_STR(expected), TK_STR(curr().type));
 #define consume_end_stmt() if(!match_end_stmt()) return SyntaxError("expected statement end")
-#define check(B) err = B; if(err) return err
+#define check(B) if((err = B)) return err
 
 PrattRule Compiler::rules[kTokenCount];
 
@@ -18,7 +18,7 @@ NameScope Compiler::name_scope() const noexcept{
 
 CodeObject_ Compiler::push_global_context() noexcept{
     CodeObject_ co = std::make_shared<CodeObject>(lexer.src, lexer.src->filename);
-    co->start_line = i == 0 ? 1 : prev().line;
+    co->start_line = __i == 0 ? 1 : prev().line;
     contexts.push_back(CodeEmitContext(vm, co, contexts.size()));
     return co;
 }
@@ -26,7 +26,7 @@ CodeObject_ Compiler::push_global_context() noexcept{
 FuncDecl_ Compiler::push_f_context(Str name) noexcept{
     FuncDecl_ decl = std::make_shared<FuncDecl>();
     decl->code = std::make_shared<CodeObject>(lexer.src, name);
-    decl->code->start_line = i == 0 ? 1 : prev().line;
+    decl->code->start_line = __i == 0 ? 1 : prev().line;
     decl->nested = name_scope() == NAME_LOCAL;
     contexts.push_back(CodeEmitContext(vm, decl->code, contexts.size()));
     contexts.back().func = decl;
@@ -34,13 +34,13 @@ FuncDecl_ Compiler::push_f_context(Str name) noexcept{
 }
 
 Error* Compiler::pop_context() noexcept{
-    assert(s_expr().empty());
+    assert(ctx()->s_size() == 0);
     // add a `return None` in the end as a guard
     // previously, we only do this if the last opcode is not a return
     // however, this is buggy...since there may be a jump to the end (out of bound) even if the last opcode is a return
     ctx()->emit_(OP_RETURN_VALUE, 1, BC_KEEPLINE, true);
     // find the last valid token
-    int j = i - 1;
+    int j = __i - 1;
     while(tk(j).type == TK("@eol") || tk(j).type == TK("@dedent") || tk(j).type == TK("@eof"))
         j--;
     ctx()->co->end_line = tk(j).line;
@@ -205,8 +205,8 @@ Error* Compiler::EXPR_TUPLE(bool allow_slice) noexcept{
     } while(match(TK(",")));
     TupleExpr* e = make_expr<TupleExpr>(count);
     for(int i=count-1; i>=0; i--)
-        e->items[i] = s_expr().popx_back();
-    s_expr().push_back(e);
+        e->items[i] = ctx()->s_popx();
+    ctx()->s_push(e);
     return NULL;
 }
 
@@ -214,40 +214,40 @@ Error* Compiler::EXPR_VARS() noexcept{
     int count = 0;
     do {
         consume(TK("@id"));
-        s_expr().push_back(make_expr<NameExpr>(prev().str(), name_scope()));
+        ctx()->s_push(make_expr<NameExpr>(prev().str(), name_scope()));
         count += 1;
     } while(match(TK(",")));
     if(count > 1){
         TupleExpr* e = make_expr<TupleExpr>(count);
         for(int i=count-1; i>=0; i--)
-            e->items[i] = s_expr().popx_back();
-        s_expr().push_back(e);
+            e->items[i] = ctx()->s_popx();
+        ctx()->s_push(e);
     }
     return NULL;
 }
 
 Error* Compiler::exprLiteral() noexcept{
-    s_expr().push_back(make_expr<LiteralExpr>(prev().value));
+    ctx()->s_push(make_expr<LiteralExpr>(prev().value));
     return NULL;
 }
 
 Error* Compiler::exprLong() noexcept{
-    s_expr().push_back(make_expr<LongExpr>(prev().str()));
+    ctx()->s_push(make_expr<LongExpr>(prev().str()));
     return NULL;
 }
 
 Error* Compiler::exprImag() noexcept{
-    s_expr().push_back(make_expr<ImagExpr>(std::get<f64>(prev().value)));
+    ctx()->s_push(make_expr<ImagExpr>(std::get<f64>(prev().value)));
     return NULL;
 }
 
 Error* Compiler::exprBytes() noexcept{
-    s_expr().push_back(make_expr<BytesExpr>(std::get<Str>(prev().value)));
+    ctx()->s_push(make_expr<BytesExpr>(std::get<Str>(prev().value)));
     return NULL;
 }
 
 Error* Compiler::exprFString() noexcept{
-    s_expr().push_back(make_expr<FStringExpr>(std::get<Str>(prev().value)));
+    ctx()->s_push(make_expr<FStringExpr>(std::get<Str>(prev().value)));
     return NULL;
 }
 
@@ -261,38 +261,38 @@ Error* Compiler::exprLambda() noexcept{
     }
     // https://github.com/pocketpy/pocketpy/issues/37
     check(parse_expression(PREC_LAMBDA + 1));
-    ctx()->emit_expr();
+    ctx()->s_emit_top();
     ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE);
     check(pop_context());
     LambdaExpr* e = make_expr<LambdaExpr>(decl);
     e->line = line;
-    s_expr().push_back(e);
+    ctx()->s_push(e);
     return NULL;
 }
 
 Error* Compiler::exprOr() noexcept{
     auto e = make_expr<OrExpr>();
-    e->lhs = s_expr().popx_back();
+    e->lhs = ctx()->s_popx();
     Error* err = parse_expression(PREC_LOGICAL_OR + 1);
     if(err){
         delete_expr(e);
         return err;
     }
-    e->rhs = s_expr().popx_back();
-    s_expr().push_back(e);
+    e->rhs = ctx()->s_popx();
+    ctx()->s_push(e);
     return NULL;
 }
 
 Error* Compiler::exprAnd() noexcept{
     auto e = make_expr<AndExpr>();
-    e->lhs = s_expr().popx_back();
+    e->lhs = ctx()->s_popx();
     Error* err = parse_expression(PREC_LOGICAL_AND + 1);
     if(err){
         delete_expr(e);
         return err;
     }
-    e->rhs = s_expr().popx_back();
-    s_expr().push_back(e);
+    e->rhs = ctx()->s_popx();
+    ctx()->s_push(e);
     return NULL;
 }
 
@@ -305,31 +305,31 @@ Error* Compiler::exprTernary() noexcept{
     check(parse_expression(PREC_TERNARY + 1));  // [true_expr, cond, false_expr]
     auto e = make_expr<TernaryExpr>();
     e->line = line;
-    e->false_expr = s_expr().popx_back();
-    e->cond = s_expr().popx_back();
-    e->true_expr = s_expr().popx_back();
-    s_expr().push_back(e);
+    e->false_expr = ctx()->s_popx();
+    e->cond = ctx()->s_popx();
+    e->true_expr = ctx()->s_popx();
+    ctx()->s_push(e);
     return NULL;
 }
 
 Error* Compiler::exprBinaryOp() noexcept{
     auto e = make_expr<BinaryExpr>(prev().type);
-    e->lhs = s_expr().popx_back();
+    e->lhs = ctx()->s_popx();
     Error* err = parse_expression(rules[e->op].precedence + 1);
     if(err){
         delete_expr(e);
         return err;
     }
-    e->rhs = s_expr().popx_back();
-    s_expr().push_back(std::move(e));
+    e->rhs = ctx()->s_popx();
+    ctx()->s_push(std::move(e));
     return NULL;
 }
 
 Error* Compiler::exprNot() noexcept{
     Error* err;
     check(parse_expression(PREC_LOGICAL_NOT + 1));
-    NotExpr* e = make_expr<NotExpr>(s_expr().popx_back());
-    s_expr().push_back(e);
+    NotExpr* e = make_expr<NotExpr>(ctx()->s_popx());
+    ctx()->s_push(e);
     return NULL;
 }
 
@@ -338,10 +338,10 @@ Error* Compiler::exprUnaryOp() noexcept{
     TokenIndex op = prev().type;
     check(parse_expression(PREC_UNARY + 1));
     switch(op) {
-        case TK("-"): s_expr().push_back(make_expr<NegatedExpr>(s_expr().popx_back())); break;
-        case TK("~"): s_expr().push_back(make_expr<InvertExpr>(s_expr().popx_back())); break;
-        case TK("*"): s_expr().push_back(make_expr<StarredExpr>(s_expr().popx_back(), 1)); break;
-        case TK("**"): s_expr().push_back(make_expr<StarredExpr>(s_expr().popx_back(), 2)); break;
+        case TK("-"): ctx()->s_push(make_expr<NegatedExpr>(ctx()->s_popx())); break;
+        case TK("~"): ctx()->s_push(make_expr<InvertExpr>(ctx()->s_popx())); break;
+        case TK("*"): ctx()->s_push(make_expr<StarredExpr>(ctx()->s_popx(), 1)); break;
+        case TK("**"): ctx()->s_push(make_expr<StarredExpr>(ctx()->s_popx(), 2)); break;
         default: assert(false);
     }
     return NULL;
@@ -353,9 +353,9 @@ Error* Compiler::exprGroup() noexcept{
     check(EXPR_TUPLE());  // () is just for change precedence
     match_newlines_repl();
     consume(TK(")"));
-    if(s_expr().back()->is_tuple()) return NULL;
-    Expr* g = make_expr<GroupedExpr>(s_expr().popx_back());
-    s_expr().push_back(g);
+    if(ctx()->s_top()->is_tuple()) return NULL;
+    Expr* g = make_expr<GroupedExpr>(ctx()->s_popx());
+    ctx()->s_push(g);
     return NULL;
 }
 
@@ -372,11 +372,11 @@ Error* Compiler::consume_comp(Opcode op0, Opcode op1) noexcept{
         has_cond = true;
     }
     CompExpr* ce = make_expr<CompExpr>(op0, op1);
-    if(has_cond) ce->cond = s_expr().popx_back();
-    ce->iter = s_expr().popx_back();
-    ce->vars = s_expr().popx_back();
-    ce->expr = s_expr().popx_back();
-    s_expr().push_back(ce);
+    if(has_cond) ce->cond = ctx()->s_popx();
+    ce->iter = ctx()->s_popx();
+    ce->vars = ctx()->s_popx();
+    ce->expr = ctx()->s_popx();
+    ctx()->s_push(ce);
     match_newlines_repl();
     return NULL;
 }
@@ -401,8 +401,8 @@ Error* Compiler::exprList() noexcept{
     ListExpr* e = make_expr<ListExpr>(count);
     e->line = line;  // override line
     for(int i=count-1; i>=0; i--)
-        e->items[i] = s_expr().popx_back();
-    s_expr().push_back(e);
+        e->items[i] = ctx()->s_popx();
+    ctx()->s_push(e);
     return NULL;
 }
 
@@ -414,30 +414,31 @@ Error* Compiler::exprMap() noexcept{
         match_newlines_repl();
         if(curr().type == TK("}")) break;
         check(EXPR());  // [key]
-        int star_level = s_expr().back()->star_level();
+        int star_level = ctx()->s_top()->star_level();
         if(star_level == 2 || curr().type == TK(":")) { parsing_dict = true; }
         if(parsing_dict) {
             if(star_level == 2) {
                 DictItemExpr* dict_item = make_expr<DictItemExpr>();
                 dict_item->key = NULL;
-                dict_item->value = s_expr().popx_back();
-                s_expr().push_back(dict_item);
+                dict_item->value = ctx()->s_popx();
+                ctx()->s_push(dict_item);
             } else {
                 consume(TK(":"));
                 check(EXPR());
                 DictItemExpr* dict_item = make_expr<DictItemExpr>();
-                dict_item->value = s_expr().popx_back();
-                dict_item->key = s_expr().popx_back();
-                s_expr().push_back(dict_item);
+                dict_item->value = ctx()->s_popx();
+                dict_item->key = ctx()->s_popx();
+                ctx()->s_push(dict_item);
             }
         }
         count += 1;
         match_newlines_repl();
         if(count == 1 && match(TK("for"))) {
-            if(parsing_dict)
+            if(parsing_dict){
                 check(consume_comp(OP_BUILD_DICT, OP_DICT_ADD));
-            else
+            }else{
                 check(consume_comp(OP_BUILD_SET, OP_SET_ADD));
+            }
             consume(TK("}"));
             return NULL;
         }
@@ -452,16 +453,16 @@ Error* Compiler::exprMap() noexcept{
         se = make_expr<SetExpr>(count);
     }
     for(int i=count-1; i>=0; i--)
-        se->items[i] = s_expr().popx_back();
-    s_expr().push_back(se);
+        se->items[i] = ctx()->s_popx();
+    ctx()->s_push(se);
     return NULL;
 }
 
 Error* Compiler::exprCall() noexcept{
     Error* err;
     CallExpr* e = make_expr<CallExpr>();
-    e->callable = s_expr().popx_back();
-    s_expr().push_back(e);     // push onto the stack in advance
+    e->callable = ctx()->s_popx();
+    ctx()->s_push(e);     // push onto the stack in advance
     do {
         match_newlines_repl();
         if(curr().type == TK(")")) break;
@@ -470,16 +471,16 @@ Error* Compiler::exprCall() noexcept{
             StrName key(prev().sv());
             consume(TK("="));
             check(EXPR());
-            e->kwargs.push_back({key, s_expr().popx_back()});
+            e->kwargs.push_back({key, ctx()->s_popx()});
         } else {
             check(EXPR());
-            if(s_expr().back()->star_level() == 2) {
+            if(ctx()->s_top()->star_level() == 2) {
                 // **kwargs
-                e->kwargs.push_back({"**", s_expr().popx_back()});
+                e->kwargs.push_back({"**", ctx()->s_popx()});
             } else {
                 // positional argument
                 if(!e->kwargs.empty()) return SyntaxError("positional argument follows keyword argument");
-                e->args.push_back(s_expr().popx_back());
+                e->args.push_back(ctx()->s_popx());
             }
         }
         match_newlines_repl();
@@ -492,32 +493,32 @@ Error* Compiler::exprName() noexcept{
     StrName name(prev().sv());
     NameScope scope = name_scope();
     if(ctx()->global_names.contains(name)) { scope = NAME_GLOBAL; }
-    s_expr().push_back(make_expr<NameExpr>(name, scope));
+    ctx()->s_push(make_expr<NameExpr>(name, scope));
     return NULL;
 }
 
 Error* Compiler::exprAttrib() noexcept{
     consume(TK("@id"));
-    s_expr().push_back(make_expr<AttribExpr>(s_expr().popx_back(), StrName::get(prev().sv())));
+    ctx()->s_push(make_expr<AttribExpr>(ctx()->s_popx(), StrName::get(prev().sv())));
     return NULL;
 }
 
 Error* Compiler::exprSlice0() noexcept{
     Error* err;
     SliceExpr* slice = make_expr<SliceExpr>();
-    s_expr().push_back(slice);     // push onto the stack in advance
+    ctx()->s_push(slice);     // push onto the stack in advance
     if(is_expression()) {  // :<stop>
         check(EXPR());
-        slice->stop = s_expr().popx_back();
+        slice->stop = ctx()->s_popx();
         // try optional step
         if(match(TK(":"))) {  // :<stop>:<step>
             check(EXPR());
-            slice->step = s_expr().popx_back();
+            slice->step = ctx()->s_popx();
         }
     } else if(match(TK(":"))) {
         if(is_expression()) {  // ::<step>
             check(EXPR());
-            slice->step = s_expr().popx_back();
+            slice->step = ctx()->s_popx();
         }  // else ::
     }  // else :
     return NULL;
@@ -526,38 +527,40 @@ Error* Compiler::exprSlice0() noexcept{
 Error* Compiler::exprSlice1() noexcept{
     Error* err;
     SliceExpr* slice = make_expr<SliceExpr>();
-    slice->start = s_expr().popx_back();
-    s_expr().push_back(slice);     // push onto the stack in advance
+    slice->start = ctx()->s_popx();
+    ctx()->s_push(slice);     // push onto the stack in advance
     if(is_expression()) {  // <start>:<stop>
         check(EXPR());
-        slice->stop = s_expr().popx_back();
+        slice->stop = ctx()->s_popx();
         // try optional step
         if(match(TK(":"))) {  // <start>:<stop>:<step>
             check(EXPR());
-            slice->step = s_expr().popx_back();
+            slice->step = ctx()->s_popx();
         }
     } else if(match(TK(":"))) {  // <start>::<step>
         check(EXPR());
-        slice->step = s_expr().popx_back();
+        slice->step = ctx()->s_popx();
     }  // else <start>:
     return NULL;
 }
 
 Error* Compiler::exprSubscr() noexcept{
     Error* err;
-    SubscrExpr* e = make_expr<SubscrExpr>();
-    s_expr().push_back(e);
+    int line = prev().line;
     match_newlines_repl();
-    e->lhs = s_expr().popx_back();  // a
     check(EXPR_TUPLE(true));
-    e->rhs = s_expr().popx_back();  // a[<expr>]
     match_newlines_repl();
-    consume(TK("]"));
+    consume(TK("]"));           // [lhs, rhs]
+    SubscrExpr* e = make_expr<SubscrExpr>();
+    e->line = line;
+    e->rhs = ctx()->s_popx();   // [lhs]
+    e->lhs = ctx()->s_popx();   // []
+    ctx()->s_push(e);
     return NULL;
 }
 
 Error* Compiler::exprLiteral0() noexcept{
-    s_expr().push_back(make_expr<Literal0Expr>(prev().type));
+    ctx()->s_push(make_expr<Literal0Expr>(prev().type));
     return NULL;
 }
 
@@ -697,7 +700,7 @@ Error* Compiler::parse_expression(int precedence, bool allow_slice) noexcept{
 Error* Compiler::compile_if_stmt() noexcept{
     Error* err;
     check(EXPR());  // condition
-    ctx()->emit_expr();
+    ctx()->s_emit_top();
     int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line);
     err = compile_block_body();
     if(err) return err;
@@ -721,7 +724,7 @@ Error* Compiler::compile_while_loop() noexcept{
     Error* err;
     CodeBlock* block = ctx()->enter_block(CodeBlockType::WHILE_LOOP);
     check(EXPR());  // condition
-    ctx()->emit_expr();
+    ctx()->s_emit_top();
     int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line);
     check(compile_block_body());
     ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE, true);
@@ -740,11 +743,11 @@ Error* Compiler::compile_for_loop() noexcept{
     check(EXPR_VARS());     // [vars]
     consume(TK("in"));
     check(EXPR_TUPLE());    // [vars, iter]
-    ctx()->emit_expr();     // [vars]
+    ctx()->s_emit_top();     // [vars]
     ctx()->emit_(OP_GET_ITER_NEW, BC_NOARG, BC_KEEPLINE);
     CodeBlock* block = ctx()->enter_block(CodeBlockType::FOR_LOOP);
     int for_codei = ctx()->emit_(OP_FOR_ITER, ctx()->curr_iblock, BC_KEEPLINE);
-    Expr* vars = s_expr().popx_back();
+    Expr* vars = ctx()->s_popx();
     bool ok = vars->emit_store(ctx());
     delete_expr(vars);
     if(!ok) return SyntaxError();  // this error occurs in `vars` instead of this line, but...nevermind
@@ -776,7 +779,7 @@ Error* Compiler::compile_try_except() noexcept{
             consume(TK("except"));
             if(is_expression()) {
                 check(EXPR());  // push assumed type on to the stack
-                ctx()->emit_expr();
+                ctx()->s_emit_top();
                 ctx()->emit_(OP_EXCEPTION_MATCH, BC_NOARG, prev().line);
                 if(match(TK("as"))) {
                     consume(TK("@id"));
@@ -836,10 +839,6 @@ Error* Compiler::compile_decorated() noexcept{
         if(!match_newlines_repl()) return SyntaxError();
     } while(match(TK("@")));
 
-    array<Expr*> decorators(count);
-    for(int i = count - 1; i >= 0; i--)
-        decorators[i] = s_expr().popx_back();
-    
     if(match(TK("class"))) {
         check(compile_class(count));
     } else {
@@ -863,7 +862,7 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
         case TK("&="):
         case TK("|="):
         case TK("^="): {
-            Expr* lhs_p = s_expr().back();
+            Expr* lhs_p = ctx()->s_top();
             if(lhs_p->is_starred()) return SyntaxError();
             if(ctx()->is_compiling_class){
                 return SyntaxError("can't use inplace operator in class definition");
@@ -873,14 +872,15 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
             // a.x += 1;    a should be evaluated only once
             // -1 to remove =; inplace=true
             auto e = make_expr<BinaryExpr>(prev().type - 1, true);
-            e->lhs = s_expr().popx_back();
+            e->lhs = ctx()->s_popx();
             check(EXPR_TUPLE());
-            e->rhs = s_expr().popx_back();
+            e->rhs = ctx()->s_popx();
             if(e->rhs->is_starred()) return SyntaxError();
             e->emit_(ctx());
             bool ok = lhs_p->emit_store_inplace(ctx());
             if(!ok) return SyntaxError();
             *is_assign = true;
+            return NULL;
         }
         case TK("="): {
             int n = 0;
@@ -889,16 +889,17 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
                 n += 1;
             }
             // stack size is n+1
-            ctx()->emit_expr();     // emit and pop
+            ctx()->s_emit_top();     // emit and pop
             for(int j = 1; j < n; j++)
                 ctx()->emit_(OP_DUP_TOP, BC_NOARG, BC_KEEPLINE);
             for(int j = 0; j < n; j++) {
-                auto e = s_expr().popx_back();
+                auto e = ctx()->s_popx();
                 if(e->is_starred()) return SyntaxError();
                 bool ok = e->emit_store(ctx());
                 if(!ok) return SyntaxError();
             }
             *is_assign = true;
+            return NULL;
         }
         default: *is_assign = false;
     }
@@ -928,14 +929,14 @@ Error* Compiler::compile_stmt() noexcept{
         case TK("yield"):
             if(contexts.size() <= 1) return SyntaxError("'yield' outside function");
             check(EXPR_TUPLE());
-            ctx()->emit_expr();
+            ctx()->s_emit_top();
             ctx()->emit_(OP_YIELD_VALUE, BC_NOARG, kw_line);
             consume_end_stmt();
             break;
         case TK("yield from"):
             if(contexts.size() <= 1) return SyntaxError("'yield from' outside function");
             check(EXPR_TUPLE());
-            ctx()->emit_expr();
+            ctx()->s_emit_top();
 
             ctx()->emit_(OP_GET_ITER_NEW, BC_NOARG, kw_line);
             ctx()->enter_block(CodeBlockType::FOR_LOOP);
@@ -950,7 +951,7 @@ Error* Compiler::compile_stmt() noexcept{
                 ctx()->emit_(OP_RETURN_VALUE, 1, kw_line);
             } else {
                 check(EXPR_TUPLE());
-                ctx()->emit_expr();
+                ctx()->s_emit_top();
                 consume_end_stmt();
                 ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, kw_line);
             }
@@ -968,12 +969,12 @@ Error* Compiler::compile_stmt() noexcept{
         /*************************************************/
         case TK("assert"): {
             check(EXPR());  // condition
-            ctx()->emit_expr();
+            ctx()->s_emit_top();
             int index = ctx()->emit_(OP_POP_JUMP_IF_TRUE, BC_NOARG, kw_line);
             int has_msg = 0;
             if(match(TK(","))) {
                 check(EXPR());  // message
-                ctx()->emit_expr();
+                ctx()->s_emit_top();
                 has_msg = 1;
             }
             ctx()->emit_(OP_RAISE_ASSERT, has_msg, kw_line);
@@ -990,19 +991,19 @@ Error* Compiler::compile_stmt() noexcept{
             break;
         case TK("raise"): {
             check(EXPR());
-            ctx()->emit_expr();
+            ctx()->s_emit_top();
             ctx()->emit_(OP_RAISE, BC_NOARG, kw_line);
             consume_end_stmt();
         } break;
         case TK("del"): {
             check(EXPR_TUPLE());
-            Expr* e = s_expr().popx_back();
+            Expr* e = ctx()->s_popx();
             if(!e->emit_del(ctx())) return SyntaxError();
             consume_end_stmt();
         } break;
         case TK("with"): {
             check(EXPR());  // [ <expr> ]
-            ctx()->emit_expr();
+            ctx()->s_emit_top();
             ctx()->enter_block(CodeBlockType::CONTEXT_MANAGER);
             Expr* as_name = nullptr;
             if(match(TK("as"))) {
@@ -1045,13 +1046,13 @@ Error* Compiler::compile_stmt() noexcept{
 
             bool is_typed_name = false;  // e.g. x: int
             // eat variable's type hint if it is a single name
-            if(s_expr().back()->is_name()) {
+            if(ctx()->s_top()->is_name()) {
                 if(match(TK(":"))) {
                     check(consume_type_hints());
                     is_typed_name = true;
 
                     if(ctx()->is_compiling_class) {
-                        NameExpr* ne = static_cast<NameExpr*>(s_expr().back());
+                        NameExpr* ne = static_cast<NameExpr*>(ctx()->s_top());
                         ctx()->emit_(OP_ADD_CLASS_ANNOTATION, ne->name.index, BC_KEEPLINE);
                     }
                 }
@@ -1059,18 +1060,18 @@ Error* Compiler::compile_stmt() noexcept{
             bool is_assign = false;
             check(try_compile_assignment(&is_assign));
             if(!is_assign) {
-                if(!s_expr().empty() && s_expr().back()->is_starred()) { 
+                if(ctx()->s_size() > 0 && ctx()->s_top()->is_starred()) { 
                     return SyntaxError();
                 }
                 if(!is_typed_name) {
-                    ctx()->emit_expr();
+                    ctx()->s_emit_top();
                     if((mode() == CELL_MODE || mode() == REPL_MODE) && name_scope() == NAME_GLOBAL) {
                         ctx()->emit_(OP_PRINT_EXPR, BC_NOARG, BC_KEEPLINE);
                     } else {
                         ctx()->emit_(OP_POP_TOP, BC_NOARG, BC_KEEPLINE);
                     }
                 } else {
-                    ctx()->emit_expr(false);
+                    ctx()->s_pop();
                 }
             }
             consume_end_stmt();
@@ -1083,7 +1084,7 @@ Error* Compiler::compile_stmt() noexcept{
 Error* Compiler::consume_type_hints() noexcept{
     Error* err;
     check(EXPR());
-    Expr* e = s_expr().popx_back();
+    Expr* e = ctx()->s_popx();
     delete_expr(e);
     return NULL;
 }
@@ -1096,7 +1097,7 @@ Error* Compiler::compile_class(int decorators) noexcept{
     if(match(TK("("))) {
         if(is_expression()) {
             check(EXPR());
-            base = s_expr().popx_back();
+            base = ctx()->s_popx();
         }
         consume(TK(")"));
     }
@@ -1114,10 +1115,9 @@ Error* Compiler::compile_class(int decorators) noexcept{
     check(compile_block_body());
     ctx()->is_compiling_class = false;
 
-    assert(s_expr().size() == decorators);
     if(decorators > 0) {
         ctx()->emit_(OP_BEGIN_CLASS_DECORATION, BC_NOARG, BC_KEEPLINE);
-        ctx()->emit_decorators(decorators);
+        ctx()->s_emit_decorators(decorators);
         ctx()->emit_(OP_END_CLASS_DECORATION, BC_NOARG, BC_KEEPLINE);
     }
 
@@ -1209,8 +1209,7 @@ Error* Compiler::compile_function(int decorators) noexcept{
     }
     ctx()->emit_(OP_LOAD_FUNCTION, ctx()->add_func_decl(decl), prev().line);
 
-    assert(s_expr().size() == decorators);
-    ctx()->emit_decorators(decorators);
+    ctx()->s_emit_decorators(decorators);
 
     if(!ctx()->is_compiling_class) {
         auto e = make_expr<NameExpr>(decl_name, name_scope());
@@ -1273,11 +1272,15 @@ Compiler::Compiler(VM* vm, std::string_view source, const Str& filename, Compile
 }
 
 Error* Compiler::compile(CodeObject_* out) noexcept{
-    assert(i == 0);  // make sure it is the first time to compile
+    assert(__i == 0);  // make sure it is the first time to compile
 
     Error* err;
     check(lexer.run());
 
+    // for(int i=0; i<lexer.nexts.size(); i++){
+    //     printf("%s: %s\n", TK_STR(tk(i).type), tk(i).str().escape().c_str());
+    // }
+
     CodeObject_ code = push_global_context();
 
     advance();         // skip @sof, so prev() is always valid
@@ -1285,7 +1288,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{
 
     if(mode() == EVAL_MODE) {
         check(EXPR_TUPLE());
-        ctx()->emit_expr();
+        ctx()->s_emit_top();
         consume(TK("@eof"));
         ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE);
         check(pop_context());
@@ -1293,7 +1296,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{
         return NULL;
     } else if(mode() == JSON_MODE) {
         check(EXPR());
-        Expr* e = s_expr().popx_back();
+        Expr* e = ctx()->s_popx();
         if(!e->is_json_object()) return SyntaxError("expect a JSON object, literal or array");
         consume(TK("@eof"));
         e->emit_(ctx());
@@ -1314,7 +1317,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{
 
 Compiler::~Compiler(){
     for(CodeEmitContext& ctx: contexts){
-        for(Expr* e: ctx.s_expr) delete_expr(e);
+        ctx.s_clean();
     }
 }
 

src/compiler/expr.cpp

@@ -40,18 +40,11 @@ void CodeEmitContext::exit_block() noexcept{
     }
 }
 
-// clear the expression stack and generate bytecode
-void CodeEmitContext::emit_expr(bool emit) noexcept{
-    // assert(s_expr.size() == 1);
-    Expr* e = s_expr.popx_back();
-    if(emit) e->emit_(this);
-    delete_expr(e);
-}
-
-void CodeEmitContext::emit_decorators(int count) noexcept{
+void CodeEmitContext::s_emit_decorators(int count) noexcept{
+    assert(s_size() >= count);
     // [obj]
     for(int i=0; i<count; i++) {
-        Expr* deco = s_expr.popx_back();
+        Expr* deco = s_popx();
         deco->emit_(this);                           // [obj, f]
         emit_(OP_ROT_TWO, BC_NOARG, deco->line);     // [f, obj]
         emit_(OP_LOAD_NULL, BC_NOARG, BC_KEEPLINE);  // [f, obj, NULL]

src/compiler/lexer.cpp

@@ -501,7 +501,7 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
 Error* Lexer::_error(bool lexer_err, const char* type, const char* msg, va_list args, i64 userdata) noexcept{
     PK_THREAD_LOCAL Error err;
     err.type = type;
-    err.src = src.get();
+    err.src = src;
     if(lexer_err){
         err.lineno = current_line;
         err.cursor = curr_char;
@@ -529,16 +529,19 @@ Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{
 Lexer::Lexer(VM* vm, std::shared_ptr<SourceData> src) noexcept : vm(vm), src(src){
     this->token_start = src->source.c_str();
     this->curr_char = src->source.c_str();
-    this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level, {}});
-    this->indents.push_back(0);
 }
 
 Error* Lexer::run() noexcept{
+    assert(!this->used);
+    this->used = true;
     if(src->is_precompiled) {
         from_precompiled();
         return NULL;
     }
-    assert(curr_char == src->source.c_str());
+    // push initial tokens
+    this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level, {}});
+    this->indents.push_back(0);
+
     bool eof = false;
     while(!eof) {
         Error* err = lex_one_token(&eof);

src/interpreter/vm.cpp

@@ -182,7 +182,9 @@ PyVar VM::exec(std::string_view source, Str filename, CompileMode mode, PyObject
 #endif
         CodeObject_ code = compile(source, filename, mode);
         return _exec(code, _module);
-    } catch(TopLevelException e) { stderr_write(e.summary() + "\n"); } catch(const std::exception& e) {
+    } catch(TopLevelException e) {
+        stderr_write(e.summary() + "\n");
+    } catch(const std::exception& e) {
         Str msg = "An std::exception occurred! It could be a bug.\n";
         msg = msg + e.what() + "\n";
         stderr_write(msg);
@@ -1374,12 +1376,13 @@ PyObject* VM::bind(PyObject* obj, const char* sig, NativeFuncC fn, any userdata,
 }
 
 PyObject* VM::bind(PyObject* obj, const char* sig, const char* docstring, NativeFuncC fn, any userdata, BindType bt) {
-    CodeObject_ co;
-    try {
-        // fn(a, b, *c, d=1) -> None
-        co = compile(_S("def ", sig, " : pass"), "<bind>", EXEC_MODE);
-    } catch(TopLevelException) { throw std::runtime_error("invalid signature: " + std::string(sig)); }
-    if(co->func_decls.size() != 1) { throw std::runtime_error("expected 1 function declaration"); }
+    char buffer[256];
+    int length = snprintf(buffer, sizeof(buffer), "def %s : pass", sig);
+    std::string_view source(buffer, length);
+    // fn(a, b, *c, d=1) -> None
+    CodeObject_ co = compile(source, "<bind>", EXEC_MODE);
+    assert(co->func_decls.size() == 1);
+    
     FuncDecl_ decl = co->func_decls[0];
     decl->docstring = docstring;
     PyObject* f_obj = heap.gcnew<NativeFunc>(tp_native_func, fn, decl, std::move(userdata));
@@ -1455,7 +1458,7 @@ void VM::__raise_exc(bool re_raise) {
     int current_line = frame->co->lines[actual_ip].lineno;  // current line
     auto current_f_name = frame->co->name.sv();             // current function name
     if(frame->_callable == nullptr) current_f_name = "";    // not in a function
-    e.st_push(frame->co->src.get(), current_line, nullptr, current_f_name);
+    e.st_push(frame->co->src, current_line, nullptr, current_f_name);
 
     if(next_ip >= 0) {
         throw InternalException(InternalExceptionType::Handled, next_ip);

tests/44_decorator.py

@@ -38,4 +38,43 @@ def fib(n):
         return n
     return fib(n-1) + fib(n-2)
 
-assert fib(32) == 2178309
+assert fib(32) == 2178309
+
+def wrapped(cls):
+    return int
+
+@wrapped
+@wrapped
+@wrapped
+@wrapped
+class A:
+    def __init__(self) -> None:
+        pass
+
+assert A('123') == 123
+
+# validate the decorator order
+res = []
+
+def w(x):
+    res.append('w')
+    return x
+
+def w1(x):
+    res.append('w1')
+    return x
+
+def w2(x):
+    res.append('w2')
+    return x
+
+@w
+@w1
+@w2
+def f():
+    pass
+
+f()
+assert res == ['w2', 'w1', 'w']
+
+

tests/82_dataclasses.py

@@ -15,15 +15,3 @@ assert asdict(A(1, '555')) == {'x': 1, 'y': '555'}
 assert A(1, 'N') == A(1, 'N')
 assert A(1, 'N') != A(1, 'M')
 
-def wrapped(cls):
-    return int
-
-@wrapped
-@wrapped
-@wrapped
-@wrapped
-class A:
-    def __init__(self) -> None:
-        pass
-
-assert A('123') == 123

tests/94_compile.py

@@ -1,2 +1,5 @@
+assert eval("1+2") == 3
+
 code = compile("1+2", "<eval>", "eval")
+# print(code)
 assert eval(code) == 3