up

src/ceval.h

@@ -18,6 +18,9 @@ __NEXT_STEP:;
     //heap._auto_collect(this);
 
     const Bytecode& byte = frame->next_bytecode();
+
+    // std::cout << frame->stack_info() << " " << OP_NAMES[byte.op] << std::endl;
+
     switch (byte.op)
     {
     case OP_NO_OP: DISPATCH();
@@ -296,7 +299,7 @@ __NEXT_STEP:;
             }
             CodeObject_ code = compile(source, name.str(), EXEC_MODE);
             PyObject* new_mod = new_module(name);
-            _exec(code, new_mod);
+            _exec(code, new_mod, builtins);
             new_mod->attr()._try_perfect_rehash();
         }
         frame->push(ext_mod);

src/codeobject.h

@@ -57,7 +57,7 @@ struct CodeObject {
     std::vector<Bytecode> codes;
     List consts;
     std::vector<StrName> names;
-    std::set<StrName> global_names;
+    std::set<Str> global_names;
     std::vector<CodeBlock> blocks = { CodeBlock{NO_BLOCK, -1} };
     std::map<StrName, int> labels;
     std::vector<FuncDecl_> func_decls;

src/common.h

@@ -28,16 +28,19 @@
 #include <variant>
 #include <type_traits>
 
-#define PK_VERSION				"0.9.6"
+#define PK_VERSION				"0.9.7"
 
 // debug macros
-#define DEBUG_NO_BUILTIN_MODULES	1
-#define DEBUG_MODE					1
+#define DEBUG_NO_BUILTIN_MODULES	0
+#define DEBUG_EXTRA_CHECK			1
+#define DEBUG_DIS_REPL				0
+#define DEBUG_DIS_REPL_MIN			1
+#define DEBUG_FULL_EXCEPTION		1
 
 #if (defined(__ANDROID__) && __ANDROID_API__ <= 22) || defined(__EMSCRIPTEN__)
 #define PK_ENABLE_FILEIO 		0
 #else
-#define PK_ENABLE_FILEIO 		1
+#define PK_ENABLE_FILEIO 		(1-DEBUG_NO_BUILTIN_MODULES)
 #endif
 
 #if defined(__EMSCRIPTEN__) || defined(__arm__) || defined(__i386__)

src/compiler.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "codeobject.h"
+#include "common.h"
 #include "expr.h"
 
 namespace pkpy{
@@ -152,6 +154,17 @@ class Compiler {
         parse_expression(PREC_TUPLE, push_stack);
     }
 
+    // special case for `for loop` and `comp`
+    Expr_ EXPR_VARS(){
+        std::vector<Expr_> items;
+        do {
+            consume(TK("@id"));
+            items.push_back(make_expr<NameExpr>(prev().str(), name_scope()));
+        } while(match(TK(",")));
+        if(items.size()==1) return std::move(items[0]);
+        return make_expr<TupleExpr>(std::move(items));
+    }
+
     template <typename T, typename... Args>
     std::unique_ptr<T> make_expr(Args&&... args) {
         std::unique_ptr<T> expr = std::make_unique<T>(std::forward<Args>(args)...);
@@ -269,10 +282,9 @@ class Compiler {
     template<typename T>
     void _consume_comp(Expr_ expr){
         static_assert(std::is_base_of<CompExpr, T>::value);
-        std::unique_ptr<CompExpr> ce = std::make_unique<T>();
+        std::unique_ptr<CompExpr> ce = make_expr<T>();
         ce->expr = std::move(expr);
-        EXPR_TUPLE();   // must be a lvalue
-        ce->vars = ctx()->s_expr.popx();
+        ce->vars = EXPR_VARS();
         consume(TK("in"));
         EXPR();
         ce->iter = ctx()->s_expr.popx();
@@ -374,7 +386,12 @@ class Compiler {
 
     // PASS
     void exprName(){
-        ctx()->s_expr.push(make_expr<NameExpr>(prev().str(), name_scope()));
+        Str name = prev().str();
+        NameScope scope = name_scope();
+        if(ctx()->co->global_names.count(name)){
+            scope = NAME_GLOBAL;
+        }
+        ctx()->s_expr.push(make_expr<NameExpr>(name, scope));
     }
 
     // PASS
@@ -389,26 +406,65 @@ class Compiler {
     void exprSubscr() {
         auto e = make_expr<SubscrExpr>();
         e->a = ctx()->s_expr.popx();
-        std::vector<Expr_> items;
-        do {
-            EXPR_TUPLE();
-            items.push_back(ctx()->s_expr.popx());
-        } while(match(TK(":")));
-        consume(TK("]"));
-        switch(items.size()){
-            case 1:
-                e->b = std::move(items[0]);
-                break;
-            case 2: case 3: {
-                auto slice = make_expr<SliceExpr>();
-                slice->start = std::move(items[0]);
-                slice->stop = std::move(items[1]);
-                if(items.size()==3){
-                    slice->step = std::move(items[2]);
-                }
-                e->b = std::move(slice);
-            } break;
-            default: SyntaxError(); break;
+        auto slice = make_expr<SliceExpr>();
+        bool is_slice = false;
+        // a[<0> <state:1> : state<3> : state<5>]
+        int state = 0;
+        do{
+            switch(state){
+                case 0:
+                    if(match(TK(":"))){
+                        is_slice=true;
+                        state=2;
+                        break;
+                    }
+                    if(match(TK("]"))) SyntaxError();
+                    EXPR_TUPLE();
+                    slice->start = ctx()->s_expr.popx();
+                    state=1;
+                    break;
+                case 1:
+                    if(match(TK(":"))){
+                        is_slice=true;
+                        state=2;
+                        break;
+                    }
+                    if(match(TK("]"))) goto __SUBSCR_END;
+                    SyntaxError("expected ':' or ']'");
+                    break;
+                case 2:
+                    if(match(TK(":"))){
+                        state=4;
+                        break;
+                    }
+                    if(match(TK("]"))) goto __SUBSCR_END;
+                    EXPR_TUPLE();
+                    slice->stop = ctx()->s_expr.popx();
+                    state=3;
+                    break;
+                case 3:
+                    if(match(TK(":"))){
+                        state=4;
+                        break;
+                    }
+                    if(match(TK("]"))) goto __SUBSCR_END;
+                    SyntaxError("expected ':' or ']'");
+                    break;
+                case 4:
+                    if(match(TK("]"))) goto __SUBSCR_END;
+                    EXPR_TUPLE();
+                    slice->step = ctx()->s_expr.popx();
+                    state=5;
+                    break;
+                case 5: consume(TK("]")); goto __SUBSCR_END;
+            }
+        }while(true);
+__SUBSCR_END:
+        if(is_slice){
+            e->b = std::move(slice);
+        }else{
+            if(state != 1) UNREACHABLE();
+            e->b = std::move(slice->start);
         }
         ctx()->s_expr.push(std::move(e));
     }
@@ -535,8 +591,7 @@ class Compiler {
 
     // PASS
     void compile_for_loop() {
-        EXPR_TUPLE();
-        Expr_ vars = ctx()->s_expr.popx();
+        Expr_ vars = EXPR_VARS();
         consume(TK("in"));
         EXPR(false);
         ctx()->emit(OP_GET_ITER, BC_NOARG, BC_KEEPLINE);
@@ -550,31 +605,32 @@ class Compiler {
     }
 
     void compile_try_except() {
-        // ctx()->enter_block(TRY_EXCEPT);
-        // ctx()->emit(OP_TRY_BLOCK_ENTER, BC_NOARG, prev().line);
-        // compile_block_body();
-        // ctx()->emit(OP_TRY_BLOCK_EXIT, BC_NOARG, BC_KEEPLINE);
-        // std::vector<int> patches = {
-        //     ctx()->emit(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE)
-        // };
-        // ctx()->exit_block();
-
-        // do {
-        //     consume(TK("except"));
-        //     if(match(TK("@id"))){
-        //         int name_idx = ctx()->add_name(prev().str(), NAME_SPECIAL);
-        //         emit(OP_EXCEPTION_MATCH, name_idx);
-        //     }else{
-        //         emit(OP_LOAD_TRUE);
-        //     }
-        //     int patch = emit(OP_POP_JUMP_IF_FALSE);
-        //     emit(OP_POP_TOP);       // pop the exception on match
-        //     compile_block_body();
-        //     patches.push_back(emit(OP_JUMP_ABSOLUTE));
-        //     patch_jump(patch);
-        // }while(curr().type == TK("except"));
-        // emit(OP_RE_RAISE);      // no match, re-raise
-        // for (int patch : patches) patch_jump(patch);
+        ctx()->enter_block(TRY_EXCEPT);
+        ctx()->emit(OP_TRY_BLOCK_ENTER, BC_NOARG, prev().line);
+        compile_block_body();
+        ctx()->emit(OP_TRY_BLOCK_EXIT, BC_NOARG, BC_KEEPLINE);
+        std::vector<int> patches = {
+            ctx()->emit(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE)
+        };
+        ctx()->exit_block();
+        do {
+            consume(TK("except"));
+            if(match(TK("@id"))){
+                int namei = ctx()->add_name(prev().str());
+                ctx()->emit(OP_EXCEPTION_MATCH, namei, prev().line);
+            }else{
+                ctx()->emit(OP_LOAD_TRUE, BC_NOARG, BC_KEEPLINE);
+            }
+            int patch = ctx()->emit(OP_POP_JUMP_IF_FALSE, BC_NOARG, BC_KEEPLINE);
+            // pop the exception on match
+            ctx()->emit(OP_POP_TOP, BC_NOARG, BC_KEEPLINE);
+            compile_block_body();
+            patches.push_back(ctx()->emit(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE));
+            ctx()->patch_jump(patch);
+        }while(curr().type == TK("except"));
+        // no match, re-raise
+        ctx()->emit(OP_RE_RAISE, BC_NOARG, BC_KEEPLINE);
+        for (int patch : patches) ctx()->patch_jump(patch);
     }
 
     void compile_decorated(){
@@ -611,7 +667,7 @@ class Compiler {
                 break;
             default: return false;
         }
-        std::cout << ctx()->_log_s_expr() << std::endl;
+        // std::cout << ctx()->_log_s_expr() << std::endl;
         Expr_ rhs = ctx()->s_expr.popx();
 
         if(lhs_p->is_starred() || rhs->is_starred()){
@@ -785,7 +841,6 @@ class Compiler {
             if(enable_type_hints && match(TK(":"))) consume(TK("@id"));
 
             if(state == 0 && curr().type == TK("=")) state = 2;
-
             switch (state)
             {
                 case 0: decl->args.push_back(name); break;

src/expr.h

@@ -357,8 +357,8 @@ struct DictItemExpr: Expr{
     std::vector<const Expr*> children() const override { return {key.get(), value.get()}; }
 
     void emit(CodeEmitContext* ctx) override {
-        key->emit(ctx);
         value->emit(ctx);
+        key->emit(ctx);     // reverse order
         ctx->emit(OP_BUILD_TUPLE, 2, line);
     }
 };
@@ -462,9 +462,11 @@ struct CompExpr: Expr{
         if(cond){
             cond->emit(ctx);
             int patch = ctx->emit(OP_POP_JUMP_IF_FALSE, BC_NOARG, BC_KEEPLINE);
+            expr->emit(ctx);
             ctx->emit(op1(), BC_NOARG, BC_KEEPLINE);
             ctx->patch_jump(patch);
         }else{
+            expr->emit(ctx);
             ctx->emit(op1(), BC_NOARG, BC_KEEPLINE);
         }
         ctx->emit(OP_LOOP_CONTINUE, BC_NOARG, BC_KEEPLINE);

src/frame.h

@@ -14,6 +14,7 @@ struct Frame {
     const CodeObject* co;
     PyObject* _module;
     NameDict_ _locals;
+    NameDict_ _closure;
     const uint64_t id;
     std::vector<std::pair<int, std::vector<PyObject*>>> s_try_block;
     const NameDict* names[5];     // name resolution array, zero terminated
@@ -21,14 +22,16 @@ struct Frame {
     NameDict& f_locals() noexcept { return *_locals; }
     NameDict& f_globals() noexcept { return _module->attr(); }
 
-    Frame(const CodeObject_& co, PyObject* _module, NameDict_ _locals=nullptr, NameDict_ _closure=nullptr)
-            : co(co.get()), _module(_module), _locals(_locals), id(kFrameGlobalId++) {
+    Frame(const CodeObject_& co, PyObject* _module, PyObject* builtins, NameDict_ _locals=nullptr, NameDict_ _closure=nullptr)
+            : co(co.get()), _module(_module), _locals(_locals), _closure(_closure), id(kFrameGlobalId++) {
         memset(names, 0, sizeof(names));
         int i = 0;
         if(_locals != nullptr) names[i++] = _locals.get();
         if(_closure != nullptr) names[i++] = _closure.get();
-        names[i++] = &_module->attr();
-        // names[i++] = builtins
+        names[i++] = &_module->attr();  // borrowed reference
+        if(builtins != nullptr){
+            names[i++] = &builtins->attr(); // borrowed reference
+        }
     }
 
     const Bytecode& next_bytecode() {
@@ -41,26 +44,26 @@ struct Frame {
         return co->src->snapshot(line);
     }
 
-    // Str stack_info(){
-    //     StrStream ss;
-    //     ss << "[";
-    //     for(int i=0; i<_data.size(); i++){
-    //         ss << OBJ_TP_NAME(_data[i]);
-    //         if(i != _data.size()-1) ss << ", ";
-    //     }
-    //     ss << "]";
-    //     return ss.str();
-    // }
+    Str stack_info(){
+        StrStream ss;
+        ss << "[";
+        for(int i=0; i<_data.size(); i++){
+            ss << (i64)_data[i];
+            if(i != _data.size()-1) ss << ", ";
+        }
+        ss << "]";
+        return ss.str();
+    }
 
     void pop(){
-#if DEBUG_MODE
+#if DEBUG_EXTRA_CHECK
         if(_data.empty()) throw std::runtime_error("_data.empty() is true");
 #endif
         _data.pop_back();
     }
 
     PyObject* popx(){
-#if DEBUG_MODE
+#if DEBUG_EXTRA_CHECK
         if(_data.empty()) throw std::runtime_error("_data.empty() is true");
 #endif
         PyObject* ret = _data.back();
@@ -69,21 +72,21 @@ struct Frame {
     }
 
     PyObject*& top(){
-#if DEBUG_MODE
+#if DEBUG_EXTRA_CHECK
         if(_data.empty()) throw std::runtime_error("_data.empty() is true");
 #endif
         return _data.back();
     }
 
     PyObject*& top_1(){
-#if DEBUG_MODE
+#if DEBUG_EXTRA_CHECK
         if(_data.size() < 2) throw std::runtime_error("_data.size() < 2");
 #endif
         return _data[_data.size()-2];
     }
 
     PyObject*& top_2(){
-#if DEBUG_MODE
+#if DEBUG_EXTRA_CHECK
         if(_data.size() < 3) throw std::runtime_error("_data.size() < 3");
 #endif
         return _data[_data.size()-3];
@@ -152,12 +155,8 @@ struct Frame {
     void _mark() const {
         for(PyObject* obj : _data) OBJ_MARK(obj);
         OBJ_MARK(_module);
-
-        int i = 0;  // names[0] is ensured to be non-null
-        do{
-            names[i++]->_mark();
-        }while(names[i] != nullptr);
-
+        _locals->_mark();
+        _closure->_mark();
         for(auto& p : s_try_block){
             for(PyObject* obj : p.second) OBJ_MARK(obj);
         }

src/pocketpy.h

@@ -15,7 +15,9 @@ inline CodeObject_ VM::compile(Str source, Str filename, CompileMode mode) {
     try{
         return compiler.compile();
     }catch(Exception& e){
-        // std::cout << e.summary() << std::endl;
+#if DEBUG_FULL_EXCEPTION
+        std::cerr << e.summary() << std::endl;
+#endif
         _error(e);
         return nullptr;
     }
@@ -93,12 +95,12 @@ inline void init_builtins(VM* _vm) {
 
     _vm->bind_builtin_func<1>("eval", [](VM* vm, Args& args) {
         CodeObject_ code = vm->compile(CAST(Str&, args[0]), "<eval>", EVAL_MODE);
-        return vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals);
+        return vm->_exec(code, vm->top_frame()->_module, vm->builtins, vm->top_frame()->_locals);
     });
 
     _vm->bind_builtin_func<1>("exec", [](VM* vm, Args& args) {
         CodeObject_ code = vm->compile(CAST(Str&, args[0]), "<exec>", EXEC_MODE);
-        vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals);
+        vm->_exec(code, vm->top_frame()->_module, vm->builtins, vm->top_frame()->_locals);
         return vm->None;
     });
 
@@ -597,7 +599,7 @@ inline void add_module_json(VM* vm){
     vm->bind_func<1>(mod, "loads", [](VM* vm, Args& args) {
         const Str& expr = CAST(Str&, args[0]);
         CodeObject_ code = vm->compile(expr, "<json>", JSON_MODE);
-        return vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals);
+        return vm->_exec(code, vm->top_frame()->_module, vm->builtins, vm->top_frame()->_locals);
     });
 
     vm->bind_func<1>(mod, "dumps", CPP_LAMBDA(vm->call(args[0], __json__, no_arg())));
@@ -750,7 +752,7 @@ inline void add_module_random(VM* vm){
     PyObject* mod = vm->new_module("random");
     Random::register_class(vm, mod);
     CodeObject_ code = vm->compile(kPythonLibs["random"], "random.py", EXEC_MODE);
-    vm->_exec(code, mod);
+    vm->_exec(code, mod, vm->builtins);
 }
 
 inline void add_module_gc(VM* vm){
@@ -778,11 +780,11 @@ inline void VM::post_init(){
     }
 
     CodeObject_ code = compile(kPythonLibs["builtins"], "<builtins>", EXEC_MODE);
-    this->_exec(code, this->builtins);
+    this->_exec(code, this->builtins, nullptr);
     code = compile(kPythonLibs["_dict"], "<builtins>", EXEC_MODE);
-    this->_exec(code, this->builtins);
+    this->_exec(code, this->builtins, nullptr);
     code = compile(kPythonLibs["_set"], "<builtins>", EXEC_MODE);
-    this->_exec(code, this->builtins);
+    this->_exec(code, this->builtins, nullptr);
 
     // property is defined in builtins.py so we need to add it after builtins is loaded
     _t(tp_object)->attr().set(__class__, property(CPP_LAMBDA(vm->_t(args[0]))));

src/vm.h

@@ -93,7 +93,7 @@ public:
     }
 
     Frame* top_frame() const {
-#if DEBUG_MODE
+#if DEBUG_EXTRA_CHECK
         if(callstack.empty()) UNREACHABLE();
 #endif
         return callstack.top().get();
@@ -166,13 +166,15 @@ public:
         if(_module == nullptr) _module = _main;
         try {
             CodeObject_ code = compile(source, filename, mode);
+#if DEBUG_DIS_REPL
             if(_module == _main) std::cout << disassemble(code) << '\n';
-            return _exec(code, _module);
+#endif
+            return _exec(code, _module, builtins);
         }catch (const Exception& e){
             *_stderr << e.summary() << '\n';
 
         }
-#if !DEBUG_MODE
+#if !DEBUG_FULL_EXCEPTION
         catch (const std::exception& e) {
             *_stderr << "An std::exception occurred! It could be a bug.\n";
             *_stderr << e.what() << '\n';
@@ -607,11 +609,12 @@ inline Str VM::disassemble(CodeObject_ co){
                 argStr += " (" + BITWISE_SPECIAL_METHODS[byte.arg].str() + ")";
                 break;
         }
-        ss << argStr;
-        // ss << pad(argStr, 20);      // may overflow
-        // ss << co->blocks[byte.block].to_string();
+        ss << pad(argStr, 40);      // may overflow
+        ss << co->blocks[byte.block].type;
         if(i != co->codes.size() - 1) ss << '\n';
     }
+
+#if !DEBUG_DIS_REPL_MIN
     StrStream consts;
     consts << "co_consts: ";
     consts << CAST(Str, asRepr(VAR(co->consts)));
@@ -624,7 +627,7 @@ inline Str VM::disassemble(CodeObject_ co){
     }
     names << CAST(Str, asRepr(VAR(list)));
     ss << '\n' << consts.str() << '\n' << names.str();
-
+#endif
     for(auto& decl: co->func_decls){
         ss << "\n\n" << "Disassembly of " << decl->name.str() << ":\n";
         ss << disassemble(decl->code);
@@ -753,7 +756,7 @@ inline PyObject* VM::call(PyObject* callable, Args args, const Args& kwargs, boo
             locals->set(key, kwargs[i+1]);
         }
         PyObject* _module = fn._module != nullptr ? fn._module : top_frame()->_module;
-        auto _frame = _new_frame(fn.decl->code, _module, locals, fn._closure);
+        auto _frame = _new_frame(fn.decl->code, _module, builtins, locals, fn._closure);
         if(fn.decl->code->is_generator) return PyIter(Generator(this, std::move(_frame)));
         callstack.push(std::move(_frame));
         if(opCall) return _py_op_call;