Merge branch 'main' into pr/207

.gitignore

@@ -17,7 +17,6 @@ web/lib
 
 plugins/unity/
 plugins/macos/pocketpy/pocketpy.*
-include/pocketpy/_generated.h
 main.exe
 main.obj
 pocketpy.exp

CMakeLists.txt

@@ -5,23 +5,6 @@ project(pocketpy)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-# if cmake < 3.12, find_package(Python3) will not work, we directly use python3
-if (CMAKE_VERSION VERSION_LESS 3.12)
-    set(Python3_EXECUTABLE python3)
-else()
-    find_package(Python3 COMPONENTS Interpreter)
-endif()
-
-execute_process(
-    COMMAND ${Python3_EXECUTABLE} prebuild.py
-    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-    RESULT_VARIABLE PREBUILD_RESULT
-)
-
-if(NOT ${PREBUILD_RESULT} EQUAL 0)
-    message(FATAL_ERROR "prebuild.py: ${PREBUILD_RESULT}")
-endif()
-
 if(MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /utf-8 /O2")
 else()

amalgamate.py

@@ -60,7 +60,8 @@ for seq in pipeline:
 				text += remove_copied_include(f.read()) + '\n'
 				copied.add(j)
 
-with open("amalgamated/pocketpy.h", "wt", encoding='utf-8') as f:
+# use LF line endings instead of CRLF
+with open("amalgamated/pocketpy.h", "wt", encoding='utf-8', newline='\n') as f:
 	final_text = \
 r'''/*
  *  Copyright (c) 2023 blueloveTH
@@ -77,7 +78,7 @@ shutil.copy("src2/main.cpp", "amalgamated/main.cpp")
 with open("amalgamated/main.cpp", "rt", encoding='utf-8') as f:
 	text = f.read()
 text = text.replace('#include "pocketpy/pocketpy.h"', '#include "pocketpy.h"')
-with open("amalgamated/main.cpp", "wt", encoding='utf-8') as f:
+with open("amalgamated/main.cpp", "wt", encoding='utf-8', newline='\n') as f:
 	f.write(text)
 
 if sys.platform in ['linux', 'darwin']:
@@ -89,7 +90,7 @@ print("amalgamated/pocketpy.h")
 
 def sync(path):
 	shutil.copy("amalgamated/pocketpy.h", os.path.join(path, "pocketpy.h"))
-	with open(os.path.join(path, "pocketpy.cpp"), "wt", encoding='utf-8') as f:
+	with open(os.path.join(path, "pocketpy.cpp"), "wt", encoding='utf-8', newline='\n') as f:
 		f.write("#include \"pocketpy.h\"\n")
 
 sync("plugins/macos/pocketpy")

build_with_warnings.sh

@@ -0,0 +1,5 @@
+SRC=$(find src/ -name "*.cpp")
+
+FLAGS="-std=c++17 -O1 -stdlib=libc++ -Iinclude -W -Wno-unused-parameter"
+
+clang++ $FLAGS -o main -O1 src2/main.cpp $SRC

include/pocketpy/codeobject.h

@@ -58,15 +58,20 @@ using CodeObject_ = std::shared_ptr<CodeObject>;
 using FuncDecl_ = std::shared_ptr<FuncDecl>;
 
 struct CodeObject {
+    struct LineInfo{
+        int lineno;             // line number for each bytecode
+        bool is_virtual;        // whether this bytecode is virtual (not in source code)
+    };
+
     std::shared_ptr<SourceData> src;
     Str name;
     bool is_generator = false;
 
     std::vector<Bytecode> codes;
-    std::vector<int> iblocks;    // block index for each bytecode
-    std::vector<int> lines;     // line number for each bytecode
+    std::vector<int> iblocks;       // block index for each bytecode
+    std::vector<LineInfo> lines;
     List consts;
-    std::vector<StrName> varnames;      // local variables
+    pod_vector<StrName> varnames;      // local variables
     NameDictInt varnames_inv;
     std::vector<CodeBlock> blocks = { CodeBlock(CodeBlockType::NO_BLOCK, -1, 0, 0) };
     NameDictInt labels;
@@ -90,8 +95,8 @@ struct FuncDecl {
         PyObject* value;        // default value
     };
     CodeObject_ code;           // code object of this function
-    std::vector<int> args;      // indices in co->varnames
-    std::vector<KwArg> kwargs;  // indices in co->varnames
+    pod_vector<int> args;      // indices in co->varnames
+    pod_vector<KwArg> kwargs;  // indices in co->varnames
     int starred_arg = -1;       // index in co->varnames, -1 if no *arg
     int starred_kwarg = -1;     // index in co->varnames, -1 if no **kwarg
     bool nested = false;        // whether this function is nested
@@ -104,7 +109,7 @@ struct FuncDecl {
 
     void add_kwarg(int index, StrName key, PyObject* value){
         kw_to_index.set(key, index);
-        kwargs.push_back({index, key, value});
+        kwargs.push_back(KwArg{index, key, value});
     }
     
     void _gc_mark() const;

include/pocketpy/common.h

@@ -18,6 +18,7 @@
 #include <type_traits>
 #include <random>
 #include <deque>
+#include <initializer_list>
 
 #define PK_VERSION				"1.4.1"
 

include/pocketpy/compiler.h

@@ -20,7 +20,8 @@ class Compiler {
     PK_ALWAYS_PASS_BY_POINTER(Compiler)
 
     inline static PrattRule rules[kTokenCount];
-    std::unique_ptr<Lexer> lexer;
+
+    Lexer lexer;
     stack<CodeEmitContext> contexts;
     VM* vm;
     bool unknown_global_scope;     // for eval/exec() call
@@ -39,7 +40,7 @@ class Compiler {
     void advance(int delta=1) { i += delta; }
 
     CodeEmitContext* ctx() { return &contexts.top(); }
-    CompileMode mode() const{ return lexer->src->mode; }
+    CompileMode mode() const{ return lexer.src->mode; }
     NameScope name_scope() const;
     CodeObject_ push_global_context();
     FuncDecl_ push_f_context(Str name);
@@ -61,8 +62,9 @@ class Compiler {
     Expr_ EXPR_VARS();  // special case for `for loop` and `comp`
 
     template <typename T, typename... Args>
-    std::unique_ptr<T> make_expr(Args&&... args) {
-        std::unique_ptr<T> expr = std::make_unique<T>(std::forward<Args>(args)...);
+    unique_ptr_64<T> make_expr(Args&&... args) {
+        void* p = pool64_alloc(sizeof(T));
+        unique_ptr_64<T> expr(new (p) T(std::forward<Args>(args)...));
         expr->line = prev().line;
         return expr;
     }
@@ -70,7 +72,7 @@ class Compiler {
     template<typename T>
     void _consume_comp(Expr_ expr){
         static_assert(std::is_base_of<CompExpr, T>::value);
-        std::unique_ptr<CompExpr> ce = make_expr<T>();
+        unique_ptr_64<CompExpr> ce = make_expr<T>();
         ce->expr = std::move(expr);
         ce->vars = EXPR_VARS();
         consume(TK("in"));
@@ -130,9 +132,9 @@ class Compiler {
     PyObject* to_object(const TokenValue& value);
     PyObject* read_literal();
 
-    void SyntaxError(Str msg){ lexer->throw_err("SyntaxError", msg, err().line, err().start); }
-    void SyntaxError(){ lexer->throw_err("SyntaxError", "invalid syntax", err().line, err().start); }
-    void IndentationError(Str msg){ lexer->throw_err("IndentationError", msg, err().line, err().start); }
+    void SyntaxError(Str msg){ lexer.throw_err("SyntaxError", msg, err().line, err().start); }
+    void SyntaxError(){ lexer.throw_err("SyntaxError", "invalid syntax", err().line, err().start); }
+    void IndentationError(Str msg){ lexer.throw_err("IndentationError", msg, err().line, err().start); }
 
 public:
     Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope=false);

include/pocketpy/error.h

@@ -30,7 +30,7 @@ struct SourceData {
     CompileMode mode;
 
     Str source;
-    std::vector<const char*> line_starts;
+    pod_vector<const char*> line_starts;
     
     SourceData(std::string_view source, const Str& filename, CompileMode mode);
     SourceData(const Str& filename, CompileMode mode);

include/pocketpy/expr.h

@@ -10,7 +10,47 @@ namespace pkpy{
 
 struct CodeEmitContext;
 struct Expr;
-typedef std::unique_ptr<Expr> Expr_;
+
+#define PK_POOL64_DELETE(ptr) if(ptr != nullptr) { ptr->~T(); pool64_dealloc(ptr); ptr = nullptr; }
+
+template<typename T>
+class unique_ptr_64{
+    T* ptr;
+public:
+    unique_ptr_64(): ptr(nullptr) {}
+    unique_ptr_64(T* ptr): ptr(ptr) {}
+    T* operator->() const { return ptr; }
+    T* get() const { return ptr; }
+    T* release() { T* p = ptr; ptr = nullptr; return p; }
+
+    unique_ptr_64(const unique_ptr_64&) = delete;
+    unique_ptr_64& operator=(const unique_ptr_64&) = delete;
+
+    bool operator==(std::nullptr_t) const { return ptr == nullptr; }
+    bool operator!=(std::nullptr_t) const { return ptr != nullptr; }
+
+    ~unique_ptr_64(){ PK_POOL64_DELETE(ptr) }
+
+    template<typename U>
+    unique_ptr_64(unique_ptr_64<U>&& other): ptr(other.release()) {}
+
+    operator bool() const { return ptr != nullptr; }
+
+    template<typename U>
+    unique_ptr_64& operator=(unique_ptr_64<U>&& other) {
+        PK_POOL64_DELETE(ptr)
+        ptr = other.release();
+        return *this;
+    }
+
+    unique_ptr_64& operator=(std::nullptr_t) {
+        PK_POOL64_DELETE(ptr)
+        ptr = nullptr;
+        return *this;
+    }
+};
+
+typedef unique_ptr_64<Expr> Expr_;
 
 struct Expr{
     int line = 0;
@@ -27,13 +67,11 @@ struct Expr{
 
     // for OP_DELETE_XXX
     [[nodiscard]] virtual bool emit_del(CodeEmitContext* ctx) {
-        PK_UNUSED(ctx);
         return false;
     }
 
     // for OP_STORE_XXX
     [[nodiscard]] virtual bool emit_store(CodeEmitContext* ctx) {
-        PK_UNUSED(ctx);
         return false;
     }
 };
@@ -60,7 +98,7 @@ struct CodeEmitContext{
     CodeBlock* enter_block(CodeBlockType type);
     void exit_block();
     void emit_expr();   // clear the expression stack and generate bytecode
-    int emit_(Opcode opcode, uint16_t arg, int line);
+    int emit_(Opcode opcode, uint16_t arg, int line, bool is_virtual=false);
     void patch_jump(int index);
     bool add_label(StrName name);
     int add_varname(StrName name);
@@ -316,7 +354,7 @@ struct BinaryExpr: Expr{
     Expr_ lhs;
     Expr_ rhs;
     bool is_compare() const override;
-    void _emit_compare(CodeEmitContext* ctx, std::vector<int>& jmps);
+    void _emit_compare(CodeEmitContext* ctx, pod_vector<int>& jmps);
     void emit_(CodeEmitContext* ctx) override;
 };
 

include/pocketpy/frame.h

@@ -20,7 +20,6 @@ struct FastLocals{
     PyObject* operator[](int i) const { return a[i]; }
 
     FastLocals(const CodeObject* co, PyObject** a): varnames_inv(&co->varnames_inv), a(a) {}
-    FastLocals(const FastLocals& other): varnames_inv(other.varnames_inv), a(other.a) {}
 
     PyObject** try_get_name(StrName name);
     NameDict_ to_namedict();
@@ -127,4 +126,12 @@ struct Frame {
     }
 };
 
+struct FrameId{
+    std::vector<pkpy::Frame>* data;
+    int index;
+    FrameId(std::vector<pkpy::Frame>* data, int index) : data(data), index(index) {}
+    Frame* operator->() const { return &data->operator[](index); }
+    Frame* get() const { return &data->operator[](index); }
+};
+
 }; // namespace pkpy

include/pocketpy/lexer.h

@@ -104,7 +104,7 @@ struct Lexer {
     const char* curr_char;
     int current_line = 1;
     std::vector<Token> nexts;
-    stack<int> indents;
+    stack_no_copy<int, pod_vector<int>> indents;
     int brackets_level = 0;
     bool used = false;
 

include/pocketpy/linalg.h

@@ -10,7 +10,6 @@ struct Vec2{
     float x, y;
     Vec2() : x(0.0f), y(0.0f) {}
     Vec2(float x, float y) : x(x), y(y) {}
-    Vec2(const Vec2& v) = default;
 
     Vec2 operator+(const Vec2& v) const { return Vec2(x + v.x, y + v.y); }
     Vec2 operator-(const Vec2& v) const { return Vec2(x - v.x, y - v.y); }
@@ -34,7 +33,6 @@ struct Vec3{
     float x, y, z;
     Vec3() : x(0.0f), y(0.0f), z(0.0f) {}
     Vec3(float x, float y, float z) : x(x), y(y), z(z) {}
-    Vec3(const Vec3& v) = default;
 
     Vec3 operator+(const Vec3& v) const { return Vec3(x + v.x, y + v.y, z + v.z); }
     Vec3 operator-(const Vec3& v) const { return Vec3(x - v.x, y - v.y, z - v.z); }
@@ -57,7 +55,6 @@ struct Vec4{
     float x, y, z, w;
     Vec4() : x(0.0f), y(0.0f), z(0.0f), w(0.0f) {}
     Vec4(float x, float y, float z, float w) : x(x), y(y), z(z), w(w) {}
-    Vec4(const Vec4& v) = default;
 
     Vec4 operator+(const Vec4& v) const { return Vec4(x + v.x, y + v.y, z + v.z, w + v.w); }
     Vec4 operator-(const Vec4& v) const { return Vec4(x - v.x, y - v.y, z - v.z, w - v.w); }
@@ -88,7 +85,6 @@ struct Mat3x3{
 
     Mat3x3();
     Mat3x3(float, float, float, float, float, float, float, float, float);
-    Mat3x3(const Mat3x3& other) = default;
 
     static Mat3x3 zeros();
     static Mat3x3 ones();
@@ -122,7 +118,6 @@ struct PyVec2: Vec2 {
 
     PyVec2() : Vec2() {}
     PyVec2(const Vec2& v) : Vec2(v) {}
-    PyVec2(const PyVec2& v) = default;
     Vec2* _() { return this; }
 
     static void _register(VM* vm, PyObject* mod, PyObject* type);
@@ -133,7 +128,6 @@ struct PyVec3: Vec3 {
 
     PyVec3() : Vec3() {}
     PyVec3(const Vec3& v) : Vec3(v) {}
-    PyVec3(const PyVec3& v) = default;
     Vec3* _() { return this; }
 
     static void _register(VM* vm, PyObject* mod, PyObject* type);
@@ -144,7 +138,6 @@ struct PyVec4: Vec4{
 
     PyVec4(): Vec4(){}
     PyVec4(const Vec4& v): Vec4(v){}
-    PyVec4(const PyVec4& v) = default;
     Vec4* _(){ return this; }
 
     static void _register(VM* vm, PyObject* mod, PyObject* type);
@@ -155,7 +148,6 @@ struct PyMat3x3: Mat3x3{
 
     PyMat3x3(): Mat3x3(){}
     PyMat3x3(const Mat3x3& other): Mat3x3(other){}
-    PyMat3x3(const PyMat3x3& other) = default;
     Mat3x3* _(){ return this; }
 
     static void _register(VM* vm, PyObject* mod, PyObject* type);

include/pocketpy/namedict.h

@@ -91,11 +91,17 @@ struct SmallNameDict{
     uint16_t capacity() const { return PK_SMALL_NAME_DICT_CAPACITY; }
 };
 
+template<typename T>
+struct NameDictItem{
+    StrName first;
+    T second;
+};
+
 template<typename T>
 struct LargeNameDict {
     PK_ALWAYS_PASS_BY_POINTER(LargeNameDict)
 
-    using Item = std::pair<StrName, T>;
+    using Item = NameDictItem<T>;
     static constexpr uint16_t kInitialCapacity = 32;
     static_assert(is_pod<T>::value);
 
@@ -223,8 +229,8 @@ while(!_items[i].first.empty()) {           \
         }
     }
 
-    std::vector<StrName> keys() const {
-        std::vector<StrName> v;
+    pod_vector<StrName> keys() const {
+        pod_vector<StrName> v;
         for(uint16_t i=0; i<_capacity; i++){
             if(_items[i].first.empty()) continue;
             v.push_back(_items[i].first);
@@ -307,18 +313,18 @@ struct NameDictImpl{
         else _large.apply(func);
     }
 
-    std::vector<StrName> keys() const{
-        std::vector<StrName> v;
+    pod_vector<StrName> keys() const{
+        pod_vector<StrName> v;
         apply([&](StrName key, V val){
             v.push_back(key);
         });
         return v;
     }
 
-    std::vector<std::pair<StrName, V>> items() const{
-        std::vector<std::pair<StrName, V>> v;
+    pod_vector<NameDictItem<V>> items() const{
+        pod_vector<NameDictItem<V>> v;
         apply([&](StrName key, V val){
-            v.push_back({key, val});
+            v.push_back(NameDictItem<V>{key, val});
         });
         return v;
     }

include/pocketpy/obj.h

@@ -75,7 +75,6 @@ struct Bytes{
     Bytes(const Str& str): Bytes(str.sv()) {}
     operator bool() const noexcept { return _data != nullptr; }
 
-    Bytes(const std::vector<unsigned char>& v);
     Bytes(std::string_view sv);
     Bytes(const Bytes& rhs);
     Bytes(Bytes&& rhs) noexcept;

include/pocketpy/profiler.h

@@ -4,28 +4,30 @@
 
 namespace pkpy {
 
-struct LineRecord{
+struct _LineRecord{
     int line;
     i64 hits;
     clock_t time;
 
-    LineRecord(): line(-1), hits(0), time(0) {}
+    _LineRecord(): line(-1), hits(0), time(0) {}
     bool is_valid() const { return line != -1; }
 };
 
-struct LineProfiler{
+struct _FrameRecord{
+    FrameId frame;
     clock_t prev_time;
-    LineRecord* prev_record;
-    int prev_line;
+    _LineRecord* prev_record;
+};
 
+struct LineProfiler{
     // filename -> records
-    std::map<std::string_view, std::vector<LineRecord>> records;
-
+    std::map<std::string_view, std::vector<_LineRecord>> records;
+    stack<_FrameRecord> frames;
     std::set<FuncDecl*> functions;
 
     void begin();
-    void _step(Frame* frame);
-    void _step_end();
+    void _step(FrameId frame);
+    void _step_end(FrameId frame, int line);
     void end();
     Str stats();
 };

include/pocketpy/str.h

@@ -76,8 +76,8 @@ struct Str{
     int index(const Str& sub, int start=0) const;
     Str replace(char old, char new_) const;
     Str replace(const Str& old, const Str& new_, int count=-1) const;
-    std::vector<std::string_view> split(const Str& sep) const;
-    std::vector<std::string_view> split(char sep) const;
+    pod_vector<std::string_view> split(const Str& sep) const;
+    pod_vector<std::string_view> split(char sep) const;
     int count(const Str& sub) const;
 
     /*************unicode*************/

include/pocketpy/vector.h

@@ -7,25 +7,36 @@ namespace pkpy{
 
 template<typename T>
 struct pod_vector{
-    static_assert(64 % sizeof(T) == 0);
+    static constexpr int SizeT = sizeof(T);
+    static constexpr int N = 64 / SizeT;
+
+    // static_assert(64 % SizeT == 0);
     static_assert(is_pod<T>::value);
-    static constexpr int N = 64 / sizeof(T);
     static_assert(N >= 4);
+
     int _size;
     int _capacity;
     T* _data;
 
+    using size_type = int;
+
     pod_vector(): _size(0), _capacity(N) {
-        _data = (T*)pool64_alloc(_capacity * sizeof(T));
+        _data = (T*)pool64_alloc(_capacity * SizeT);
+    }
+
+    // support initializer list
+    pod_vector(std::initializer_list<T> il): _size(il.size()), _capacity(std::max(N, _size)) {
+        _data = (T*)pool64_alloc(_capacity * SizeT);
+        for(int i=0; i<_size; i++) _data[i] = *(il.begin() + i);
     }
 
     pod_vector(int size): _size(size), _capacity(std::max(N, size)) {
-        _data = (T*)pool64_alloc(_capacity * sizeof(T));
+        _data = (T*)pool64_alloc(_capacity * SizeT);
     }
 
     pod_vector(const pod_vector& other): _size(other._size), _capacity(other._capacity) {
-        _data = (T*)pool64_alloc(_capacity * sizeof(T));
-        memcpy(_data, other._data, sizeof(T) * _size);
+        _data = (T*)pool64_alloc(_capacity * SizeT);
+        memcpy(_data, other._data, SizeT * _size);
     }
 
     pod_vector(pod_vector&& other) noexcept {
@@ -63,9 +74,9 @@ struct pod_vector{
         if(cap <= _capacity) return;
         _capacity = cap;
         T* old_data = _data;
-        _data = (T*)pool64_alloc(_capacity * sizeof(T));
+        _data = (T*)pool64_alloc(_capacity * SizeT);
         if(old_data != nullptr){
-            memcpy(_data, old_data, sizeof(T) * _size);
+            memcpy(_data, old_data, SizeT * _size);
             pool64_dealloc(old_data);
         }
     }
@@ -146,7 +157,7 @@ public:
 	void pop(){ vec.pop_back(); }
 	void clear(){ vec.clear(); }
 	bool empty() const { return vec.empty(); }
-	size_t size() const { return vec.size(); }
+	typename Container::size_type size() const { return vec.size(); }
 	T& top(){ return vec.back(); }
 	const T& top() const { return vec.back(); }
 	T popx(){ T t = std::move(vec.back()); vec.pop_back(); return t; }

include/pocketpy/vm.h

@@ -29,19 +29,17 @@ namespace pkpy{
 #define DEF_NATIVE_2(ctype, ptype)                                      \
     template<> inline ctype py_cast<ctype>(VM* vm, PyObject* obj) {     \
         vm->check_non_tagged_type(obj, vm->ptype);                      \
-        return PK_OBJ_GET(ctype, obj);                                     \
+        return PK_OBJ_GET(ctype, obj);                                  \
     }                                                                   \
     template<> inline ctype _py_cast<ctype>(VM* vm, PyObject* obj) {    \
-        PK_UNUSED(vm);                                                  \
-        return PK_OBJ_GET(ctype, obj);                                     \
+        return PK_OBJ_GET(ctype, obj);                                  \
     }                                                                   \
     template<> inline ctype& py_cast<ctype&>(VM* vm, PyObject* obj) {   \
         vm->check_non_tagged_type(obj, vm->ptype);                      \
-        return PK_OBJ_GET(ctype, obj);                                     \
+        return PK_OBJ_GET(ctype, obj);                                  \
     }                                                                   \
     template<> inline ctype& _py_cast<ctype&>(VM* vm, PyObject* obj) {  \
-        PK_UNUSED(vm);                                                  \
-        return PK_OBJ_GET(ctype, obj);                                     \
+        return PK_OBJ_GET(ctype, obj);                                  \
     }                                                                   \
     inline PyObject* py_var(VM* vm, const ctype& value) { return vm->heap.gcnew<ctype>(vm->ptype, value);}     \
     inline PyObject* py_var(VM* vm, ctype&& value) { return vm->heap.gcnew<ctype>(vm->ptype, std::move(value));}
@@ -56,7 +54,7 @@ struct PyTypeInfo{
     StrName name;
     bool subclass_enabled;
 
-    std::vector<StrName> annotated_fields;
+    pod_vector<StrName> annotated_fields = {};
 
     // cached special methods
     // unary operators
@@ -105,14 +103,6 @@ struct PyTypeInfo{
 
 };
 
-struct FrameId{
-    std::vector<pkpy::Frame>* data;
-    int index;
-    FrameId(std::vector<pkpy::Frame>* data, int index) : data(data), index(index) {}
-    Frame* operator->() const { return &data->operator[](index); }
-    Frame* get() const { return &data->operator[](index); }
-};
-
 typedef void(*PrintFunc)(const char*, int);
 
 class VM {
@@ -324,13 +314,12 @@ public:
     template<typename T, typename __T>
     PyObject* bind_notimplemented_constructor(__T&& type) {
         return bind_constructor<-1>(std::forward<__T>(type), [](VM* vm, ArgsView args){
-            PK_UNUSED(args);
             vm->NotImplementedError();
             return vm->None;
         });
     }
 
-    int normalized_index(int index, int size);
+    i64 normalized_index(i64 index, int size);
     PyObject* py_next(PyObject* obj);
     bool py_callable(PyObject* obj);
     
@@ -396,7 +385,7 @@ public:
 
     struct ImportContext{
         std::vector<Str> pending;
-        std::vector<bool> pending_is_init;   // a.k.a __init__.py
+        pod_vector<bool> pending_is_init;   // a.k.a __init__.py
         struct Temp{
             ImportContext* ctx;
             Temp(ImportContext* ctx, Str name, bool is_init) : ctx(ctx){
@@ -416,7 +405,7 @@ public:
 
     ImportContext _import_context;
     PyObject* py_import(Str path, bool throw_err=true);
-    ~VM();
+    virtual ~VM();
 
 #if PK_DEBUG_CEVAL_STEP
     void _log_s_data(const char* title = nullptr);
@@ -481,7 +470,6 @@ template<> inline T py_cast<T>(VM* vm, PyObject* obj){  \
     return 0;                                               \
 }                                                           \
 template<> inline T _py_cast<T>(VM* vm, PyObject* obj){     \
-    PK_UNUSED(vm);                                          \
     if(is_small_int(obj)) return (T)(PK_BITS(obj) >> 2);    \
     return (T)PK_OBJ_GET(i64, obj);                         \
 }
@@ -542,12 +530,10 @@ PY_VAR_INT(unsigned long long)
 #undef PY_VAR_INT
 
 inline PyObject* py_var(VM* vm, float _val){
-    PK_UNUSED(vm);
     return tag_float(static_cast<f64>(_val));
 }
 
 inline PyObject* py_var(VM* vm, double _val){
-    PK_UNUSED(vm);
     return tag_float(static_cast<f64>(_val));
 }
 
@@ -599,7 +585,6 @@ inline PyObject* py_var(VM* vm, std::string_view val){
 }
 
 inline PyObject* py_var(VM* vm, NoReturn val){
-    PK_UNUSED(val);
     return vm->None;
 }
 
@@ -629,24 +614,4 @@ PyObject* VM::bind_func(PyObject* obj, Str name, NativeFuncC fn, UserData userda
     return nf;
 }
 
-/***************************************************/
-
-template<typename T>
-PyObject* PyArrayGetItem(VM* vm, PyObject* obj, PyObject* index){
-    static_assert(std::is_same_v<T, List> || std::is_same_v<T, Tuple>);
-    const T& self = _CAST(T&, obj);
-
-    if(is_non_tagged_type(index, vm->tp_slice)){
-        const Slice& s = _CAST(Slice&, index);
-        int start, stop, step;
-        vm->parse_int_slice(s, self.size(), start, stop, step);
-        List new_list;
-        for(int i=start; step>0?i<stop:i>stop; i+=step) new_list.push_back(self[i]);
-        return VAR(T(std::move(new_list)));
-    }
-
-    int i = CAST(int, index);
-    i = vm->normalized_index(i, self.size());
-    return self[i];
-}
 }   // namespace pkpy

prebuild.py

@@ -2,7 +2,7 @@ import os
 
 def generate_python_sources():
     sources = {}
-    for file in os.listdir("python"):
+    for file in sorted(os.listdir("python")):
         if not file.endswith(".py"):
             continue
         key = file.split(".")[0]
@@ -36,5 +36,6 @@ namespace pkpy{
 '''
     return header
 
-with open("include/pocketpy/_generated.h", "w", encoding='utf-8') as f:
+# use LF line endings instead of CRLF
+with open("include/pocketpy/_generated.h", "wt", encoding='utf-8', newline='\n') as f:
     f.write(generate_python_sources())

scripts/genstub.py

@@ -25,7 +25,8 @@ for line in lines:
         ret + ' ' + body + ' {\n' + mock_string + '\n}\n'
     )
 
-with open('src2/pocketpy_c.c', 'w') as f:
+# use LF line endings instead of CRLF
+with open('src2/pocketpy_c.c', 'wt', encoding='utf-8', newline='\n') as f:
     f.write('''
 #include "pocketpy_c.h"
 

src/ceval.cpp

@@ -68,7 +68,7 @@ PyObject* VM::_run_top_frame(){
 
 #define CEVAL_STEP_CALLBACK() \
     if(_ceval_on_step) _ceval_on_step(this, frame.get(), byte); \
-    if(_profiler) _profiler->_step(frame.get());
+    if(_profiler) _profiler->_step(frame);
 
 #define DISPATCH_OP_CALL() { frame = top_frame(); goto __NEXT_FRAME; }
 __NEXT_FRAME:

src/collections.cpp

@@ -76,25 +76,25 @@ namespace pkpy
         vm->bind__getitem__(PK_OBJ_GET(Type, type), [](VM *vm, PyObject* _0, PyObject* _1)
         {
             PyDeque &self = _CAST(PyDeque &, _0);
-            int index = CAST(int, _1);
+            i64 index = CAST(i64, _1);
             index = vm->normalized_index(index, self.dequeItems.size()); // error is handled by the vm->normalized_index
-            return self.dequeItems.at(index);
+            return self.dequeItems[index];
         });
         // sets the item at the given index, if index is negative, it will be treated as index + len(deque)
         // if the index is out of range, IndexError will be thrown --> required for [] operator
         vm->bind__setitem__(PK_OBJ_GET(Type, type), [](VM *vm, PyObject* _0, PyObject* _1, PyObject* _2)
         {
             PyDeque &self = _CAST(PyDeque&, _0);
-            int index = CAST(int, _1);
+            i64 index = CAST(i64, _1);
             index = vm->normalized_index(index, self.dequeItems.size()); // error is handled by the vm->normalized_index
-            self.dequeItems.at(index) = _2;
+            self.dequeItems[index] = _2;
         });
         // erases the item at the given index, if index is negative, it will be treated as index + len(deque)
         // if the index is out of range, IndexError will be thrown --> required for [] operator
         vm->bind__delitem__(PK_OBJ_GET(Type, type), [](VM *vm, PyObject* _0, PyObject* _1)
         {
             PyDeque &self = _CAST(PyDeque&, _0);
-            int index = CAST(int, _1);
+            i64 index = CAST(i64, _1);
             index = vm->normalized_index(index, self.dequeItems.size()); // error is handled by the vm->normalized_index
             self.dequeItems.erase(self.dequeItems.begin() + index);
         });
@@ -258,17 +258,11 @@ namespace pkpy
                      // Return the position of x in the deque (at or after index start and before index stop). Returns the first match or raises ValueError if not found.
                      PyDeque &self = _CAST(PyDeque &, args[0]);
                      PyObject *obj = args[1];
-                     int start = 0, stop = self.dequeItems.size(); // default values
-                     if (!vm->py_eq(args[2], vm->None))
-                         start = CAST(int, args[2]);
-                     if (!vm->py_eq(args[3], vm->None))
-                         stop = CAST(int, args[3]);
+                     int start = CAST_DEFAULT(int, args[2], 0);
+                     int stop = CAST_DEFAULT(int, args[3], self.dequeItems.size());
                      int index = self.findIndex(vm, obj, start, stop);
-                     if (index != -1)
-                         return VAR(index);
-                     else
-                         vm->ValueError(_CAST(Str &, vm->py_repr(obj)) + " is not in deque");
-                     return vm->None;
+                     if (index < 0) vm->ValueError(_CAST(Str &, vm->py_repr(obj)) + " is not in deque");
+                     return VAR(index);
                  });
         // NEW: returns the index of the given object in the deque
         vm->bind(type, "__contains__(self, obj) -> bool",

src/compiler.cpp

@@ -9,7 +9,7 @@ namespace pkpy{
     }
 
     CodeObject_ Compiler::push_global_context(){
-        CodeObject_ co = std::make_shared<CodeObject>(lexer->src, lexer->src->filename);
+        CodeObject_ co = std::make_shared<CodeObject>(lexer.src, lexer.src->filename);
         co->start_line = i==0 ? 1 : prev().line;
         contexts.push(CodeEmitContext(vm, co, contexts.size()));
         return co;
@@ -17,7 +17,7 @@ namespace pkpy{
 
     FuncDecl_ Compiler::push_f_context(Str name){
         FuncDecl_ decl = std::make_shared<FuncDecl>();
-        decl->code = std::make_shared<CodeObject>(lexer->src, name);
+        decl->code = std::make_shared<CodeObject>(lexer.src, name);
         decl->code->start_line = i==0 ? 1 : prev().line;
         decl->nested = name_scope() == NAME_LOCAL;
         contexts.push(CodeEmitContext(vm, decl->code, contexts.size()));
@@ -32,14 +32,14 @@ namespace pkpy{
         // add a `return None` in the end as a guard
         // previously, we only do this if the last opcode is not a return
         // however, this is buggy...since there may be a jump to the end (out of bound) even if the last opcode is a return
-        ctx()->emit_(OP_RETURN_VALUE, 1, BC_KEEPLINE);
+        ctx()->emit_(OP_RETURN_VALUE, 1, BC_KEEPLINE, true);
         // find the last valid token
         int j = i-1;
         while(tokens[j].type == TK("@eol") || tokens[j].type == TK("@dedent") || tokens[j].type == TK("@eof")) j--;
         ctx()->co->end_line = tokens[j].line;
 
         // some check here
-        std::vector<Bytecode>& codes = ctx()->co->codes;
+        auto& codes = ctx()->co->codes;
         if(ctx()->co->varnames.size() > PK_MAX_CO_VARNAMES){
             SyntaxError("maximum number of local variables exceeded");
         }
@@ -627,7 +627,7 @@ __EAT_DOTS_END:
         ctx()->emit_expr();
         int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line);
         compile_block_body();
-        ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE);
+        ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE, true);
         ctx()->patch_jump(patch);
         ctx()->exit_block();
         // optional else clause
@@ -647,7 +647,7 @@ __EAT_DOTS_END:
         bool ok = vars->emit_store(ctx());
         if(!ok) SyntaxError();  // this error occurs in `vars` instead of this line, but...nevermind
         compile_block_body();
-        ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE);
+        ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE, true);
         ctx()->exit_block();
         // optional else clause
         if (match(TK("else"))) {
@@ -659,7 +659,7 @@ __EAT_DOTS_END:
     void Compiler::compile_try_except() {
         ctx()->enter_block(CodeBlockType::TRY_EXCEPT);
         compile_block_body();
-        std::vector<int> patches = {
+        pod_vector<int> patches = {
             ctx()->emit_(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE)
         };
         ctx()->exit_block();
@@ -805,9 +805,9 @@ __EAT_DOTS_END:
                 ctx()->co->is_generator = true;
                 ctx()->emit_(OP_GET_ITER, BC_NOARG, kw_line);
                 ctx()->enter_block(CodeBlockType::FOR_LOOP);
-                ctx()->emit_(OP_FOR_ITER, BC_NOARG, BC_KEEPLINE);
-                ctx()->emit_(OP_YIELD_VALUE, BC_NOARG, BC_KEEPLINE);
-                ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE);
+                ctx()->emit_(OP_FOR_ITER, BC_NOARG, kw_line);
+                ctx()->emit_(OP_YIELD_VALUE, BC_NOARG, kw_line);
+                ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), kw_line);
                 ctx()->exit_block();
                 consume_end_stmt();
                 break;
@@ -912,7 +912,7 @@ __EAT_DOTS_END:
                 }
                 ctx()->emit_(OP_WITH_ENTER, BC_NOARG, prev().line);
                 // [ <expr> <expr>.__enter__() ]
-                if(as_name){
+                if(as_name != nullptr){
                     bool ok = as_name->emit_store(ctx());
                     if(!ok) SyntaxError();
                 }else{
@@ -1178,13 +1178,11 @@ __EAT_DOTS_END:
         return nullptr;
     }
 
-    Compiler::Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope){
+    Compiler::Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope)
+            :lexer(vm, std::make_shared<SourceData>(source, filename, mode)){
         this->vm = vm;
         this->used = false;
         this->unknown_global_scope = unknown_global_scope;
-        this->lexer = std::make_unique<Lexer>(
-            vm, std::make_shared<SourceData>(source, filename, mode)
-        );
         init_pratt_rules();
     }
 
@@ -1193,7 +1191,7 @@ __EAT_DOTS_END:
         PK_ASSERT(!used)
         used = true;
 
-        tokens = lexer->run();
+        tokens = lexer.run();
         CodeObject_ code = push_global_context();
 
         advance();          // skip @sof, so prev() is always valid

src/error.cpp

@@ -24,7 +24,7 @@ namespace pkpy{
         if(lineno == -1) return {nullptr, nullptr};
         lineno -= 1;
         if(lineno < 0) lineno = 0;
-        const char* _start = line_starts.at(lineno);
+        const char* _start = line_starts[lineno];
         const char* i = _start;
         // max 300 chars
         while(*i != '\n' && *i != '\0' && i-_start < 300) i++;

src/expr.cpp

@@ -41,7 +41,7 @@ namespace pkpy{
 
         if(curr_type == CodeBlockType::FOR_LOOP){
             // add a no op here to make block check work
-            emit_(OP_NO_OP, BC_NOARG, BC_KEEPLINE);
+            emit_(OP_NO_OP, BC_NOARG, BC_KEEPLINE, true);
         }
     }
 
@@ -52,14 +52,14 @@ namespace pkpy{
         expr->emit_(this);
     }
 
-    int CodeEmitContext::emit_(Opcode opcode, uint16_t arg, int line) {
+    int CodeEmitContext::emit_(Opcode opcode, uint16_t arg, int line, bool is_virtual) {
         co->codes.push_back(Bytecode{(uint8_t)opcode, arg});
         co->iblocks.push_back(curr_block_i);
-        co->lines.push_back(line);
+        co->lines.push_back(CodeObject::LineInfo{line, is_virtual});
         int i = co->codes.size() - 1;
-        if(line==BC_KEEPLINE){
-            if(i>=1) co->lines[i] = co->lines[i-1];
-            else co->lines[i] = 1;
+        if(line == BC_KEEPLINE){
+            if(i >= 1) co->lines[i].lineno = co->lines[i-1].lineno;
+            else co->lines[i].lineno = 1;
         }
         return i;
     }
@@ -613,7 +613,7 @@ namespace pkpy{
         }
     }
 
-    void BinaryExpr::_emit_compare(CodeEmitContext* ctx, std::vector<int>& jmps){
+    void BinaryExpr::_emit_compare(CodeEmitContext* ctx, pod_vector<int>& jmps){
         if(lhs->is_compare()){
             static_cast<BinaryExpr*>(lhs.get())->_emit_compare(ctx, jmps);
         }else{
@@ -637,7 +637,7 @@ namespace pkpy{
     }
 
     void BinaryExpr::emit_(CodeEmitContext* ctx) {
-        std::vector<int> jmps;
+        pod_vector<int> jmps;
         if(is_compare() && lhs->is_compare()){
             // (a < b) < c
             static_cast<BinaryExpr*>(lhs.get())->_emit_compare(ctx, jmps);

src/io.cpp

@@ -36,7 +36,6 @@ unsigned char* _default_import_handler(const char* name_p, int name_size, int* o
     unsigned char* buffer = new unsigned char[buffer_size];
     fseek(fp, 0, SEEK_SET);
     size_t sz = io_fread(buffer, 1, buffer_size, fp);
-    PK_UNUSED(sz);
     fclose(fp);
     *out_size = buffer_size;
     return buffer;

src/lexer.cpp

@@ -64,11 +64,11 @@ static bool is_unicode_Lo_char(uint32_t c) {
         // https://docs.python.org/3/reference/lexical_analysis.html#indentation
         if(spaces > indents.top()){
             indents.push(spaces);
-            nexts.push_back(Token{TK("@indent"), token_start, 0, current_line, brackets_level});
+            nexts.push_back(Token{TK("@indent"), token_start, 0, current_line, brackets_level, {}});
         } else if(spaces < indents.top()){
             while(spaces < indents.top()){
                 indents.pop();
-                nexts.push_back(Token{TK("@dedent"), token_start, 0, current_line, brackets_level});
+                nexts.push_back(Token{TK("@dedent"), token_start, 0, current_line, brackets_level, {}});
             }
             if(spaces != indents.top()){
                 return false;
@@ -109,8 +109,8 @@ static bool is_unicode_Lo_char(uint32_t c) {
                 }
             }
             // handle multibyte char
-            std::string u8str(curr_char, u8bytes);
-            if(u8str.size() != u8bytes) return 2;
+            Str u8str(curr_char, u8bytes);
+            if(u8str.size != u8bytes) return 2;
             uint32_t value = 0;
             for(int k=0; k < u8bytes; k++){
                 uint8_t b = u8str[k];
@@ -204,7 +204,7 @@ static bool is_unicode_Lo_char(uint32_t c) {
 
     Str Lexer::eat_string_until(char quote, bool raw) {
         bool quote3 = match_n_chars(2, quote);
-        std::vector<char> buff;
+        pod_vector<char> buff;
         while (true) {
             char c = eatchar_include_newline();
             if (c == quote){
@@ -467,7 +467,7 @@ static bool is_unicode_Lo_char(uint32_t c) {
     Lexer::Lexer(VM* vm, std::shared_ptr<SourceData> src) : vm(vm), src(src) {
         this->token_start = src->source.c_str();
         this->curr_char = src->source.c_str();
-        this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level});
+        this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level, {}});
         this->indents.push(0);
     }
 

src/linalg.cpp

@@ -422,19 +422,16 @@ static Vec2 SmoothDamp(Vec2 current, Vec2 target, PyVec2& currentVelocity, float
 
         // @staticmethod
         vm->bind(type, "zeros()", [](VM* vm, ArgsView args){
-            PK_UNUSED(args);
             return VAR_T(PyMat3x3, Mat3x3::zeros());
         }, {}, BindType::STATICMETHOD);
 
         // @staticmethod
         vm->bind(type, "ones()", [](VM* vm, ArgsView args){
-            PK_UNUSED(args);
             return VAR_T(PyMat3x3, Mat3x3::ones());
         }, {}, BindType::STATICMETHOD);
 
         // @staticmethod
         vm->bind(type, "identity()", [](VM* vm, ArgsView args){
-            PK_UNUSED(args);
             return VAR_T(PyMat3x3, Mat3x3::identity());
         }, {}, BindType::STATICMETHOD);
 

src/memory.cpp

@@ -123,8 +123,8 @@ struct DoubleLinkedList{
 
 template<int __BlockSize=128>
 struct MemoryPool{
-    static const size_t __MaxBlocks = 256*1024 / __BlockSize;
-    static const size_t __MinArenaCount = PK_GC_MIN_THRESHOLD*100 / (256*1024);
+    static const int __MaxBlocks = 256*1024 / __BlockSize;
+    static const int __MinArenaCount = PK_GC_MIN_THRESHOLD*100 / (256*1024);
 
     struct Block{
         void* arena;

src/obj.cpp

@@ -14,11 +14,6 @@ namespace pkpy{
     }
     bool Bytes::operator!=(const Bytes& rhs) const{ return !(*this == rhs); }
 
-    Bytes::Bytes(const std::vector<unsigned char>& v){
-        _data = new unsigned char[v.size()];
-        _size = v.size();
-        for(int i=0; i<_size; i++) _data[i] = v[i];
-    }
     Bytes::Bytes(std::string_view sv){
         _data = new unsigned char[sv.size()];
         _size = sv.size();

src/pocketpy.cpp

@@ -6,6 +6,27 @@ namespace pkpy{
 void add_module_cjson(VM* vm);
 #endif
 
+template<typename T>
+PyObject* PyArrayGetItem(VM* vm, PyObject* _0, PyObject* _1){
+    static_assert(std::is_same_v<T, List> || std::is_same_v<T, Tuple>);
+    const T& self = _CAST(T&, _0);
+    i64 index;
+    if(try_cast_int(_1, &index)){
+        index = vm->normalized_index(index, self.size());
+        return self[index];
+    }
+    if(is_non_tagged_type(_1, vm->tp_slice)){
+        const Slice& s = _CAST(Slice&, _1);
+        int start, stop, step;
+        vm->parse_int_slice(s, self.size(), start, stop, step);
+        List new_list;
+        for(int i=start; step>0?i<stop:i>stop; i+=step) new_list.push_back(self[i]);
+        return VAR(T(std::move(new_list)));
+    }
+    vm->TypeError("indices must be integers or slices");
+    PK_UNREACHABLE()
+}
+
 void init_builtins(VM* _vm) {
 #define BIND_NUM_ARITH_OPT(name, op)                                                                    \
     _vm->bind##name(VM::tp_int, [](VM* vm, PyObject* lhs, PyObject* rhs) {                              \
@@ -278,11 +299,11 @@ void init_builtins(VM* _vm) {
     _vm->bind_func<1>(_vm->builtins, "dir", [](VM* vm, ArgsView args) {
         std::set<StrName> names;
         if(!is_tagged(args[0]) && args[0]->is_attr_valid()){
-            std::vector<StrName> keys = args[0]->attr().keys();
+            auto keys = args[0]->attr().keys();
             names.insert(keys.begin(), keys.end());
         }
         const NameDict& t_attr = vm->_t(args[0])->attr();
-        std::vector<StrName> keys = t_attr.keys();
+        auto keys = t_attr.keys();
         names.insert(keys.begin(), keys.end());
         List ret;
         for (StrName name : names) ret.push_back(VAR(name.sv()));
@@ -529,7 +550,7 @@ void init_builtins(VM* _vm) {
             vm->parse_int_slice(s, self.u8_length(), start, stop, step);
             return VAR(self.u8_slice(start, stop, step));
         }
-        int i = CAST(int, _1);
+        i64 i = CAST(i64, _1);
         i = vm->normalized_index(i, self.u8_length());
         return VAR(self.u8_getitem(i));
     });
@@ -547,7 +568,7 @@ void init_builtins(VM* _vm) {
         const Str& self = _CAST(Str&, args[0]);
         const Str& sep = CAST(Str&, args[1]);
         if(sep.empty()) vm->ValueError("empty separator");
-        std::vector<std::string_view> parts;
+        pod_vector<std::string_view> parts;
         if(sep.size == 1){
             parts = self.split(sep[0]);
         }else{
@@ -560,7 +581,7 @@ void init_builtins(VM* _vm) {
 
     _vm->bind(_vm->_t(VM::tp_str), "splitlines(self)", [](VM* vm, ArgsView args) {
         const Str& self = _CAST(Str&, args[0]);
-        std::vector<std::string_view> parts;
+        pod_vector<std::string_view> parts;
         parts = self.split('\n');
         List ret(parts.size());
         for(int i=0; i<parts.size(); i++) ret[i] = VAR(Str(parts[i]));
@@ -573,18 +594,20 @@ void init_builtins(VM* _vm) {
         return VAR(self.count(s));
     });
 
-    _vm->bind_method<1>(VM::tp_str, "index", [](VM* vm, ArgsView args) {
+    _vm->bind(_vm->_t(VM::tp_str), "index(self, value, __start=0)", [](VM* vm, ArgsView args) {
         const Str& self = _CAST(Str&, args[0]);
-        const Str& sub = CAST(Str&, args[1]);
-        int index = self.index(sub);
-        if(index == -1) vm->ValueError("substring not found");
+        const Str& value = CAST(Str&, args[1]);
+        int start = CAST(int, args[2]);
+        int index = self.index(value, start);
+        if(index < 0) vm->ValueError("substring not found");
         return VAR(index);
     });
 
-    _vm->bind_method<1>(VM::tp_str, "find", [](VM* vm, ArgsView args) {
+    _vm->bind(_vm->_t(VM::tp_str), "find(self, value, __start=0)", [](VM* vm, ArgsView args) {
         const Str& self = _CAST(Str&, args[0]);
-        const Str& sub = CAST(Str&, args[1]);
-        return VAR(self.index(sub));
+        const Str& value = CAST(Str&, args[1]);
+        int start = CAST(int, args[2]);
+        return VAR(self.index(value, start));
     });
 
     _vm->bind_method<1>(VM::tp_str, "startswith", [](VM* vm, ArgsView args) {
@@ -782,10 +805,11 @@ void init_builtins(VM* _vm) {
         return vm->True;
     });
 
-    _vm->bind_method<1>(VM::tp_list, "index", [](VM* vm, ArgsView args) {
+    _vm->bind(_vm->_t(VM::tp_list), "index(self, value, __start=0)", [](VM* vm, ArgsView args) {
         List& self = _CAST(List&, args[0]);
         PyObject* obj = args[1];
-        for(int i=0; i<self.size(); i++){
+        int start = CAST(int, args[2]);
+        for(int i=start; i<self.size(); i++){
             if(vm->py_eq(self[i], obj)) return VAR(i);
         }
         vm->ValueError(_CAST(Str&, vm->py_repr(obj)) + " is not in list");
@@ -812,7 +836,7 @@ void init_builtins(VM* _vm) {
             return self.popx_back();
         }
         if(args.size() == 1+1){
-            int index = CAST(int, args[1]);
+            i64 index = CAST(i64, args[1]);
             index = vm->normalized_index(index, self.size());
             PyObject* ret = self[index];
             self.erase(index);
@@ -924,13 +948,13 @@ void init_builtins(VM* _vm) {
     _vm->bind__getitem__(VM::tp_list, PyArrayGetItem<List>);
     _vm->bind__setitem__(VM::tp_list, [](VM* vm, PyObject* _0, PyObject* _1, PyObject* _2){
         List& self = _CAST(List&, _0);
-        int i = CAST(int, _1);
+        i64 i = CAST(i64, _1);
         i = vm->normalized_index(i, self.size());
         self[i] = _2;
     });
     _vm->bind__delitem__(VM::tp_list, [](VM* vm, PyObject* _0, PyObject* _1){
         List& self = _CAST(List&, _0);
-        int i = CAST(int, _1);
+        i64 i = CAST(i64, _1);
         i = vm->normalized_index(i, self.size());
         self.erase(i);
     });
@@ -1024,18 +1048,18 @@ void init_builtins(VM* _vm) {
     // tp_bytes
     _vm->bind_constructor<2>(_vm->_t(VM::tp_bytes), [](VM* vm, ArgsView args){
         List& list = CAST(List&, args[1]);
-        std::vector<unsigned char> buffer(list.size());
+        unsigned char* buffer = new unsigned char[list.size()];
         for(int i=0; i<list.size(); i++){
             i64 b = CAST(i64, list[i]);
             if(b<0 || b>255) vm->ValueError("byte must be in range[0, 256)");
             buffer[i] = (char)b;
         }
-        return VAR(Bytes(buffer));
+        return VAR(Bytes(buffer, list.size()));
     });
 
     _vm->bind__getitem__(VM::tp_bytes, [](VM* vm, PyObject* obj, PyObject* index) {
         const Bytes& self = _CAST(Bytes&, obj);
-        int i = CAST(int, index);
+        i64 i = CAST(i64, index);
         i = vm->normalized_index(i, self.size());
         return VAR(self[i]);
     });
@@ -1108,15 +1132,15 @@ void init_builtins(VM* _vm) {
     _vm->bind_method<0>(VM::tp_mappingproxy, "values", [](VM* vm, ArgsView args) {
         MappingProxy& self = _CAST(MappingProxy&, args[0]);
         List values;
-        for(auto& item : self.attr().items()) values.push_back(item.second);
+        for(auto [k, v] : self.attr().items()) values.push_back(v);
         return VAR(std::move(values));
     });
 
     _vm->bind_method<0>(VM::tp_mappingproxy, "items", [](VM* vm, ArgsView args) {
         MappingProxy& self = _CAST(MappingProxy&, args[0]);
         List items;
-        for(auto& item : self.attr().items()){
-            PyObject* t = VAR(Tuple(VAR(item.first.sv()), item.second));
+        for(auto [k, v] : self.attr().items()){
+            PyObject* t = VAR(Tuple(VAR(k.sv()), v));
             items.push_back(std::move(t));
         }
         return VAR(std::move(items));
@@ -1156,11 +1180,11 @@ void init_builtins(VM* _vm) {
         ss << "mappingproxy({";
         bool first = true;
         vm->_repr_recursion_set.insert(_0);
-        for(auto& item : self.attr().items()){
+        for(auto [k, v] : self.attr().items()){
             if(!first) ss << ", ";
             first = false;
-            ss << item.first.escape() << ": ";
-            ss << CAST(Str, vm->py_repr(item.second));
+            ss << k.escape() << ": ";
+            ss << CAST(Str, vm->py_repr(v));
         }
         vm->_repr_recursion_set.erase(_0);
         ss << "})";
@@ -1436,7 +1460,6 @@ void VM::post_init(){
 
     // type
     bind__getitem__(tp_type, [](VM* vm, PyObject* self, PyObject* _){
-        PK_UNUSED(_);
         return self;        // for generics
     });
 

src/profiler.cpp

@@ -15,47 +15,69 @@ static std::string to_string_1f(f64 x){
 }
 
 void LineProfiler::begin(){
-    prev_time = 0;
-    prev_record = nullptr;
-    prev_line = -1;
-    records.clear();
+    frames.clear();
 }
 
-void LineProfiler::_step(Frame *frame){
+void LineProfiler::_step(FrameId frame){
+    auto line_info = frame->co->lines[frame->_ip];
+    if(line_info.is_virtual) return;
     std::string_view filename = frame->co->src->filename.sv();
-    int line = frame->co->lines[frame->_ip];
-    // std::string_view function = frame->co->name.sv();
+    int line = line_info.lineno;
 
-    if(prev_record == nullptr){
-        prev_time = clock();
+    if(frames.empty()){
+        frames.push({frame, clock(), nullptr});
     }else{
-        _step_end();
+        _step_end(frame, line);
     }
 
-    std::vector<LineRecord>& file_records = records[filename];
+    auto& file_records = records[filename];
     if(file_records.empty()){
+        // initialize file_records
         int total_lines = frame->co->src->line_starts.size();
         file_records.resize(total_lines + 1);
         for(int i=1; i<=total_lines; i++){
             file_records[i].line = i;
         }
     }
-    prev_record = &file_records.at(line);
+
+    frames.top().prev_record = &file_records.at(line);
 }
 
-void LineProfiler::_step_end(){
+void LineProfiler::_step_end(FrameId frame, int line){
     clock_t now = clock();
-    clock_t delta = now - prev_time;
-    prev_time = now;
-    if(prev_record->line != prev_line){
-        prev_record->hits++;
-        prev_line = prev_record->line;
+    _FrameRecord& top_frame_record = frames.top();
+    _LineRecord* prev_record = top_frame_record.prev_record;
+
+    int id_delta = frame.index - top_frame_record.frame.index;
+    PK_ASSERT(id_delta >= -1 && id_delta <= 1);
+
+    // current line is about to change
+    if(prev_record->line != line){
+        clock_t delta = now - top_frame_record.prev_time;
+        top_frame_record.prev_time = now;
+        if(id_delta != 1) prev_record->hits++;
+        prev_record->time += delta;
+    }
+    
+    if(id_delta == 1){
+        frames.push({frame, now, nullptr});
+    }else{
+        if(id_delta == -1) frames.pop();
     }
-    prev_record->time += delta;
 }
 
 void LineProfiler::end(){
-    _step_end();
+    clock_t now = clock();
+    _FrameRecord& top_frame_record = frames.top();
+    _LineRecord* prev_record = top_frame_record.prev_record;
+
+    clock_t delta = now - top_frame_record.prev_time;
+    top_frame_record.prev_time = now;
+    prev_record->hits++;
+    prev_record->time += delta;
+
+    frames.pop();
+    PK_ASSERT(frames.empty());
 }
 
 Str LineProfiler::stats(){
@@ -65,7 +87,7 @@ Str LineProfiler::stats(){
         int end_line = decl->code->end_line;
         if(start_line == -1 || end_line == -1) continue;
         std::string_view filename = decl->code->src->filename.sv();
-        std::vector<LineRecord>& file_records = records[filename];
+        std::vector<_LineRecord>& file_records = records[filename];
         if(file_records.empty()) continue;
         clock_t total_time = 0;
         for(int line = start_line; line <= end_line; line++){
@@ -77,7 +99,7 @@ Str LineProfiler::stats(){
         ss << "Line #      Hits         Time  Per Hit   % Time  Line Contents\n";
         ss << "==============================================================\n";
         for(int line = start_line; line <= end_line; line++){
-            const LineRecord& record = file_records.at(line);
+            const _LineRecord& record = file_records.at(line);
             if(!record.is_valid()) continue;
             ss << left_pad(std::to_string(line), 6);
             if(record.hits == 0){

src/random.cpp

@@ -61,7 +61,7 @@ struct Random{
             Random& self = _CAST(Random&, args[0]);
             auto [data, size] = vm->_cast_array(args[1]);
             if(size == 0) vm->IndexError("cannot choose from an empty sequence");
-            std::vector<f64> cum_weights(size);
+            pod_vector<f64> cum_weights(size);
             if(args[2] == vm->None){
                 for(int i = 0; i < size; i++) cum_weights[i] = i + 1;
             }else{

src/str.cpp

@@ -14,7 +14,7 @@ int utf8len(unsigned char c, bool suppress){
 }
 
 #define PK_STR_ALLOCATE()                                   \
-        if(this->size < sizeof(this->_inlined)){            \
+        if(this->size < (int)sizeof(this->_inlined)){       \
             this->data = this->_inlined;                    \
         }else{                                              \
             this->data = (char*)pool64_alloc(this->size+1); \
@@ -345,8 +345,8 @@ int utf8len(unsigned char c, bool suppress){
         return _byte_index_to_unicode(size);
     }
 
-    std::vector<std::string_view> Str::split(const Str& sep) const{
-        std::vector<std::string_view> result;
+    pod_vector<std::string_view> Str::split(const Str& sep) const{
+        pod_vector<std::string_view> result;
         std::string_view tmp;
         int start = 0;
         while(true){
@@ -361,8 +361,8 @@ int utf8len(unsigned char c, bool suppress){
         return result;
     }
 
-    std::vector<std::string_view> Str::split(char sep) const{
-        std::vector<std::string_view> result;
+    pod_vector<std::string_view> Str::split(char sep) const{
+        pod_vector<std::string_view> result;
         int i = 0;
         for(int j = 0; j < size; j++){
             if(data[j] == sep){

src/vm.cpp

@@ -79,9 +79,6 @@ namespace pkpy{
         _main = nullptr;
         _last_exception = nullptr;
         _import_handler = [](const char* name_p, int name_size, int* out_size) -> unsigned char*{
-            PK_UNUSED(name_p);
-            PK_UNUSED(name_size);
-            PK_UNUSED(out_size);
             return nullptr;
         };
         init_builtin_types();
@@ -254,7 +251,7 @@ namespace pkpy{
         return false;
     }
 
-    int VM::normalized_index(int index, int size){
+    i64 VM::normalized_index(i64 index, int size){
         if(index < 0) index += size;
         if(index < 0 || index >= size){
             IndexError(std::to_string(index) + " not in [0, " + std::to_string(size) + ")");
@@ -281,7 +278,7 @@ namespace pkpy{
 
     PyObject* VM::py_import(Str path, bool throw_err){
         if(path.empty()) vm->ValueError("empty module name");
-        static auto f_join = [](const std::vector<std::string_view>& cpnts){
+        static auto f_join = [](const pod_vector<std::string_view>& cpnts){
             SStream ss;
             for(int i=0; i<cpnts.size(); i++){
                 if(i != 0) ss << ".";
@@ -297,7 +294,7 @@ namespace pkpy{
             Str curr_path = _import_context.pending.back();
             bool curr_is_init = _import_context.pending_is_init.back();
             // convert relative path to absolute path
-            std::vector<std::string_view> cpnts = curr_path.split('.');
+            pod_vector<std::string_view> cpnts = curr_path.split('.');
             int prefix = 0;     // how many dots in the prefix
             for(int i=0; i<path.length(); i++){
                 if(path[i] == '.') prefix++;
@@ -317,7 +314,7 @@ namespace pkpy{
         PyObject* ext_mod = _modules.try_get(name);
         if(ext_mod != nullptr) return ext_mod;
 
-        std::vector<std::string_view> path_cpnts = path.split('.');
+        pod_vector<std::string_view> path_cpnts = path.split('.');
         // check circular import
         if(_import_context.pending.size() > 128){
             ImportError("maximum recursion depth exceeded while importing");
@@ -603,7 +600,7 @@ Str VM::disassemble(CodeObject_ co){
         return s + std::string(n - s.length(), ' ');
     };
 
-    std::vector<int> jumpTargets;
+    pod_vector<int> jumpTargets;
     for(auto byte : co->codes){
         if(byte.op == OP_JUMP_ABSOLUTE || byte.op == OP_POP_JUMP_IF_FALSE || byte.op == OP_SHORTCUT_IF_FALSE_OR_POP || byte.op == OP_FOR_ITER){
             jumpTargets.push_back(byte.arg);
@@ -618,11 +615,11 @@ Str VM::disassemble(CodeObject_ co){
     int prev_line = -1;
     for(int i=0; i<co->codes.size(); i++){
         const Bytecode& byte = co->codes[i];
-        Str line = std::to_string(co->lines[i]);
-        if(co->lines[i] == prev_line) line = "";
+        Str line = std::to_string(co->lines[i].lineno);
+        if(co->lines[i].lineno == prev_line) line = "";
         else{
             if(prev_line != -1) ss << "\n";
-            prev_line = co->lines[i];
+            prev_line = co->lines[i].lineno;
         }
 
         std::string pointer;
@@ -632,7 +629,9 @@ Str VM::disassemble(CodeObject_ co){
             pointer = "   ";
         }
         ss << pad(line, 8) << pointer << pad(std::to_string(i), 3);
-        ss << " " << pad(OP_NAMES[byte.op], 25) << " ";
+        std::string bc_name(OP_NAMES[byte.op]);
+        if(co->lines[i].is_virtual) bc_name += '*';
+        ss << " " << pad(bc_name, 25) << " ";
         // ss << pad(byte.arg == -1 ? "" : std::to_string(byte.arg), 5);
         std::string argStr = _opcode_argstr(this, byte, co.get());
         ss << argStr;
@@ -1258,7 +1257,7 @@ void VM::_raise(bool re_raise){
 
     int actual_ip = frame->_ip;
     if(e._ip_on_error >= 0 && e._code_on_error == (void*)frame->co) actual_ip = e._ip_on_error;
-    int current_line = frame->co->lines[actual_ip];         // current line
+    int current_line = frame->co->lines[actual_ip].lineno;         // current line
     auto current_f_name = frame->co->name.sv();             // current function name
     if(frame->_callable == nullptr) current_f_name = "";    // not in a function
     e.st_push(frame->co->src, current_line, nullptr, current_f_name);

tests/04_str.py

@@ -223,3 +223,22 @@ test(0, 100000)
 test(-100, 100)
 test(-100000, 100000)
 test(-2**30, 2**30)
+
+
+a = '123'
+assert a.index('2') == 1
+assert a.index('1') == 0
+assert a.index('3') == 2
+
+assert a.index('2', 1) == 1
+assert a.index('1', 0) == 0
+
+try:
+    a.index('1', 1)
+    exit(1)
+except ValueError:
+    pass
+
+assert a.find('1') == 0
+assert a.find('1', 1) == -1
+

tests/05_list.py

@@ -116,3 +116,16 @@ assert A()[::, :2] == (slice(None, None, None), slice(None, 2, None))
 assert A()['b':'c':1, :] == (slice('b', 'c', 1), slice(None, None, None))
 assert A()[1:2, :A()[3:4, ::-1]] == (slice(1, 2, None), slice(None, (slice(3, 4, None), slice(None, None, -1)), None))
 
+a = [1, 2, 3]
+assert a.index(2) == 1
+assert a.index(1) == 0
+assert a.index(3) == 2
+
+assert a.index(2, 1) == 1
+assert a.index(1, 0) == 0
+
+try:
+    a.index(1, 1)
+    exit(1)
+except ValueError:
+    pass

tests/83_array2d.py

@@ -107,7 +107,19 @@ assert A().get(0, 0, default=2) == 0
 
 # test alive_neighbors
 a = array2d(3, 3, default=0)
-a.count_neighbors(0) == a
+a[1, 1] = 1
+"""     moore    von_neumann
+0 0 0   1 1 1    0 1 0
+0 1 0   1 0 1    1 0 1
+0 0 0   1 1 1    0 1 0
+"""
+moore_result = array2d(3, 3, default=1)
+moore_result[1, 1] = 0
+
+von_neumann_result = array2d(3, 3, default=0)
+von_neumann_result[0, 1] = von_neumann_result[1, 0] = von_neumann_result[1, 2] = von_neumann_result[2, 1] = 1
+a.count_neighbors(0, 'moore') == moore_result
+a.count_neighbors(0, 'von_neumann') == von_neumann_result
 
 # test slice get
 a = array2d(5, 5, default=0)

tests/84_line_profiler.py

@@ -1,15 +1,25 @@
 from line_profiler import LineProfiler
 
-def my_func():
+def f2(x):
     a = 0
-    for i in range(1000000):
-        a += i
+    for i in range(x):
+        if i % 5 == 0:
+            a += i
     return a
 
+def f1(x):
+    res = f2(x)
+    return res
+
 lp = LineProfiler()
 
-lp.add_function(my_func)
+lp.add_function(f2)
 
-lp.runcall(my_func)
+# lp.runcall(f2, 1000000)
+# lp.print_stats()
+###############################
 
+lp.add_function(f1)
+lp.runcall(f1, 1000000)
 lp.print_stats()
+