fix https://github.com/blueloveTH/pocketpy/issues/99

src/dict.h

@@ -8,23 +8,40 @@
 namespace pkpy{
 
 struct Dict{
-    using Item = std::pair<PyObject*, PyObject*>;
+    struct Item{
+        PyObject* first;
+        PyObject* second;
+    };
+
+    struct ItemNode{
+        int prev;
+        int next;
+    };
+
     static constexpr int __Capacity = 8;
     static constexpr float __LoadFactor = 0.67f;
+    // by ensuring this, we can use pool64 to alloc ItemNode and pool128 to alloc Item
+    static_assert(sizeof(Item) == 2*sizeof(ItemNode));
     static_assert(sizeof(Item) * __Capacity <= 128);
+    static_assert(sizeof(ItemNode) * __Capacity <= 64);
 
     VM* vm;
     int _capacity;
     int _mask;
     int _size;
     int _critical_size;
+    int _head_idx;          // for order preserving
+    int _tail_idx;          // for order preserving
     Item* _items;
-    
+    ItemNode* _nodes;       // for order preserving
+
     Dict(VM* vm): vm(vm), _capacity(__Capacity),
             _mask(__Capacity-1),
-            _size(0), _critical_size(__Capacity*__LoadFactor+0.5f){
+            _size(0), _critical_size(__Capacity*__LoadFactor+0.5f), _head_idx(-1), _tail_idx(-1){
         _items = (Item*)pool128.alloc(_capacity * sizeof(Item));
         memset(_items, 0, _capacity * sizeof(Item));
+        _nodes = (ItemNode*)pool64.alloc(_capacity * sizeof(ItemNode));
+        memset(_nodes, -1, _capacity * sizeof(ItemNode));
     }
 
     int size() const { return _size; }
@@ -35,8 +52,12 @@ struct Dict{
         _mask = other._mask;
         _size = other._size;
         _critical_size = other._critical_size;
+        _head_idx = other._head_idx;
+        _tail_idx = other._tail_idx;
         _items = other._items;
+        _nodes = other._nodes;
         other._items = nullptr;
+        other._nodes = nullptr;
     }
 
     Dict(const Dict& other){
@@ -45,8 +66,12 @@ struct Dict{
         _mask = other._mask;
         _size = other._size;
         _critical_size = other._critical_size;
+        _head_idx = other._head_idx;
+        _tail_idx = other._tail_idx;
         _items = (Item*)pool128.alloc(_capacity * sizeof(Item));
         memcpy(_items, other._items, _capacity * sizeof(Item));
+        _nodes = (ItemNode*)pool64.alloc(_capacity * sizeof(ItemNode));
+        memcpy(_nodes, other._nodes, _capacity * sizeof(ItemNode));
     }
 
     Dict& operator=(const Dict&) = delete;
@@ -55,15 +80,23 @@ struct Dict{
     void _probe(PyObject* key, bool& ok, int& i) const;
 
     void set(PyObject* key, PyObject* val){
+        // do possible rehash
+        if(_size+1 > _critical_size) _rehash();
         bool ok; int i;
         _probe(key, ok, i);
         if(!ok) {
             _size++;
-            if(_size > _critical_size){
-                _rehash();
-                _probe(key, ok, i);
-            }
             _items[i].first = key;
+
+            // append to tail
+            if(_size == 0+1){
+                _head_idx = i;
+                _tail_idx = i;
+            }else{
+                _nodes[i].prev = _tail_idx;
+                _nodes[_tail_idx].next = i;
+                _tail_idx = i;
+            }
         }
         _items[i].second = val;
     }
@@ -73,15 +106,19 @@ struct Dict{
         int old_capacity = _capacity;
         _capacity *= 2;
         _mask = _capacity - 1;
-        _critical_size = _capacity * __LoadFactor + 0.5f;
+        _size = 0;
+        _critical_size = _capacity*__LoadFactor+0.5f;
+        _head_idx = -1;
+        _tail_idx = -1;
+        pool64.dealloc(_nodes);
         _items = (Item*)pool128.alloc(_capacity * sizeof(Item));
         memset(_items, 0, _capacity * sizeof(Item));
+        _nodes = (ItemNode*)pool64.alloc(_capacity * sizeof(ItemNode));
+        memset(_nodes, -1, _capacity * sizeof(ItemNode));
+
         for(int i=0; i<old_capacity; i++){
             if(old_items[i].first == nullptr) continue;
-            bool ok; int j;
-            _probe(old_items[i].first, ok, j);
-            if(ok) FATAL_ERROR();
-            _items[j] = old_items[i];
+            set(old_items[i].first, old_items[i].second);
         }
         pool128.dealloc(old_items);
     }
@@ -106,45 +143,82 @@ struct Dict{
         _items[i].first = nullptr;
         _items[i].second = nullptr;
         _size--;
-    }
 
-    void update(const Dict& other){
-        for(int i=0; i<other._capacity; i++){
-            if(other._items[i].first == nullptr) continue;
-            set(other._items[i].first, other._items[i].second);
+        if(_size == 0){
+            _head_idx = -1;
+            _tail_idx = -1;
+        }else{
+            if(_head_idx == i){
+                _head_idx = _nodes[i].next;
+                _nodes[_head_idx].prev = -1;
+            }else if(_tail_idx == i){
+                _tail_idx = _nodes[i].prev;
+                _nodes[_tail_idx].next = -1;
+            }else{
+                _nodes[_nodes[i].prev].next = _nodes[i].next;
+                _nodes[_nodes[i].next].prev = _nodes[i].prev;
+            }
         }
+        _nodes[i].prev = -1;
+        _nodes[i].next = -1;
     }
 
-    std::vector<Item> items() const {
-        std::vector<Item> v;
-        for(int i=0; i<_capacity; i++){
-            if(_items[i].first == nullptr) continue;
-            v.push_back(_items[i]);
-        }
-        return v;
+    void update(const Dict& other){
+        other.apply([&](PyObject* k, PyObject* v){ set(k, v); });
     }
 
     template<typename __Func>
     void apply(__Func f) const {
-        for(int i=0; i<_capacity; i++){
-            if(_items[i].first == nullptr) continue;
+        int i = _head_idx;
+        while(i != -1){
             f(_items[i].first, _items[i].second);
+            i = _nodes[i].next;
+        }
+    }
+
+    Tuple keys() const{
+        Tuple t(_size);
+        int i = _head_idx;
+        int j = 0;
+        while(i != -1){
+            t[j++] = _items[i].first;
+            i = _nodes[i].next;
         }
+        PK_ASSERT(j == _size);
+        return t;
+    }
+
+    Tuple values() const{
+        Tuple t(_size);
+        int i = _head_idx;
+        int j = 0;
+        while(i != -1){
+            t[j++] = _items[i].second;
+            i = _nodes[i].next;
+        }
+        PK_ASSERT(j == _size);
+        return t;
     }
 
     void clear(){
-        memset(_items, 0, _capacity * sizeof(Item));
         _size = 0;
+        _head_idx = -1;
+        _tail_idx = -1;
+        memset(_items, 0, _capacity * sizeof(Item));
+        memset(_nodes, -1, _capacity * sizeof(ItemNode));
     }
 
-    ~Dict(){ if(_items!=nullptr) pool128.dealloc(_items); }
+    ~Dict(){
+        if(_items==nullptr) return;
+        pool128.dealloc(_items);
+        pool64.dealloc(_nodes);
+    }
 
     void _gc_mark() const{
-        for(int i=0; i<_capacity; i++){
-            if(_items[i].first == nullptr) continue;
-            PK_OBJ_MARK(_items[i].first);
-            PK_OBJ_MARK(_items[i].second);
-        }
+        apply([](PyObject* k, PyObject* v){
+            PK_OBJ_MARK(k);
+            PK_OBJ_MARK(v);
+        });
     }
 };
 

src/easing.h

@@ -12,31 +12,31 @@ namespace pkpy{
 
 static const double PI = 3.1415926545;
 
-inline static double easeLinear( double x ) {
+static double easeLinear( double x ) {
     return x;
 }
 
-inline static double easeInSine( double x ) {
+static double easeInSine( double x ) {
     return 1.0 - std::cos( x * PI / 2 );
 }
 
-inline static double easeOutSine( double x ) {
+static double easeOutSine( double x ) {
 	return std::sin( x * PI / 2 );
 }
 
-inline static double easeInOutSine( double x ) {
+static double easeInOutSine( double x ) {
 	return -( std::cos( PI * x ) - 1 ) / 2;
 }
 
-inline static double easeInQuad( double x ) {
+static double easeInQuad( double x ) {
     return x * x;
 }
 
-inline static double easeOutQuad( double x ) {
+static double easeOutQuad( double x ) {
     return 1 - std::pow( 1 - x, 2 );
 }
 
-inline static double easeInOutQuad( double x ) {
+static double easeInOutQuad( double x ) {
     if( x < 0.5 ) {
         return 2 * x * x;
     } else {
@@ -44,15 +44,15 @@ inline static double easeInOutQuad( double x ) {
     }
 }
 
-inline static double easeInCubic( double x ) {
+static double easeInCubic( double x ) {
     return x * x * x;
 }
 
-inline static double easeOutCubic( double x ) {
+static double easeOutCubic( double x ) {
     return 1 - std::pow( 1 - x, 3 );
 }
 
-inline static double easeInOutCubic( double x ) {
+static double easeInOutCubic( double x ) {
     if( x < 0.5 ) {
         return 4 * x * x * x;
     } else {
@@ -60,15 +60,15 @@ inline static double easeInOutCubic( double x ) {
     }
 }
 
-inline static double easeInQuart( double x ) {
+static double easeInQuart( double x ) {
     return std::pow( x, 4 );
 }
 
-inline static double easeOutQuart( double x ) {
+static double easeOutQuart( double x ) {
     return 1 - std::pow( 1 - x, 4 );
 }
 
-inline static double easeInOutQuart( double x ) {
+static double easeInOutQuart( double x ) {
     if( x < 0.5 ) {
         return 8 * std::pow( x, 4 );
     } else {
@@ -76,15 +76,15 @@ inline static double easeInOutQuart( double x ) {
     }
 }
 
-inline static double easeInQuint( double x ) {
+static double easeInQuint( double x ) {
     return std::pow( x, 5 );
 }
 
-inline static double easeOutQuint( double x ) {
+static double easeOutQuint( double x ) {
     return 1 - std::pow( 1 - x, 5 );
 }
 
-inline static double easeInOutQuint( double x ) {
+static double easeInOutQuint( double x ) {
     if( x < 0.5 ) {
         return 16 * std::pow( x, 5 );
     } else {
@@ -92,11 +92,11 @@ inline static double easeInOutQuint( double x ) {
     }
 }
 
-inline static double easeInExpo( double x ) {
+static double easeInExpo( double x ) {
     return x == 0 ? 0 : std::pow( 2, 10 * x - 10 );
 }
 
-inline static double easeOutExpo( double x ) {
+static double easeOutExpo( double x ) {
     return x == 1 ? 1 : 1 - std::pow( 2, -10 * x );
 }
 
@@ -112,15 +112,15 @@ inline double easeInOutExpo( double x ) {
     }
 }
 
-inline static double easeInCirc( double x ) {
+static double easeInCirc( double x ) {
     return 1 - std::sqrt( 1 - std::pow( x, 2 ) );
 }
 
-inline static double easeOutCirc( double x ) {
+static double easeOutCirc( double x ) {
     return std::sqrt( 1 - std::pow( x - 1, 2 ) );
 }
 
-inline static double easeInOutCirc( double x ) {
+static double easeInOutCirc( double x ) {
     if( x < 0.5 ) {
         return (1 - std::sqrt( 1 - std::pow( 2 * x, 2 ) )) / 2;
     } else {
@@ -128,19 +128,19 @@ inline static double easeInOutCirc( double x ) {
     }
 }
 
-inline static double easeInBack( double x ) {
+static double easeInBack( double x ) {
     const double c1 = 1.70158;
     const double c3 = c1 + 1;
     return c3 * x * x * x - c1 * x * x;
 }
 
-inline static double easeOutBack( double x ) {
+static double easeOutBack( double x ) {
     const double c1 = 1.70158;
     const double c3 = c1 + 1;
     return 1 + c3 * std::pow( x - 1, 3 ) + c1 * std::pow( x - 1, 2 );
 }
 
-inline static double easeInOutBack( double x ) {
+static double easeInOutBack( double x ) {
     const double c1 = 1.70158;
     const double c2 = c1 * 1.525;
     if( x < 0.5 ) {
@@ -150,7 +150,7 @@ inline static double easeInOutBack( double x ) {
     }
 }
 
-inline static double easeInElastic( double x ) {
+static double easeInElastic( double x ) {
     const double c4 = (2 * PI) / 3;
     if( x == 0 ) {
         return 0;
@@ -161,7 +161,7 @@ inline static double easeInElastic( double x ) {
     }
 }
 
-inline static double easeOutElastic( double x ) {
+static double easeOutElastic( double x ) {
     const double c4 = (2 * PI) / 3;
     if( x == 0 ) {
         return 0;
@@ -185,7 +185,7 @@ inline double easeInOutElastic( double x ) {
     }
 }
 
-inline static double easeOutBounce( double x ) {
+static double easeOutBounce( double x ) {
     const double n1 = 7.5625;
     const double d1 = 2.75;
     if( x < 1 / d1 ) {
@@ -202,11 +202,11 @@ inline static double easeOutBounce( double x ) {
     }
 }
 
-inline double easeInBounce( double x ) {
+static double easeInBounce( double x ) {
     return 1 - easeOutBounce(1 - x);
 }
 
-inline static double easeInOutBounce( double x ) {
+static double easeInOutBounce( double x ) {
     return x < 0.5
     ? (1 - easeOutBounce(1 - 2 * x)) / 2
     : (1 + easeOutBounce(2 * x - 1)) / 2;

src/memory.h

@@ -249,11 +249,6 @@ struct MemoryPool{
 inline MemoryPool<64> pool64;
 inline MemoryPool<128> pool128;
 
-// get the total memory usage of pkpy (across all VMs)
-inline size_t memory_usage(){
-    return pool64.allocated_size() + pool128.allocated_size();
-}
-
 template <typename T>
 struct shared_ptr {
     int* counter;

src/pocketpy.h

@@ -1058,11 +1058,8 @@ inline void init_builtins(VM* _vm) {
     });
 
     _vm->bind__iter__(_vm->tp_dict, [](VM* vm, PyObject* obj) {
-        Dict& self = _CAST(Dict&, obj);
-        auto items = self.items();
-        Tuple t(items.size());
-        for(int i=0; i<items.size(); i++) t[i] = items[i].first;
-        return vm->py_iter(VAR(std::move(t)));
+        const Dict& self = _CAST(Dict&, obj);
+        return vm->py_iter(VAR(self.keys()));
     });
 
     _vm->bind_method<-1>("dict", "get", [](VM* vm, ArgsView args) {
@@ -1081,26 +1078,22 @@ inline void init_builtins(VM* _vm) {
     });
 
     _vm->bind_method<0>("dict", "keys", [](VM* vm, ArgsView args) {
-        Dict& self = _CAST(Dict&, args[0]);
-        List keys;
-        for(auto& item : self.items()) keys.push_back(item.first);
-        return VAR(std::move(keys));
+        const Dict& self = _CAST(Dict&, args[0]);
+        return VAR(self.keys());
     });
 
     _vm->bind_method<0>("dict", "values", [](VM* vm, ArgsView args) {
-        Dict& self = _CAST(Dict&, args[0]);
-        List values;
-        for(auto& item : self.items()) values.push_back(item.second);
-        return VAR(std::move(values));
+        const Dict& self = _CAST(Dict&, args[0]);
+        return VAR(self.values());
     });
 
     _vm->bind_method<0>("dict", "items", [](VM* vm, ArgsView args) {
-        Dict& self = _CAST(Dict&, args[0]);
-        List items;
-        for(auto& item : self.items()){
-            PyObject* t = VAR(Tuple({item.first, item.second}));
-            items.push_back(std::move(t));
-        }
+        const Dict& self = _CAST(Dict&, args[0]);
+        Tuple items(self.size());
+        int j = 0;
+        self.apply([&](PyObject* k, PyObject* v){
+            items[j++] = VAR(Tuple({k, v}));
+        });
         return VAR(std::move(items));
     });
 
@@ -1127,13 +1120,15 @@ inline void init_builtins(VM* _vm) {
         std::stringstream ss;
         ss << "{";
         bool first = true;
-        for(auto& item : self.items()){
+
+        self.apply([&](PyObject* k, PyObject* v){
             if(!first) ss << ", ";
             first = false;
-            Str key = CAST(Str&, vm->py_repr(item.first));
-            Str value = CAST(Str&, vm->py_repr(item.second));
+            Str key = CAST(Str&, vm->py_repr(k));
+            Str value = CAST(Str&, vm->py_repr(v));
             ss << key << ": " << value;
-        }
+        });
+
         ss << "}";
         return VAR(ss.str());
     });
@@ -1143,13 +1138,15 @@ inline void init_builtins(VM* _vm) {
         std::stringstream ss;
         ss << "{";
         bool first = true;
-        for(auto& item : self.items()){
+
+        self.apply([&](PyObject* k, PyObject* v){
             if(!first) ss << ", ";
             first = false;
-            Str key = CAST(Str&, item.first).escape(false);
-            Str value = CAST(Str&, vm->py_json(item.second));
+            Str key = CAST(Str&, k).escape(false);
+            Str value = CAST(Str&, vm->py_json(v));
             ss << key << ": " << value;
-        }
+        });
+
         ss << "}";
         return VAR(ss.str());
     });
@@ -1159,7 +1156,9 @@ inline void init_builtins(VM* _vm) {
         if(!is_non_tagged_type(b, vm->tp_dict)) return vm->NotImplemented;
         Dict& other = _CAST(Dict&, b);
         if(self.size() != other.size()) return vm->False;
-        for(auto& item : self.items()){
+        for(int i=0; i<self._capacity; i++){
+            auto item = self._items[i];
+            if(item.first == nullptr) continue;
             PyObject* value = other.try_get(item.first);
             if(value == nullptr) return vm->False;
             if(!vm->py_equals(item.second, value)) return vm->False;

src/vm.h

@@ -1604,7 +1604,8 @@ inline void Dict::_probe(PyObject *key, bool &ok, int &i) const{
     i = vm->py_hash(key) & _mask;
     while(_items[i].first != nullptr) {
         if(vm->py_equals(_items[i].first, key)) { ok = true; break; }
-        i = (i + 1) & _mask;
+        // https://github.com/python/cpython/blob/3.8/Objects/dictobject.c#L166
+        i = ((5*i) + 1) & _mask;
     }
 }
 

tests/07_dict.py

@@ -26,10 +26,9 @@ assert len(tinydict) == 3
 assert tinydict == updated_dict
 
 dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}
-keys = dishes.keys()
-values = dishes.values()
-assert sorted(keys) == sorted(['eggs', 'sausage', 'bacon', 'spam'])
-assert sorted(values) == sorted([2, 1, 1, 500])
+# dict is now ordered
+assert dishes.keys() == ('eggs', 'sausage', 'bacon', 'spam')
+assert dishes.values() == (2, 1, 1, 500)
 
 d={1:"a",2:"b",3:"c"}
 result=[]
@@ -37,7 +36,14 @@ for k,v in d.items():
     result.append(k)
     result.append(v)
 assert len(result) == 6
-assert set(result) == set([1, 'a', 2, 'b', 3, 'c'])
+
+del d[2]
+assert len(d) == 2
+assert d.keys() == (1, 3)
+assert d.values() == ('a', 'c')
+del d[1]
+del d[3]
+assert len(d) == 0
 
 # test __eq__
 d1 = {1:2, 3:4}
@@ -66,4 +72,12 @@ assert b == {1: 2, 3: 4}
 a = {1:2, 3:4, 7:8}
 b = {**a, 1:5, 3:6}
 c = {**a, **b}
-assert c == {1: 5, 3: 6, 7: 8}
+assert c == {1: 5, 3: 6, 7: 8}
+
+a = {}
+for i in range(1000):
+    a[i] = i
+assert len(a) == 1000
+for i in range(1000):
+    del a[i]
+assert len(a) == 0