some fix

include/pocketpy/compiler/lexer.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <stdbool.h>
 #include "pocketpy/common/str.h"
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {

include/pocketpy/compiler/lexer.hpp

@@ -11,19 +11,21 @@ typedef uint8_t TokenIndex;
 
 // clang-format off
 constexpr const char* kTokens[] = {
-    "is not", "not in", "yield from",
     "@eof", "@eol", "@sof",
     "@id", "@num", "@str", "@fstr", "@long", "@bytes", "@imag",
     "@indent", "@dedent",
+    // These 3 are compound keywords which are generated on the fly
+    "is not", "not in", "yield from",
     /*****************************************/
     "+", "+=", "-", "-=",   // (INPLACE_OP - 1) can get '=' removed
     "*", "*=", "/", "/=", "//", "//=", "%", "%=",
     "&", "&=", "|", "|=", "^", "^=", 
     "<<", "<<=", ">>", ">>=",
     /*****************************************/
-    ".", ",", ":", ";", "#", "(", ")", "[", "]", "{", "}",
-    "**", "=", ">", "<", "..", "...", "->", "@", "==", "!=", ">=", "<=",
-    "++", "--", "~",
+    "(", ")", "[", "]", "{", "}",
+    ".", "..", "...", ",", ":", ";",
+    "**", "->", "#", "@",
+    ">", "<", "=", "==", "!=", ">=", "<=", "~",
     /** KW_BEGIN **/
     // NOTE: These keywords should be sorted in ascending order!!
     "False", "None", "True", "and", "as", "assert", "break", "class", "continue",

include/pocketpy/objects/error.h

@@ -34,6 +34,15 @@ void pkpy_Exception__dtor(pkpy_Exception* self);
 void pkpy_Exception__stpush(pkpy_Exception* self, pkpy_SourceData_ src, int lineno, const char* cursor, const char* name);
 pkpy_Str pkpy_Exception__summary(pkpy_Exception* self);
 
+struct Error{
+    const char* type;
+    pkpy_SourceData_ src;
+    int lineno;
+    const char* cursor;
+    char msg[100];
+    int64_t userdata;
+};
+
 #ifdef __cplusplus
 }
 #endif

include/pocketpy/objects/error.hpp

@@ -59,13 +59,6 @@ struct TopLevelException : std::exception {
     }
 };
 
-struct Error{
-    const char* type;
-    pkpy_SourceData_ src;
-    int lineno;
-    const char* cursor;
-    char msg[100];
-    i64 userdata;
-};
+
 
 }  // namespace pkpy

src/compiler/lexer.c

@@ -20,7 +20,7 @@ c11_string pkpy_TokenDeserializer__read_string(pkpy_TokenDeserializer* self, cha
     const char* start = self->curr;
     while(*self->curr != c)
         self->curr++;
-    c11_string retval = {start, self->curr - start};
+    c11_string retval = {start, (int)(self->curr-start)};
     self->curr++;  // skip the delimiter
     return retval;
 }

src/compiler/lexer.cpp

@@ -390,13 +390,7 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
                 return NULL;
             }
             case '=': add_token_2('=', TK("="), TK("==")); return NULL;
-            case '+':
-                if(matchchar('+')) {
-                    add_token(TK("++"));
-                } else {
-                    add_token_2('=', TK("+"), TK("+="));
-                }
-                return NULL;
+            case '+': add_token_2('=', TK("+"), TK("+=")); return NULL;
             case '>': {
                 if(matchchar('='))
                     add_token(TK(">="));
@@ -416,16 +410,12 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
                 return NULL;
             }
             case '-': {
-                if(matchchar('-')) {
-                    add_token(TK("--"));
-                } else {
-                    if(matchchar('='))
-                        add_token(TK("-="));
-                    else if(matchchar('>'))
-                        add_token(TK("->"));
-                    else
-                        add_token(TK("-"));
-                }
+                if(matchchar('='))
+                    add_token(TK("-="));
+                else if(matchchar('>'))
+                    add_token(TK("->"));
+                else
+                    add_token(TK("-"));
                 return NULL;
             }
             case '!':