Support new opcodes

ASTNode.h

@@ -18,7 +18,7 @@ class ASTNode {
         NODE_COMPREHENSION, NODE_LOADBUILDCLASS, NODE_AWAITABLE,
         NODE_FORMATTEDVALUE, NODE_JOINEDSTR, NODE_CONST_MAP,
         NODE_ANNOTATED_VAR, NODE_CHAINSTORE, NODE_TERNARY,
-        NODE_KW_NAMES_MAP,
+        NODE_KW_NAMES_MAP, NODE_CALL_INTRINSIC_1, NODE_CALL_INTRINSIC_2,
 
         // Empty node types
         NODE_LOCALS,
@@ -757,4 +757,26 @@ class ASTTernary : public ASTNode
     PycRef<ASTNode> m_else_expr;
 };
 
+class ASTCallIntrinsic1: public ASTNode
+{
+public:
+    enum Function {
+        INTRINSIC_1_INVALID, INTRINSIC_PRINT, INTRINSIC_IMPORT_STAR,
+        INTRINSIC_STOPITERATION_ERROR, INTRINSIC_ASYNC_GEN_WRAP,
+        INTRINSIC_UNARY_POSITIVE, INTRINSIC_LIST_TO_TUPLE, INTRINSIC_TYPEVAR,
+        INTRINSIC_PARAMSPEC, INTRINSIC_TYPEVARTUPLE,
+        INTRINSIC_SUBSCRIPT_GENERIC, INTRINSIC_TYPEALIAS,
+    };
+};
+
+class ASTCallIntrinsic2: public ASTNode
+{
+public:
+    enum Function {
+        INTRINSIC_2_INVALID, INTRINSIC_PREP_RERAISE_STAR,
+        INTRINSIC_TYPEVAR_WITH_BOUND, INTRINSIC_TYPEVAR_WITH_CONSTRAINTS,
+        INTRINSIC_SET_FUNCTION_TYPE_PARAMS, INTRINSIC_SET_TYPEPARAM_DEFAULT,
+    };
+};
+
 #endif

ASTree.cpp

@@ -6,6 +6,7 @@
 #include "FastStack.h"
 #include "pyc_numeric.h"
 #include "bytecode.h"
+#include <iostream>
 
 // This must be a triple quote (''' or """), to handle interpolated string literals containing the opposite quote style.
 // E.g. f'''{"interpolated "123' literal"}'''    -> valid.
@@ -1448,29 +1449,46 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             break;
         case Pyc::LIST_EXTEND_A:
             {
+                if (operand != 1) {
+                    fprintf(stderr, "LIST_EXTEND operand list is not at the top of the stack\n");
+                    break;
+                }
+
                 PycRef<ASTNode> rhs = stack.top();
                 stack.pop();
                 PycRef<ASTList> lhs = stack.top().cast<ASTList>();
                 stack.pop();
 
-                if (rhs.type() != ASTNode::NODE_OBJECT) {
-                    fprintf(stderr, "Unsupported argument found for LIST_EXTEND\n");
-                    break;
-                }
+                if (rhs.type() == ASTNode::NODE_OBJECT) {
 
-                // I've only ever seen this be a SMALL_TUPLE, but let's be careful...
-                PycRef<PycObject> obj = rhs.cast<ASTObject>()->object();
-                if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) {
-                    fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n");
-                    break;
-                }
+                    // I've only ever seen this be a SMALL_TUPLE, but let's be careful...
+                    PycRef<PycObject> obj = rhs.cast<ASTObject>()->object();
+                    if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) {
+                        fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n");
+                        break;
+                    }
 
-                ASTList::value_t result = lhs->values();
-                for (const auto& it : obj.cast<PycTuple>()->values()) {
-                    result.push_back(new ASTObject(it));
+                    ASTList::value_t result = lhs->values();
+                    for (const auto& it : obj.cast<PycTuple>()->values()) {
+                        result.push_back(new ASTObject(it));
+                    }
+
+                    stack.push(new ASTList(result));
                 }
+                else if (rhs.type() == ASTNode::NODE_NAME) {
+                    ASTList::value_t result = lhs->values();
+
+                    // rhs is a variable, so to extend the list
+                    // we need to unpack rhs
+                    PycRef<ASTNode> unpacked_ref = rhs;
+                    unpacked_ref.setUnpacked(true);
 
-                stack.push(new ASTList(result));
+                    result.push_back(unpacked_ref);
+                    stack.push(new ASTList(result));
+                }
+                else {
+                    fprintf(stderr, "Unsupported argument %i found for LIST_EXTEND\n", rhs.type());
+                }
             }
             break;
         case Pyc::LOAD_ATTR_A:
@@ -1520,6 +1538,7 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             stack.push(new ASTName(code->getCellVar(mod, operand)));
             break;
         case Pyc::LOAD_FAST_A:
+        case Pyc::LOAD_FAST_CHECK_A:
             if (mod->verCompare(1, 3) < 0)
                 stack.push(new ASTName(code->getName(operand)));
             else
@@ -2584,6 +2603,77 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
                 stack.push(value);
             }
             break;
+        case Pyc::CALL_INTRINSIC_1_A:
+            {
+                PycRef<ASTNode> arg = stack.top();
+                stack.pop();
+
+                if (operand != ASTCallIntrinsic1::INTRINSIC_LIST_TO_TUPLE) {
+                    fprintf(stderr, "Unimplemented function %i", operand);
+                    break;
+                }
+
+                if (arg.type() != ASTNode::NODE_LIST) {
+                    fprintf(stderr, "Unexpected argument type %i\n", arg.type());
+                    break;
+                }
+
+                PycRef<ASTList> list = arg.cast<ASTList>();
+                ASTTuple::value_t values;
+                for (PycRef<ASTNode> val : list->values()) {
+                    values.push_back(val);
+                }
+                stack.push(new ASTTuple(values));
+            }
+            break;
+        case Pyc::CALL_FUNCTION_EX_A:
+            {
+                int has_kwmap = operand & 1;
+                ASTCall::kwparam_t kwparamList;
+                ASTCall::pparam_t pparamList;
+
+                // callable, iterable object & kwmap object (if present)
+
+                if (has_kwmap) {
+                    PycRef<ASTNode> object_or_map = stack.top();
+                    if (object_or_map.type() == ASTNode::NODE_KW_NAMES_MAP) {
+                        stack.pop();
+                        PycRef<ASTKwNamesMap> kwparams_map = object_or_map.cast<ASTKwNamesMap>();
+                        for (ASTKwNamesMap::map_t::const_iterator it = kwparams_map->values().begin(); it != kwparams_map->values().end(); it++) {
+                            kwparamList.push_front(std::make_pair(it->first, it->second));
+                        }
+                    }
+                    else {
+                        fprintf(stderr, "Unexpected object type %i\n", object_or_map.type());
+                    }
+                }
+
+                PycRef<ASTNode> iterable = stack.top();
+                stack.pop();
+
+                // Not sure how to combine these two conditions
+                if (iterable.type() == ASTNode::NODE_LIST) {
+                    PycRef<ASTList> list = iterable.cast<ASTList>();
+                    for (PycRef<ASTNode> n: list->values()) {
+                        pparamList.push_back(n);
+                    }
+                }
+                else if (iterable.type() == ASTNode::NODE_TUPLE) {
+                    PycRef<ASTTuple> tuple = iterable.cast<ASTTuple>();
+                    for (PycRef<ASTNode> n: tuple->values()) {
+                        pparamList.push_back(n);
+                    }
+                }
+                else {
+                    fprintf(stderr, "Unsupported iterable type %i\n", iterable.type());
+                }
+
+                PycRef<ASTNode> func = stack.top();
+                stack.pop();
+
+                stack.push(new ASTCall(func, pparamList, kwparamList));
+            }
+            break;
         default:
             fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode);
             cleanBuild = false;
@@ -2782,6 +2872,7 @@ void print_formatted_value(PycRef<ASTFormattedValue> formatted_value, PycModule*
 
 static std::unordered_set<ASTNode *> node_seen;
 
+// TODO: Handle m_unpack for node correctly here.
 void print_src(PycRef<ASTNode> node, PycModule* mod, std::ostream& pyc_output)
 {
     if (node == NULL) {
@@ -2906,6 +2997,10 @@ void print_src(PycRef<ASTNode> node, PycModule* mod, std::ostream& pyc_output)
         break;
     case ASTNode::NODE_LIST:
         {
+            if (node.isUnpacked()) {
+                pyc_output << "*";
+            }
+
             pyc_output << "[";
             bool first = true;
             cur_indent++;
@@ -2999,6 +3094,9 @@ void print_src(PycRef<ASTNode> node, PycModule* mod, std::ostream& pyc_output)
         }
         break;
     case ASTNode::NODE_NAME:
+        if (node.isUnpacked()) {
+            pyc_output << "*";
+        }
         pyc_output << node.cast<ASTName>()->name()->value();
         break;
     case ASTNode::NODE_NODELIST:

FastStack.h

@@ -61,6 +61,25 @@ class FastStack {
         return m_ptr == -1;
     }
 
+    void debug_print(PycModule* mod, std::ostream& pyc_output)
+    {
+        pyc_output << "---- STACK CONTENTS ----\n";
+        if (empty()) {
+            pyc_output << "empty stack\n";
+        }
+        else {
+            for (int i = m_ptr; i >= 0; i--) {
+                print_src(m_stack[i], mod, pyc_output);
+                if (i == m_ptr) {
+                    pyc_output << " <- STACK TOP";
+                }
+                pyc_output << "\n";
+            }
+        }
+        pyc_output << "------------------------\n";
+    }
+
+
 private:
     std::vector<PycRef<ASTNode>> m_stack;
     int m_ptr;

pyc_object.h

@@ -6,21 +6,21 @@
 template <class _Obj>
 class PycRef {
 public:
-    PycRef() noexcept : m_obj() { }
+    PycRef() noexcept : m_obj(), m_unpack(false) { }
 
-    PycRef(_Obj* obj) noexcept : m_obj(obj)
+    PycRef(_Obj* obj) noexcept : m_obj(obj), m_unpack(false)
     {
         if (m_obj)
             m_obj->addRef();
     }
 
-    PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj)
+    PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack)
     {
         if (m_obj)
             m_obj->addRef();
     }
 
-    PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj)
+    PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack)
     {
         obj.m_obj = nullptr;
     }
@@ -38,6 +38,7 @@ class PycRef {
         if (m_obj)
             m_obj->delRef();
         m_obj = obj;
+        m_unpack = false;
         return *this;
     }
 
@@ -48,16 +49,20 @@ class PycRef {
         if (m_obj)
             m_obj->delRef();
         m_obj = obj.m_obj;
+        m_unpack = obj.m_unpack;
         return *this;
     }
 
     PycRef<_Obj>& operator=(PycRef<_Obj>&& obj) noexcept
     {
         m_obj = obj.m_obj;
+        m_unpack = obj.m_unpack;
         obj.m_obj = nullptr;
+        obj.m_unpack = false;
         return *this;
     }
 
+    // TODO: Handle m_unpack for remaining operators
     bool operator==(_Obj* obj) const { return m_obj == obj; }
     bool operator==(const PycRef<_Obj>& obj) const { return m_obj == obj.m_obj; }
     bool operator!=(_Obj* obj) const { return m_obj != obj; }
@@ -75,16 +80,28 @@ class PycRef {
     template <class _Cast>
     PycRef<_Cast> cast() const
     {
-        _Cast* result = dynamic_cast<_Cast*>(m_obj);
-        if (!result)
+        _Cast* casted_obj = dynamic_cast<_Cast*>(m_obj);
+        if (!casted_obj)
             throw std::bad_cast();
+
+        PycRef<_Cast> result = casted_obj;
+        result.setUnpacked(m_unpack);
+
         return result;
     }
 
     bool isIdent(const _Obj* obj) const { return m_obj == obj; }
 
+    bool isUnpacked() const { return m_unpack; }
+    void setUnpacked(bool unpack) { m_unpack = unpack; }
+
 private:
     _Obj* m_obj;
+
+    // References to an object can be either packed or unpacked.
+    // Usually unpacked references will be used with variables
+    // or lists but they may arise in other places as well.
+    bool m_unpack;
 };
 
 

tests/input/list_extend_2.py

@@ -0,0 +1,2 @@
+def get(l):
+    return [*l]

tests/input/test_unpack.py

@@ -0,0 +1,6 @@
+import struct
+
+def wtob(w):
+    return struct.pack('<'+'I'*len(w), *w)
+
+wtob([12,3])

tests/tokenized/list_extend_2.txt

@@ -0,0 +1,3 @@
+def get ( l ) : <EOL>
+<INDENT>
+return [ * l ] <EOL>

tests/tokenized/test_unpack.txt

@@ -0,0 +1,6 @@
+import struct <EOL>
+def wtob ( w ) : <EOL>
+<INDENT>
+return struct . pack ( '<' + 'I' * len ( w ) , * w ) <EOL>
+<OUTDENT>
+wtob ( [ 12 , 3 ] ) <EOL>