diff --git a/repro.py b/repro.py new file mode 100644 index 000000000..86ede4efd --- /dev/null +++ b/repro.py @@ -0,0 +1,42 @@ +import sys + +from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final, Function, ListOf, NotCompiled, Member +from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr + +import typed_python.compiler.native_compiler.native_ast as n + + +from typed_python import _types +assert not _types.checkForHashInstability() + +print(n.Type.zero) +print(n.Type.zero.overloads) + +def writer(): + Cls = Forward("Cls") + + @Cls.define + class Cls(Class): + m = Member(str) + + def f(self) -> Cls: + return Cls(m='HI') + + bytesToWrite = SerializationContext().serialize((Cls, ())) + + with open("a.dat", "wb") as f: + f.write(bytesToWrite) + + +def reader(): + with open("a.dat", "rb") as f: + Cls, args = SerializationContext().deserialize(f.read()) + + # print(x.__name__) + print(Cls(*args).f()) + + +if sys.argv[1:] == ['r']: + reader() +else: + writer() diff --git a/todo.txt b/todo.txt new file mode 100644 index 000000000..aa3c72534 --- /dev/null +++ b/todo.txt @@ -0,0 +1,228 @@ +- cvpo + - be able to re-instantiate objects if they don't exist + - be able to hash an MRTG a type/CVPO graph from both the compiler and identity perspective + - the 'type' interning system should handle both CVPO and Type now + - the act of walking types and CVPO should create the CVPOs as we go + - need an object to manage this - we should be able to delete these when done? + - when we duplicate the type object we'll produce a copy of the CVPO + - we need to be able to produce the actual objects +- fully internalized types shouldn't be able to have an unstable identity at all + - NamedModuleMember should identity hash the same way regardless anyways + - we should force these Globals to resolve to something specific at resolution time +- Class/HeldClass classMembers could still have forwards. These are PyObjects and need to + - be converted to Constants +- all Forward defined types need to expose as much of their internals as are known + - not bytecounts - this should be explicitly disallowed + - expose code to allow us to get the _first_ forward that defined a given class +- dotted globals? + - if we do this, how do we resolve the dotted global value? + - this is different for identity hashing than it is for serialization? + - what were we doing in the existing system? This is very squirrely and difficult. +- fix the way we walk things in MRTG - should be unified so that we actually can see CVPOs +- ensure that TypeFunction works as intended now. Some of the weird issues should be resolved now +- fix the ModuleRepresentation stuff +- redocument everything +- finish serialization + - should be able to serialize forward defined types + - ensure we are not leaking PyObjSnapshot values + - really PyObjSnapshot is the wrong name - i think it's more like 'TypeVisiblePyObj' + which reflects the reality that some objects are visible to the typing system and some + are not + - can we get rid of the whole serialize-mrtg thing? at this point, memos should just work right? + - this might leak which would be a problem + - should CVPOs be shared_ptr? we can still create cycles in them +- fill out a full CVPO model: + - classes + - tuples + - lists and other py datastructures + - hash their original contents. they are not supposed to change +- function objects + - default arguments need to be part of the instance closure if they have state? +- build a formal model for module objects and their contents + - the only reassignment we should allow is for a module member to go from unresolved to resolved + - nothing should change after import + - this would let us detect module changes + - we could serialize/deserialize without hitting the GIL + - as part of this, move more of the serialization logic to C++ which would be faster +- ensure that we don't try to MRTG something that's a forward defined type +- fix the compiler to deal with the new globals thing now that we have it well defined +- figure out how to properly internalize python classes + - the PyObjSnapshot thing needs to actually work + - we need to make sure we can actually destroy these if we're not going to keep them +- get rid of PyObject throughout Function + - should be like a FunctionGlobal +- Value should be holding a PyObjSnapshot, not an instance +- We're using the TP compiler hash in the compiler, which makes sense. Need to ensure that + the CVOV obyes the same rules for dotted accesses that the original compiler does. +- We need to ensure that Globals retain their source information if its available + - this means that they need to allow themselves to be constants + - the whole 'globalsRaw' thing is trying to do this but its a crappy way of doing it +- take a look at object identity + - we could keep a reverse lookup from all alive python objects to the original instance + for things like lists/classes/etc + + +strategy: + * fully fleshed out CVPO - infrastructure to help us manage all of this: + * CVPO is really 'PyObjSnapshot' and represents a frozen graph of python objects + * CVPO has an 'owner' which manages its lifetime. + * there's an 'internalized' owner which represents CVPO that's owned by the core and that has been internalized by hash + * these are leaked + * CVPO models instances and Types all the way out + * that is, a 'Type' is actually modeled with its pieces, which also have CVPO + * the 'Type' and 'mPyObject' are pointers to the real thing + * when you cross into a CVPO on another 'owner' then you can stop + * instances are the same way, including classes, etc. + * we replicate the MRTG hashing functionality on CVPO so that you can hash the items in a CVPO + * you're not forced to internalize if you do this + * when we freeze a Forward, we walk it to make a CVPO graph and hash it. + * we then internalize the new part of the graph + * then we walk back over the original objects and mark them as resolving to the new ones + * then we autoresolve to update forwards that point to Cells and Modules + * serialization + * when we serialize a forward type, we just do it normally + * but when we serialize a resolved type we serialize its CVPO graph + * then when we deserialize it, we can deserialize the graph and then internalize it if we want to + * this lets us gracefully handle the 'leak' case + + + + +question: how should we handle regular python classes defined in forwards? +* they will trigger MutuallyRecursiveTypeGroup and identityHash, which should be fine. +* we've said that Forwards shouldn't participate in MRTG. + * This means we need to get a representation of the object that is completely external to 'forwards' + * we won't allow monkey patching. +* I think we need to actually build the formal graph in CVPO, and this should proceed with standard +interning rules. This will allow us to completely remove the 'Forwards' from the object. + * when a forward gets defined and pointed at a python object, we'll allow a walk inside + * the question 'typeLooksResolvable' will be valid to ask of a Class that has forwards in it + * if we ask 'isForwardDefined' we'll have to walk in and see if it references any forwards + * this walk can't be cached until we make the model of the python object because the object is mutable + * this could be slow but at least it's simple +* if you think about it what we really have is a graph of regular mutable python objects, of which + forward defined types are just a special case. Periodically we walk this graph and check if + we can do a forward resolution at which point we make a formal model +* to do this correctly we need two things: + * a full model of python objects including Function and Class, PyList, PyTuple, etc. + * a way of associating the model to a python instance + * this association should be the one we use for MRTG of python instances that are interned - their shadows + should be what we see and talk about, not the instances themselves. + * the compiler should be using this, not the original objects + * every CVPO should have a 'canonical' instance + * every PyObj should have a mapping to the CVPO that represents it if its been forward resolved + * you can ask if a python object isForwardDefined + * function instances can be forward defined as well + * if you choose to resolve (or we autoresolve, or a TypeFunction forces resolution) then we build the CVPO mapping + * then CVPO can completely replace TOPO + + + +def f(x): + class C(Class): + def g(self): + return x + +f(1) vs f(2) + +what kind of global should 'x' be? +- we could leave it a GlobalInCell but it's really not - its a constant and shouldn't be allowed to change + - part of the resolution process needs to get rid of the 'cellness' of this + - fully resolved functions should only have constants, NamedModuleMember instances, and Unbound + + +def f(x): + @Entrypoint + def g(): + return x + return g + +in this case, 'x' is held in the function's closure, so it's not really 'typelike' + + +how should this work: +* we build a graph of unresolved forward types + * these can see each other through cells, module members, and TypeFunction calls that yield Forwards. +* we trigger 'forward resolution' +* we build a complete model of all the python objects and types that are visible to the graph + * named module members are special - we stop the walk when we hit such an object and its not part of our 'resolvability' criteria even if the object is unresolved at that point. + * this is because any references that are through an actual module instead of through explicit + forwards cannot affect our identity, only our compilation + * everything else gets mapped to an explicit 'constant' +* + + +# core ideas: +# 1. it's typelike IFF it can't change during program execution +# 2. module members, class definitions, etc, don't change during program execution once set +# 3. an @Function / @Entrypoint should define an 'untyped' function to whatever degree possible +# since the user isn't providing types +# 4. at the moment we cross Entrypoint boundaries we can look at a function's type graph and +# determine how we want to handle it +# 5. we could have the idea of a 'StatefulClass' holding a class that references some state +# that's independent of its type. This could get passed to its instances +# 6. a Function's default arguments, if stateful, should be part of its data not its type +# 7. if we have an anonymous module, and that module is defined in RF, then that module's identity +# is it's "incoming state" + + +# this is just a regular TP function. nothing funny going on +@Function +def f(): + pass + + +# this should hold 'x' as a NamedModuleMember if this is defined inside of a module +# this is part of the type +@Function +def f(): + return x + + +# if the module is not named, then the module info is not part of the type. +# instead it's part of the Function's closure. 'x' should go through the closure +# and hit the dict indirectly. If the function gets passed to a Class then we'll +# pull all of that apart and re-use it. If it just becomes a free function then +# when we interpret it, it's fine, and when we compile it we get a chance to look carefully +# at it and decide what to do. +__module__ = 'some random module we can't find +@Function +def f(): + return x + + +def makeF(x): + @Function + def f(): + + + + +execution plan: +1. if a function's globals dict is not globally visible, then make it part of the _closure_ + * FunctionGlobal can point to the closure, just like the ClosureVariables + * this is always considered 'resolved' in the sense that such an object is untyped and therefore fully resolved + * need to allow the function to be recreated correctly given this case +2. allow forward function types to be instantiated if their closures are not forward. This is a forward function instance. + * forward function instances can't be executed but they can be created. They are placeholder objects ready to be resolved + just like forward types, or to be passed into a Class definition +3. allow forward function types to be resolved just like regular function types + * forward function instances also participate in instance resolution + + + + + +how to do this: +1. new graph needs a hashing methodology that's totally independent of MRTG +2. type internalization framework needs to use it + + + + +this is kind of working. But: +1. we need for everything in the core type graph to be 'Instance' +2. we need to not need to instantiate the PyTypeObject for a Type until the end +3. we should be able to do this by insisting that we wire the Instance of PythonObjectOfType + at the very end + diff --git a/typed_python/AllTypes.hpp b/typed_python/AllTypes.hpp index 1549e8044..26dab01fa 100644 --- a/typed_python/AllTypes.hpp +++ b/typed_python/AllTypes.hpp @@ -37,7 +37,6 @@ #include "ValueType.hpp" #include "AlternativeType.hpp" #include "ConcreteAlternativeType.hpp" -#include "PythonSubclassType.hpp" #include "PythonObjectOfTypeType.hpp" #include "FunctionType.hpp" #include "HeldClassType.hpp" diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index e523c3ce2..4472f81e6 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -19,17 +19,36 @@ #include "Type.hpp" #include "ReprAccumulator.hpp" + +PyDoc_STRVAR(AlternativeMatcher_doc, + "AlternativeMatcher: an instance of 'A.matches' where 'A' is an alternative.\n" +); + // Models an alternatives '.matches' result class AlternativeMatcher : public Type { public: + AlternativeMatcher() : Type(TypeCategory::catAlternativeMatcher) + { + } + AlternativeMatcher(Type* inAlternative) : Type(TypeCategory::catAlternativeMatcher) { - m_is_default_constructible = false; + m_is_forward_defined = true; m_alternative = inAlternative; - m_size = inAlternative->bytecount(); - m_is_simple = false; + } + + const char* docConcrete() { + return AlternativeMatcher_doc; + } + + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "AlternativeMatcher(" + + qualname_to_name(m_alternative->computeRecursiveName(typeStack)) + + ")"; + } - endOfConstructorInitialization(); // finish initializing the type object. + void initializeDuringDeserialization(Type* altType) { + m_alternative = altType; } template @@ -48,42 +67,43 @@ class AlternativeMatcher : public Type { visitor(m_alternative); } - bool _updateAfterForwardTypesChanged() { - bool anyChanged = false; - - std::string name = "AlternativeMatcher(" + m_alternative->name(true) + ")"; - size_t size = m_alternative->bytecount(); - - anyChanged = ( - name != m_name || - size != m_size - ); + void postInitializeConcrete() { + m_is_default_constructible = false; + m_size = m_alternative->bytecount(); + } - m_name = name; - m_stripped_name = ""; - m_size = size; + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_alternative = ((AlternativeMatcher*)forwardDefinitionOfSelf)->m_alternative; + } - return anyChanged; + Type* cloneForForwardResolutionConcrete() { + return new AlternativeMatcher(); } - void _updateTypeMemosAfterForwardResolution() { - AlternativeMatcher::Make(m_alternative, this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_alternative, groupMap); } - static AlternativeMatcher* Make(Type* alt, AlternativeMatcher* knownType=nullptr) { - PyEnsureGilAcquired getTheGil; + static AlternativeMatcher* Make(Type* alt) { + if (alt->isForwardDefined()) { + return new AlternativeMatcher(alt); + } - static std::map m; + PyEnsureGilAcquired getTheGil; - auto it = m.find(alt); + static std::map memo; - if (it == m.end()) { - it = m.insert( - std::make_pair(alt, knownType ? knownType : new AlternativeMatcher(alt)) - ).first; + auto it = memo.find(alt); + if (it != memo.end()) { + return it->second; } - return it->second; + AlternativeMatcher* res = new AlternativeMatcher(alt); + AlternativeMatcher* concrete = (AlternativeMatcher*)res->forwardResolvesTo(); + + memo[alt] = concrete; + + return concrete; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/AlternativeType.cpp b/typed_python/AlternativeType.cpp index ab255a31f..6e270143a 100644 --- a/typed_python/AlternativeType.cpp +++ b/typed_python/AlternativeType.cpp @@ -16,33 +16,6 @@ #include "AllTypes.hpp" -bool Alternative::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() == TypeCategory::catConcreteAlternative) { - other = other->getBaseType(); - } - - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - Alternative* otherO = (Alternative*)other; - - if (m_subtypes.size() != otherO->m_subtypes.size()) { - return false; - } - - for (long k = 0; k < m_subtypes.size(); k++) { - if (m_subtypes[k].first != otherO->m_subtypes[k].first) { - return false; - } - if (!m_subtypes[k].second->isBinaryCompatibleWith(otherO->m_subtypes[k].second)) { - return false; - } - } - - return true; -} - int64_t Alternative::refcount(instance_ptr i) const { if (m_all_alternatives_empty) { return 0; @@ -50,51 +23,6 @@ int64_t Alternative::refcount(instance_ptr i) const { return ((layout**)i)[0]->refcount; } -bool Alternative::_updateAfterForwardTypesChanged() { - m_arg_positions.clear(); - m_default_construction_type = nullptr; - - bool is_default_constructible = false; - bool all_alternatives_empty = true; - int default_construction_ix = 0; - - for (auto& subtype_pair: m_subtypes) { - if (subtype_pair.second->bytecount() > 0) { - all_alternatives_empty = false; - } - - if (m_arg_positions.find(subtype_pair.first) != m_arg_positions.end()) { - throw std::runtime_error("Can't create an alternative with " + - subtype_pair.first + " defined twice."); - } - - size_t argPosition = m_arg_positions.size(); - - m_arg_positions[subtype_pair.first] = argPosition; - - if (subtype_pair.second->is_default_constructible() && !is_default_constructible) { - is_default_constructible = true; - default_construction_ix = m_arg_positions[subtype_pair.first]; - } - } - - size_t size = (all_alternatives_empty ? 1 : sizeof(void*)); - - bool anyChanged = ( - size != m_size || - m_default_construction_ix != default_construction_ix || - m_all_alternatives_empty != all_alternatives_empty || - m_is_default_constructible != is_default_constructible - ); - - m_size = size; - m_default_construction_ix = default_construction_ix; - m_all_alternatives_empty = all_alternatives_empty; - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - bool Alternative::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (m_all_alternatives_empty) { if (*(uint8_t*)left < *(uint8_t*)right) { @@ -236,33 +164,69 @@ void Alternative::assign(instance_ptr self, instance_ptr other) { // static Alternative* Alternative::Make( - std::string name, - std::string moduleName, - const std::vector >& types, - const std::map& methods //methods preclude us from being in the memo - ) { - return new Alternative(name, moduleName, types, methods); -} + std::string name, + std::string moduleName, + const std::vector >& types, + const std::map& methods //methods preclude us from being in the memo +) { + bool anyForward = false; + + for (auto nameAndTup: types) { + if (nameAndTup.second->isForwardDefined()) { + anyForward = true; + } + } -Type* Alternative::concreteSubtype(size_t which) { - if (!m_subtypes_concrete.size()) { - for (long k = 0; k < m_subtypes.size(); k++) { - m_subtypes_concrete.push_back(ConcreteAlternative::Make(this, k)); + for (auto nameAndMeth: methods) { + if (nameAndMeth.second->isForwardDefined() || nameAndMeth.second->hasUnresolvedSymbols(true)) { + anyForward = true; } } + // initialize a forward Alternative + Alternative* res = new Alternative(name, moduleName, types, methods); + + // construct its subtypes - we are always mutually recursive with these so we need + // to instantiate them directly in the case of the forward + res->initializeConcreteSubclasses(); + + if (anyForward) { + return res; + } + + Alternative* concrete = (Alternative*)res->forwardResolvesTo(); + + return concrete; +} + +void Alternative::initializeConcreteSubclasses() { + if (!m_is_forward_defined) { + throw std::runtime_error( + "Alternative::initializeConcreteSubclasses only makes sense for Forward-defined " + "Alternatives." + ); + } + + for (long k = 0; k < m_subtypes.size(); k++) { + m_subtypes_concrete.push_back(new ConcreteAlternative(this, k)); + } +} + +ConcreteAlternative* Alternative::concreteSubtype(size_t which) { + if (m_subtypes_concrete.size() != m_subtypes.size()) { + throw std::runtime_error("Corrupt Alternative - somehow we don't have a subtype."); + } + if (which < 0 || which >= m_subtypes_concrete.size()) { throw std::runtime_error("Invalid alternative index."); } - return m_subtypes_concrete[which]; + return (ConcreteAlternative*)m_subtypes_concrete[which]; } Type* Alternative::pickConcreteSubclassConcrete(instance_ptr data) { - if (!m_subtypes_concrete.size()) { - for (long k = 0; k < m_subtypes.size(); k++) { - m_subtypes_concrete.push_back(ConcreteAlternative::Make(this, k)); - } + if (m_subtypes_concrete.size() != m_subtypes.size()) { + throw std::runtime_error("Corrupt Alternative - somehow we don't have a subtype."); } uint8_t i = which(data); @@ -274,7 +238,7 @@ void Alternative::constructor(instance_ptr self) { assertForwardsResolvedSufficientlyToInstantiate(); if (!m_default_construction_type) { - m_default_construction_type = ConcreteAlternative::Make(this, m_default_construction_ix); + m_default_construction_type = concreteSubtype(m_default_construction_ix); } m_default_construction_type->constructor(self); diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index f2649d247..bb632175a 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -19,6 +19,8 @@ #include "Type.hpp" #include "CompositeType.hpp" +class ConcreteAlternative; + PyDoc_STRVAR(Alternative_doc, "Alternative(\n" " name,\n" @@ -88,6 +90,10 @@ class Alternative : public Type { typedef layout* layout_ptr; + Alternative() : Type(TypeCategory::catAlternative) + { + } + Alternative(std::string name, std::string moduleName, const std::vector >& subtypes, @@ -100,38 +106,107 @@ class Alternative : public Type { m_methods(methods), m_hasGetAttributeMagicMethod(false) { + m_is_forward_defined = true; + m_name = name; - m_moduleName = moduleName; - m_is_simple = false; - m_hasGetAttributeMagicMethod = m_methods.find("__getattribute__") != m_methods.end(); + m_module_name = moduleName; if (m_subtypes.size() > 255) { throw std::runtime_error("Can't have an alternative with more than 255 subelements"); } + } + + const char* docConcrete() { + return Alternative_doc; + } + + void initializeDuringDeserialization( + std::string name, + std::string moduleName, + const std::vector >& types, + const std::map& methods, + const std::vector& subtypes_concrete + ) { + m_name = name; + m_module_name = moduleName; + m_methods = methods; + m_subtypes = types; + m_subtypes_concrete = subtypes_concrete; - m_doc = Alternative_doc; + if (m_subtypes.size() != m_subtypes_concrete.size()) { + throw std::runtime_error("Corrupt Alternative: #types != #subtypes"); + } + } - // call this _first_, so that our core properties, (which we know) are created - // before we walk down to the ConcreteAlternatives - _updateAfterForwardTypesChanged(); + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + Alternative* selfT = (Alternative*)forwardDefinitionOfSelf; + + m_name = selfT->m_name; + m_module_name = selfT->m_module_name; + m_hasGetAttributeMagicMethod = selfT->m_hasGetAttributeMagicMethod; + m_methods = selfT->m_methods; + m_subtypes = selfT->m_subtypes; + m_subtypes_concrete = selfT->m_subtypes_concrete; + m_default_construction_ix = selfT->m_default_construction_ix; + m_default_construction_type = selfT->m_default_construction_type; + } - endOfConstructorInitialization(); // finish initializing the type object. + Type* cloneForForwardResolutionConcrete() { + return new Alternative(); } - std::string nameWithModuleConcrete() { - if (m_moduleName.size() == 0) { - return m_name; + void postInitializeConcrete() { + m_hasGetAttributeMagicMethod = m_methods.find("__getattribute__") != m_methods.end(); + + m_arg_positions.clear(); + m_default_construction_type = nullptr; + + bool is_default_constructible = false; + bool all_alternatives_empty = true; + int default_construction_ix = 0; + + for (auto& subtype_pair: m_subtypes) { + if (subtype_pair.second->bytecount() > 0) { + all_alternatives_empty = false; + } + + if (m_arg_positions.find(subtype_pair.first) != m_arg_positions.end()) { + throw std::runtime_error("Can't create an alternative with " + + subtype_pair.first + " defined twice."); + } + + size_t argPosition = m_arg_positions.size(); + + m_arg_positions[subtype_pair.first] = argPosition; + + if (subtype_pair.second->is_default_constructible() && !is_default_constructible) { + is_default_constructible = true; + default_construction_ix = m_arg_positions[subtype_pair.first]; + } } - return m_moduleName + "." + m_name; + size_t size = (all_alternatives_empty ? 1 : sizeof(void*)); + + m_size = size; + m_default_construction_ix = default_construction_ix; + m_all_alternatives_empty = all_alternatives_empty; + m_is_default_constructible = is_default_constructible; } - std::string moduleNameConcrete() { - if (m_moduleName.size() == 0) { - return "builtins"; + void updateInternalTypePointersConcrete(const std::map& groupMap) { + for (auto& nameAndSub: m_subtypes) { + updateTypeRefFromGroupMap(nameAndSub.second, groupMap); } - return m_moduleName; + for (auto& sub: m_subtypes_concrete) { + updateTypeRefFromGroupMap(sub, groupMap); + } + + updateTypeRefFromGroupMap(m_default_construction_type, groupMap); + + for (auto& nameAndSub: m_methods) { + updateTypeRefFromGroupMap(nameAndSub.second, groupMap); + } } bool hasGetAttributeMagicMethod() const { @@ -146,25 +221,19 @@ class Alternative : public Type { template void _visitReferencedTypes(const visitor_type& visitor) { - for (auto& subtype_pair: m_subtypes) { - Type* t = (Type*)subtype_pair.second; - visitor(t); - assert(t == subtype_pair.second); + for (auto subtype_pair: m_subtypes) { + visitor(subtype_pair.second); } - for (long k = 0; k < m_subtypes.size(); k++) { - Type* t = concreteSubtype(k); + + for (auto t: m_subtypes_concrete) { visitor(t); } - for (auto& method_pair: m_methods) { - Type* t = (Type*)method_pair.second; - visitor(t); - assert(t == method_pair.second); + for (auto method_pair: m_methods) { + visitor(method_pair.second); } } - bool _updateAfterForwardTypesChanged(); - template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -289,6 +358,11 @@ class Alternative : public Type { typed_python_hash_type hash(instance_ptr left); + std::pair unwrap(instance_ptr self) { + size_t ix = which(self); + return std::make_pair(m_subtypes[ix].second, eltPtr(self)); + } + instance_ptr eltPtr(instance_ptr self) const; int64_t which(instance_ptr self) const; @@ -304,14 +378,14 @@ class Alternative : public Type { void assign(instance_ptr self, instance_ptr other); static Alternative* Make( - std::string name, - std::string moduleName, - const std::vector >& types, - const std::map& methods //methods preclude us from being in the memo - ); + std::string name, + std::string moduleName, + const std::vector >& types, + const std::map& methods + ); Alternative* renamed(std::string newName) { - return Make(newName, m_moduleName, m_subtypes, m_methods); + return Make(newName, m_module_name, m_subtypes, m_methods); } const std::vector >& subtypes() const { @@ -332,12 +406,16 @@ class Alternative : public Type { return m_methods; } - Type* concreteSubtype(size_t which); + const std::vector& getSubtypesConcrete() const { + return m_subtypes_concrete; + } + + ConcreteAlternative* concreteSubtype(size_t which); private: - //name of the module in which this Alternative was defined. - std::string m_moduleName; + void initializeConcreteSubclasses(); + //name of the module in which this Alternative was defined. bool m_all_alternatives_empty; int m_default_construction_ix; diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index 8dd5a56ca..d9db4c579 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -19,17 +19,31 @@ #include "Type.hpp" #include "ReprAccumulator.hpp" +PyDoc_STRVAR(BoundMethod_doc, + "BoundMethod(T, name)\n\n" + "Holds an instance of 'T' as 't' and has a call method that dispatches to t.name(...)\n" +); + + class BoundMethod : public Type { public: + BoundMethod() : Type(TypeCategory::catBoundMethod) + { + } + BoundMethod(Type* inFirstArg, std::string funcName) : Type(TypeCategory::catBoundMethod) { + m_is_forward_defined = true; + m_funcName = funcName; - m_is_default_constructible = false; m_first_arg = inFirstArg; m_size = inFirstArg->bytecount(); - m_is_simple = false; - endOfConstructorInitialization(); // finish initializing the type object. + recomputeName(); + } + + const char* docConcrete() { + return BoundMethod_doc; } template @@ -49,48 +63,58 @@ class BoundMethod : public Type { visitor(m_first_arg); } - bool _updateAfterForwardTypesChanged() { - bool anyChanged = false; + void initializeDuringDeserialization(std::string funcName, Type* firstArgType) { + m_first_arg = firstArgType; + m_funcName = funcName; + } - // note that you can't have '.' in the name of a type, so we use '::'. - // otherwise, the __name__ attribute of the type gets cut off at the last '.' and - // looks like a memory corruption issue. It also prevents you from knowing what - // type you're looking at. - std::string name = "BoundMethod(" + m_first_arg->name(true) + ", " + m_funcName + ")"; - size_t size = m_first_arg->bytecount(); + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "BoundMethod(" + + m_first_arg->computeRecursiveName(typeStack) + + ", " + m_funcName + + ")"; + } - anyChanged = ( - name != m_name || - size != m_size - ); + void postInitializeConcrete() { + m_size = m_first_arg->bytecount(); + m_is_default_constructible = false; + } - m_name = name; - m_stripped_name = ""; - m_size = size; + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_first_arg = ((BoundMethod*)forwardDefinitionOfSelf)->m_first_arg; + m_funcName = ((BoundMethod*)forwardDefinitionOfSelf)->m_funcName; + } - return anyChanged; + Type* cloneForForwardResolutionConcrete() { + return new BoundMethod(); } - void _updateTypeMemosAfterForwardResolution() { - BoundMethod::Make(m_first_arg, m_funcName, this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_first_arg, groupMap); } - static BoundMethod* Make(Type* c, std::string funcName, BoundMethod* knownType=nullptr) { + static BoundMethod* Make(Type* firstArg, std::string funcName) { + if (firstArg->isForwardDefined()) { + return new BoundMethod(firstArg, funcName); + } + PyEnsureGilAcquired getTheGil; typedef std::pair keytype; - static std::map m; - - auto it = m.find(keytype(c, funcName)); + static std::map memo; - if (it == m.end()) { - it = m.insert( - std::make_pair(keytype(c, funcName), knownType ? knownType : new BoundMethod(c, funcName)) - ).first; + auto it = memo.find(keytype(firstArg, funcName)); + if (it != memo.end()) { + return it->second; } - return it->second; + BoundMethod* res = new BoundMethod(firstArg, funcName); + BoundMethod* concrete = (BoundMethod*)res->forwardResolvesTo(); + + memo[keytype(firstArg, funcName)] = concrete; + + return concrete; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/BytesType.cpp b/typed_python/BytesType.cpp index 3a032e9b3..6511f50b6 100644 --- a/typed_python/BytesType.cpp +++ b/typed_python/BytesType.cpp @@ -476,14 +476,6 @@ void BytesType::assign(instance_ptr self, instance_ptr other) { destroy((instance_ptr)&old); } -bool BytesType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; -} - void BytesType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { stream << "b" << "'"; long bytes = count(self); diff --git a/typed_python/BytesType.hpp b/typed_python/BytesType.hpp index 374b20819..120c38129 100644 --- a/typed_python/BytesType.hpp +++ b/typed_python/BytesType.hpp @@ -36,12 +36,8 @@ class BytesType : public Type { m_name = "bytes"; m_is_default_constructible = true; m_size = sizeof(layout*); - - endOfConstructorInitialization(); // finish initializing the type object. } - bool isBinaryCompatibleWithConcrete(Type* other); - void repr(instance_ptr self, ReprAccumulator& stream, bool isStr); template @@ -203,4 +199,6 @@ class BytesType : public Type { static layout* replace(layout* l, layout* old, layout* the_new, int64_t count); static layout* translate(layout* l, layout* table, layout* to_delete); static layout* maketrans(layout* from, layout* to); + + void postInitializeConcrete() {} }; diff --git a/typed_python/ClassType.cpp b/typed_python/ClassType.cpp index 9ff6f8122..86ae4b41d 100644 --- a/typed_python/ClassType.cpp +++ b/typed_python/ClassType.cpp @@ -16,26 +16,6 @@ #include "AllTypes.hpp" -bool Class::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - Class* otherO = (Class*)other; - - return m_heldClass->isBinaryCompatibleWith(otherO->m_heldClass); -} - -bool Class::_updateAfterForwardTypesChanged() { - bool is_default_constructible = m_heldClass->is_default_constructible(); - - bool anyChanged = m_is_default_constructible != is_default_constructible; - - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - instance_ptr Class::eltPtr(instance_ptr self, int64_t ix) const { layout& l = *instanceToLayout(self); return m_heldClass->eltPtr(l.data, ix); @@ -116,7 +96,8 @@ bool Class::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool pyComparisonOp == Py_GE ? ">=" : "?", name().c_str(), name().c_str() - ); + ); + throw PythonExceptionSet(); } diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index dfa60ad98..d0889e9e5 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -32,7 +32,7 @@ PyDoc_STRVAR(Class_doc, "\n" "Methods become TypedFunction instances, and support overloading.\n" "Define data members with Member.\n" - ); +); class Class : public Type { public: @@ -45,19 +45,50 @@ class Class : public Type { typedef layout* layout_ptr; + Class() : Type(catClass) + { + } + Class(std::string name, HeldClass* inClass) : Type(catClass), m_heldClass(inClass) { + // if the Held class is not forward, then neither are we! + m_is_forward_defined = m_heldClass->isForwardDefined(); m_size = sizeof(layout*); m_is_default_constructible = inClass->is_default_constructible(); m_name = name; - m_doc = Class_doc; - m_is_simple = false; + } + + const char* docConcrete() { + return Class_doc; + } + + void initializeDuringDeserialization(std::string inName, HeldClass* held) { + m_heldClass = held; + m_name = inName; + } + + void postInitializeConcrete() { + m_size = sizeof(layout*); + m_is_default_constructible = m_heldClass->is_default_constructible(); + } - endOfConstructorInitialization(); // finish initializing the type object. + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_heldClass = ((Class*)forwardDefinitionOfSelf)->m_heldClass; + m_name = ((Class*)forwardDefinitionOfSelf)->m_name; + } + + Type* cloneForForwardResolutionConcrete() { + return new Class(); + } - inClass->setClassType(this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_heldClass, groupMap); + + _visitReferencedTypes([&](Type*& typePtr) { + updateTypeRefFromGroupMap(typePtr, groupMap); + }); } // convert an instance of the class to an actual layout pointer. Because @@ -106,25 +137,29 @@ class Class : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { } template void _visitReferencedTypes(const visitor_type& visitor) { - Type* t = m_heldClass; - visitor(t); - assert(t == m_heldClass); + if (m_heldClass) { + Type* hc = m_heldClass; + visitor(hc); + if (hc != (Type*)m_heldClass) { + if (!hc || !hc->isHeldClass()) { + throw std::runtime_error("Can't set a Class's heldClass to a non HeldClass"); + } + + m_heldClass = (HeldClass*)hc; + } + } } Type* pickConcreteSubclassConcrete(instance_ptr self) { return actualTypeForLayout(self); } - bool _updateAfterForwardTypesChanged(); - static Class* Make( std::string inName, const std::vector& bases, @@ -134,8 +169,7 @@ class Class : public Type { const std::map& staticFunctions, const std::map& propertyFunctions, const std::map& classMembers, - const std::map& classMethods, - bool isNew + const std::map& classMethods ) { std::vector heldClassBases; @@ -144,21 +178,19 @@ class Class : public Type { heldClassBases.push_back(c->getHeldClass()); } - return new Class( + HeldClass* hc = HeldClass::Make( inName, - HeldClass::Make( - inName, - heldClassBases, - isFinal, - members, - memberFunctions, - staticFunctions, - propertyFunctions, - classMembers, - classMethods, - isNew - ) + heldClassBases, + isFinal, + members, + memberFunctions, + staticFunctions, + propertyFunctions, + classMembers, + classMethods ); + + return hc->getClassType(); } static const char* pyComparisonOpToMethodName(int pyComparisonOp) { diff --git a/typed_python/ClosureVariableBinding.cpp b/typed_python/ClosureVariableBinding.cpp new file mode 100644 index 000000000..44aa08ad6 --- /dev/null +++ b/typed_python/ClosureVariableBinding.cpp @@ -0,0 +1,101 @@ +#include "PyInstance.hpp" +#include "FunctionType.hpp" + +Instance ClosureVariableBinding::extractValueOrContainingClosure(Type* closureType, instance_ptr data) { + for (long stepIx = 0; stepIx < size(); stepIx++) { + ClosureVariableBindingStep step = (*this)[stepIx]; + + if (step.isFunction()) { + closureType = (Type*)step.getFunction(); + } else + if (step.isNamedField()) { + if (closureType->isPythonObjectOfType() && closureType == PythonObjectOfType::AnyPyDict()) { + PyObject* dict = PythonObjectOfType::getPyObj(data); + if (!dict || !PyDict_Check(dict)) { + throw std::runtime_error("Invalid closure: expected a populated PyDict"); + } + PyObject* entry = PyDict_GetItemString(dict, step.getNamedField().c_str()); + if (!entry) { + // TODO: this is wrong. Really we need a pattern that lets us communicate + // the fact that this entry is empty and that we should throw a name error. + throw std::runtime_error("Invalid closure: expected " + step.getNamedField() + " of a Dict to be populated."); + } + + return Instance::create(entry); + } else + if (closureType->isNamedTuple()) { + NamedTuple* tupType = (NamedTuple*)closureType; + auto it = tupType->getNameToIndex().find(step.getNamedField()); + if (it == tupType->getNameToIndex().end()) { + throw std::runtime_error( + "Invalid closure: expected NamedTuple to have field " + + step.getNamedField() + " but it doesn't." + ); + } + + closureType = tupType->getTypes()[it->second]; + data = data + tupType->getOffsets()[it->second]; + } else + if (closureType->getTypeCategory() == Type::TypeCategory::catClass) { + Class* clsType = (Class*)closureType; + int index = clsType->getMemberIndex(step.getNamedField().c_str()); + if (index == -1) { + throw std::runtime_error("Can't find a field " + step.getNamedField() + " in class " + clsType->name()); + } + + if (!clsType->checkInitializationFlag(data, index)) { + throw std::runtime_error("Closure field " + step.getNamedField() + " is not populated."); + } + + closureType = clsType->getMemberType(index); + data = clsType->eltPtr(data, index); + } else { + throw std::runtime_error( + "Invalid closure: expected to find a Class or a NamedTuple." + ); + } + } else + if (step.isIndexedField()) { + if (!closureType->isComposite()) { + throw std::runtime_error("Invalid closure: expected a NamedTuple or Tuple but got " + closureType->name()); + } + + CompositeType* tupType = (CompositeType*)closureType; + + if (step.getIndexedField() < 0 || step.getIndexedField() >= tupType->getTypes().size()) { + throw std::runtime_error( + "Invalid closure: index " + format(step.getIndexedField()) + " is out of bounds in closure " + tupType->name() + ); + } + + closureType = tupType->getTypes()[step.getIndexedField()]; + data = data + tupType->getOffsets()[step.getIndexedField()]; + } else + if (step.isCellAccess()) { + if (!(closureType->getTypeCategory() == Type::TypeCategory::catPyCell || + closureType->getTypeCategory() == Type::TypeCategory::catTypedCell)) { + throw std::runtime_error( + "Invalid closure: expected a cell, but got " + + Type::categoryToString(closureType->getTypeCategory()) + ); + } + + if (stepIx + 1 == size()) { + // do nothing, because this function grabs the containing + // closure + } else { + if (closureType->getTypeCategory() == Type::TypeCategory::catPyCell) { + throw std::runtime_error("Corrupt closure encountered: a PyCell should always be the last step"); + } + + // it's a typed closure + data = ((TypedCellType*)closureType)->get(data); + closureType = ((TypedCellType*)closureType)->getHeldType(); + } + } else { + throw std::runtime_error("Corrupt closure variable binding enountered."); + } + } + + return Instance(data, closureType); +} diff --git a/typed_python/ClosureVariableBinding.hpp b/typed_python/ClosureVariableBinding.hpp new file mode 100644 index 000000000..69cf51875 --- /dev/null +++ b/typed_python/ClosureVariableBinding.hpp @@ -0,0 +1,362 @@ +/****************************************************************************** + Copyright 2017-2022 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Type.hpp" + +class ClosureVariableBindingStep { + enum class BindingType { + FUNCTION = 1, + NAMED_FIELD = 2, + INDEXED_FIELD = 3, + ACCESS_CELL = 4 + }; + +public: + ClosureVariableBindingStep() : + mKind(BindingType::ACCESS_CELL), + mIndexedFieldToAccess(0), + mFunctionToBind(nullptr) + {} + + ClosureVariableBindingStep(Type* bindFunction) : + mKind(BindingType::FUNCTION), + mIndexedFieldToAccess(0), + mFunctionToBind(bindFunction) + {} + + ClosureVariableBindingStep(std::string fieldAccess) : + mKind(BindingType::NAMED_FIELD), + mIndexedFieldToAccess(0), + mFunctionToBind(nullptr), + mNamedFieldToAccess(fieldAccess) + {} + + ClosureVariableBindingStep(int elementAccess) : + mKind(BindingType::INDEXED_FIELD), + mFunctionToBind(nullptr), + mIndexedFieldToAccess(elementAccess) + {} + + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + if (isFunction()) { + v.visitHash(ShaHash(1)); + v.visitTopo(getFunction()); + } else + if (isNamedField()) { + v.visitHash(ShaHash(2) + ShaHash(getNamedField())); + } else + if (isIndexedField()) { + v.visitHash(ShaHash(3) + ShaHash(getIndexedField())); + } else + if (isCellAccess()) { + v.visitHash(ShaHash(4)); + } else { + v.visitErr("Invalid ClosureVariableBindingStep found"); + } + } + + static ClosureVariableBindingStep AccessCell() { + ClosureVariableBindingStep step; + return step; + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (mKind == BindingType::FUNCTION) { + visitor(mFunctionToBind); + } + } + + bool isFunction() const { + return mKind == BindingType::FUNCTION; + } + + bool isNamedField() const { + return mKind == BindingType::NAMED_FIELD; + } + + bool isIndexedField() const { + return mKind == BindingType::INDEXED_FIELD; + } + + bool isCellAccess() const { + return mKind == BindingType::ACCESS_CELL; + } + + Type* getFunction() const { + if (!isFunction()) { + throw std::runtime_error("Binding is not a function"); + } + + return mFunctionToBind; + } + + std::string getNamedField() const { + if (!isNamedField()) { + throw std::runtime_error("Binding is not a named field bindng"); + } + + return mNamedFieldToAccess; + } + + int getIndexedField() const { + if (!isIndexedField()) { + throw std::runtime_error("Binding is not an index field bindng"); + } + + return mIndexedFieldToAccess; + } + + bool operator<(const ClosureVariableBindingStep& step) const { + if (mKind < step.mKind) { + return true; + } + + if (mKind > step.mKind) { + return false; + } + + if (mKind == BindingType::ACCESS_CELL) { + return false; + } + + if (mKind == BindingType::FUNCTION) { + return mFunctionToBind < step.mFunctionToBind; + } + + if (mKind == BindingType::NAMED_FIELD) { + return mNamedFieldToAccess < step.mNamedFieldToAccess; + } + + if (mKind == BindingType::INDEXED_FIELD) { + return mIndexedFieldToAccess < step.mIndexedFieldToAccess; + } + + return false; + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + + if (mKind == BindingType::ACCESS_CELL) { + buffer.writeUnsignedVarintObject(0, 0); + } + else if (mKind == BindingType::FUNCTION) { + buffer.writeUnsignedVarintObject(0, 1); + context.serializeNativeType(mFunctionToBind, buffer, 1); + } + else if (mKind == BindingType::NAMED_FIELD) { + buffer.writeUnsignedVarintObject(0, 2); + buffer.writeStringObject(1, mNamedFieldToAccess); + } + else if (mKind == BindingType::INDEXED_FIELD) { + buffer.writeUnsignedVarintObject(0, 3); + buffer.writeUnsignedVarintObject(1, mIndexedFieldToAccess); + } + + buffer.writeEndCompound(); + } + + template + static ClosureVariableBindingStep deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + ClosureVariableBindingStep out; + + int whichBinding = -1; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + assertWireTypesEqual(wireType, WireType::VARINT); + whichBinding = buffer.readUnsignedVarint(); + + if (whichBinding == 0) { + out = ClosureVariableBindingStep::AccessCell(); + } + } + else if (fieldNumber == 1) { + if (whichBinding == -1) { + throw std::runtime_error("Corrupt ClosureVariableBindingStep"); + } + if (whichBinding == 1) { + out = ClosureVariableBindingStep( + context.deserializeNativeType(buffer, wireType) + ); + } + else if (whichBinding == 2) { + assertWireTypesEqual(wireType, WireType::BYTES); + out = ClosureVariableBindingStep(buffer.readStringObject()); + } + else if (whichBinding == 3) { + assertWireTypesEqual(wireType, WireType::VARINT); + out = ClosureVariableBindingStep(buffer.readUnsignedVarint()); + } + } else { + throw std::runtime_error("Corrupt ClosureVariableBindingStep"); + } + }); + + return out; + } + + +private: + BindingType mKind; + + // this can be a Function or a Forward that will become a function + Type* mFunctionToBind; + + std::string mNamedFieldToAccess; + + int mIndexedFieldToAccess; +}; + + +class ClosureVariableBinding { +public: + ClosureVariableBinding() {} + + ClosureVariableBinding(const std::vector& steps) : + mSteps(new std::vector(steps)) + {} + + ClosureVariableBinding(const std::vector& steps, ClosureVariableBindingStep step) : + mSteps(new std::vector(steps)) + { + mSteps->push_back(step); + } + + ClosureVariableBinding(const ClosureVariableBinding& other) : mSteps(other.mSteps) + {} + + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + v.visitHash(ShaHash(mSteps->size())); + + for (auto step: *mSteps) { + step._visitCompilerVisibleInternals(v); + } + } + + ClosureVariableBinding& operator=(const ClosureVariableBinding& other) { + mSteps = other.mSteps; + return *this; + } + + ClosureVariableBinding operator+(ClosureVariableBindingStep step) { + if (mSteps) { + return ClosureVariableBinding(*mSteps, step); + } + + return ClosureVariableBinding(std::vector(), step); + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + for (long stepIx = 0; stepIx < size(); stepIx++) { + (*this)[stepIx].serialize(context, buffer, stepIx); + } + buffer.writeEndCompound(); + } + + template + static ClosureVariableBinding deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + std::vector steps; + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + steps.push_back(ClosureVariableBindingStep::deserialize(context, buffer, wireType)); + }); + + return ClosureVariableBinding(steps); + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (!mSteps) { + return; + } + + for (auto& step: *mSteps) { + step._visitReferencedTypes(visitor); + } + } + + ClosureVariableBinding withShiftedFrontBinding(long amount) const { + if (!size()) { + throw std::runtime_error("Empty Binding can't be shifted."); + } + + if (!(*this)[0].isIndexedField()) { + throw std::runtime_error("Shifting the first binding only makes sense if it's an indexed lookup"); + } + + std::vector steps; + steps.push_back(ClosureVariableBindingStep((*this)[0].getIndexedField() + amount)); + + for (long k = 1; k < size(); k++) { + steps.push_back((*this)[k]); + } + + return ClosureVariableBinding(steps); + } + + size_t size() const { + if (mSteps) { + return mSteps->size(); + } + + return 0; + } + + bool operator<(const ClosureVariableBinding& other) const { + if (size() < other.size()) { + return true; + } + if (size() > other.size()) { + return false; + } + if (!size()) { + return false; + } + + return *mSteps < *other.mSteps; + } + + ClosureVariableBindingStep operator[](int i) const { + if (i < 0 || i >= size()) { + throw std::runtime_error("ClosureVariableBinding index out of bounds"); + } + + return (*mSteps)[i]; + } + + Instance extractValueOrContainingClosure(Type* closureType, instance_ptr data); + +private: + std::shared_ptr > mSteps; +}; + + +inline ClosureVariableBinding operator+(const ClosureVariableBindingStep& step, const ClosureVariableBinding& binding) { + std::vector steps; + steps.push_back(step); + for (long k = 0; k < binding.size(); k++) { + steps.push_back(binding[k]); + } + return ClosureVariableBinding(steps); +} diff --git a/typed_python/CompiledSpecialization.hpp b/typed_python/CompiledSpecialization.hpp new file mode 100644 index 000000000..560e00a70 --- /dev/null +++ b/typed_python/CompiledSpecialization.hpp @@ -0,0 +1,58 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + + +#include "Type.hpp" +#include + +class CompiledSpecialization { +public: + CompiledSpecialization( + compiled_code_entrypoint funcPtr, + Type* returnType, + const std::vector& argTypes + ) : + mFuncPtr(funcPtr), + mReturnType(returnType), + mArgTypes(argTypes) + {} + + compiled_code_entrypoint getFuncPtr() const { + return mFuncPtr; + } + + Type* getReturnType() const { + return mReturnType; + } + + const std::vector& getArgTypes() const { + return mArgTypes; + } + + bool operator==(const CompiledSpecialization& other) const { + return mFuncPtr == other.mFuncPtr + && mReturnType == other.mReturnType + && mArgTypes == other.mArgTypes + ; + } + +private: + compiled_code_entrypoint mFuncPtr; + Type* mReturnType; + std::vector mArgTypes; +}; diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index 67ae300b1..221899015 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -18,6 +18,7 @@ #include "ShaHash.hpp" #include "SpecialModuleNames.hpp" +#include "VisibilityType.hpp" #include "util.hpp" #include @@ -29,6 +30,7 @@ unique hash for types and functions. ******************************/ + class VisitRecord { public: enum class kind { Hash=0, String=1, Topo=2, NameValuePair=3, Error=4 }; @@ -144,15 +146,21 @@ class LambdaVisitor { const visitor_2& nameVisit, const visitor_3& topoVisitor, const visitor_4& namedVisitor, - const visitor_5& onErr + const visitor_5& onErr, + VisibilityType visType ): mHashVisit(hashVisit), mNameVisit(nameVisit), mTopoVisitor(topoVisitor), mNamedVisitor(namedVisitor), - mOnErr(onErr) + mOnErr(onErr), + mVisType(visType) {} + VisibilityType visibilityType() const { + return mVisType; + } + void visitHash(ShaHash h) const { mHashVisit(h); } @@ -165,6 +173,119 @@ class LambdaVisitor { mTopoVisitor(topo); } + void visitInstance( + Type* objType, + instance_ptr instance + ) const { + visitTopo(TypeOrPyobj(objType)); + + if (objType->isComposite()) { + CompositeType* compType = (CompositeType*)objType; + for (long k = 0; k < compType->getTypes().size(); k++) { + visitInstance( + compType->getTypes()[k], + compType->eltPtr(instance, k) + ); + } + return; + } + + if (objType->isFunction()) { + // visit the function's closure + Function* funcT = (Function*)objType; + visitInstance( + funcT->getClosureType(), + instance + ); + return; + } + + if (objType->isOneOf()) { + OneOfType* o = (OneOfType*)objType; + auto typeAndData = o->unwrap(instance); + visitInstance(typeAndData.first, typeAndData.second); + return; + } + + if (objType->isAlternative()) { + Alternative* a = (Alternative*)objType; + visitHash(ShaHash(a->which(instance))); + auto typeAndData = a->unwrap(instance); + visitInstance(typeAndData.first, typeAndData.second); + return; + } + + if (objType->isConcreteAlternative()) { + ConcreteAlternative* a = (ConcreteAlternative*)objType; + visitInstance(a->getAlternative(), instance); + return; + } + + if (objType->isTupleOf()) { + TupleOfType* a = (TupleOfType*)objType; + size_t count = a->count(instance); + + visitHash(ShaHash(count)); + for (long k = 0; k < count; k++) { + visitInstance(a->getEltType(), a->eltPtr(instance, k)); + } + + return; + } + + if (objType->isConstDict()) { + ConstDictType* a = (ConstDictType*)objType; + size_t count = a->count(instance); + + visitHash(ShaHash(count)); + for (long k = 0; k < count; k++) { + visitInstance(a->keyType(), a->kvPairPtrKey(instance, k)); + visitInstance(a->valueType(), a->kvPairPtrValue(instance, k)); + } + + return; + } + + if (objType->isPyCell()) { + static PyCellType* pct = PyCellType::Make(); + + PyObject* o = pct->getPyObj(instance); + + if (!PyCell_Check(o) || !PyCell_Get(o)) { + return; + } + + visitTopo(PyCell_Get(o)); + return; + } + + if (objType->isBool()) { + visitHash(ShaHash(*(bool*)instance ? 1 : 0)); + return; + } + + if (objType->isRegister()) { + visitHash(ShaHash::SHA1((void*)instance, objType->bytecount())); + return; + } + if (objType->isString()) { + visitHash(ShaHash(((StringType*)objType)->toUtf8String(instance))); + return; + } + if (objType->isBytes()) { + size_t ct = ((BytesType*)objType)->count(instance); + visitHash(ShaHash(ct)); + + if (ct) { + visitHash(ShaHash::SHA1( + ((BytesType*)objType)->eltPtr(instance, 0), + ct + )); + } + return; + } + } + void visitNamedTopo(std::string name, TypeOrPyobj instance) const { mNamedVisitor(name, instance); } @@ -232,6 +353,8 @@ class LambdaVisitor { const visitor_3& mTopoVisitor; const visitor_4& mNamedVisitor; const visitor_5& mOnErr; + + VisibilityType mVisType; }; @@ -265,6 +388,7 @@ class CompilerVisibleObjectVisitor { template void visit( TypeOrPyobj obj, + VisibilityType visibility, const visitor_1& hashVisit, const visitor_2& nameVisit, const visitor_3& topoVisitor, @@ -273,8 +397,9 @@ class CompilerVisibleObjectVisitor { ) { visit( obj, + visibility, LambdaVisitor( - hashVisit, nameVisit, topoVisitor, namedVisitor, onErr + hashVisit, nameVisit, topoVisitor, namedVisitor, onErr, visibility ) ); } @@ -282,16 +407,17 @@ class CompilerVisibleObjectVisitor { template void visit( TypeOrPyobj obj, + VisibilityType visibility, const visitor_type& visitor ) { - std::vector records = recordWalk(obj); + std::vector records = recordWalk(obj, visibility); - auto it = mPastVisits.find(obj); - if (it == mPastVisits.end()) { - mPastVisits[obj] = records; + auto it = mPastVisits[visibility].find(obj); + if (it == mPastVisits[visibility].end()) { + mPastVisits[visibility][obj] = records; } else { if (it->second != records) { - checkForInstability(); + checkForInstability(visibility); throw std::runtime_error( "Found unstable object, but somehow our instability check" @@ -300,24 +426,25 @@ class CompilerVisibleObjectVisitor { } } - walk(obj, visitor); + walk(obj, visibility, visitor); } - static std::string recordWalkAsString(TypeOrPyobj obj) { + static std::string recordWalkAsString(TypeOrPyobj obj, VisibilityType visibility) { std::ostringstream s; - for (auto& record: recordWalk(obj)) { + for (auto& record: recordWalk(obj, visibility)) { s << record.toString() << "\n"; } return s.str(); } - static std::vector recordWalk(TypeOrPyobj obj) { + static std::vector recordWalk(TypeOrPyobj obj, VisibilityType visibility) { std::vector records; walk( obj, + visibility, [&](ShaHash h) { records.push_back(VisitRecord(h)); }, [&](std::string h) { records.push_back(VisitRecord(h)); }, [&](TypeOrPyobj o) { records.push_back(VisitRecord(o)); }, @@ -332,11 +459,11 @@ class CompilerVisibleObjectVisitor { mPastVisits.clear(); } - void checkForInstability() { + void checkForInstability(VisibilityType visibility) { std::vector unstable; - for (auto it = mPastVisits.begin(); it != mPastVisits.end(); ++it) { - if (it->second != recordWalk(it->first)) { + for (auto it = mPastVisits[visibility].begin(); it != mPastVisits[visibility].end(); ++it) { + if (it->second != recordWalk(it->first, visibility)) { unstable.push_back(it->first); } } @@ -352,8 +479,8 @@ class CompilerVisibleObjectVisitor { for (long k = 0; k < unstable.size() && k < 1000; k++) { s << k << " -> " << unstable[k].name() << "\n"; - std::vector linesLeft = stringifyVisitRecord(recordWalk(unstable[k])); - std::vector linesRight = stringifyVisitRecord(mPastVisits[unstable[k]]); + std::vector linesLeft = stringifyVisitRecord(recordWalk(unstable[k], visibility)); + std::vector linesRight = stringifyVisitRecord(mPastVisits[visibility][unstable[k]]); auto pad = [&](std::string s, int ct) { if (s.size() > ct) { @@ -397,6 +524,37 @@ class CompilerVisibleObjectVisitor { return lines; } + // check if this is a module's dict object and if that module is globally named. + static std::string isPyObjectGloballyIdentifiableModuleDict(PyObject* h) { + PyEnsureGilAcquired getTheGil; + + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + if (!PyDict_Check(h)) { + return std::string(); + } + + PyObject* package = PyDict_GetItemString(h, "__package__"); + PyObject* name = PyDict_GetItemString(h, "__name__"); + + if (package && name && PyUnicode_Check(name)) { + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, name)); + + if (!moduleObject) { + PyErr_Clear(); + return std::string(); + } + + PyObjectStealer dictObj(PyObject_GenericGetDict((PyObject*)moduleObject, nullptr)); + + if ((PyObject*)dictObj == h) { + return std::string(PyUnicode_AsUTF8(name)); + } + } + + return std::string(); + } + // is this a 'globally identifiable' py object, where we can just use its name to find it, // and where we are guaranteed that it won't change between invocations of the program? static bool isPyObjectGloballyIdentifiableAndStable(PyObject* h) { @@ -510,6 +668,7 @@ class CompilerVisibleObjectVisitor { template static void walk( TypeOrPyobj obj, + VisibilityType visibility, const visitor_1& hashVisit, const visitor_2& nameVisit, const visitor_3& topoVisitor, @@ -517,48 +676,17 @@ class CompilerVisibleObjectVisitor { const visitor_5& onErr ) { walk(obj, + visibility, LambdaVisitor( - hashVisit, nameVisit, topoVisitor, namedVisitor, onErr + hashVisit, nameVisit, topoVisitor, namedVisitor, onErr, visibility ) ); } - template - static void walkInstance( - Type* objType, - instance_ptr instance, - const visitor_type& visitor - ) { - if (objType->isComposite()) { - CompositeType* compType = (CompositeType*)objType; - for (long k = 0; k < compType->getTypes().size(); k++) { - walkInstance( - compType->getTypes()[k], - compType->eltPtr(instance, k), - visitor - ); - } - return; - } - - if (objType->isPyCell()) { - static PyCellType* pct = PyCellType::Make(); - - PyObject* o = pct->getPyObj(instance); - - if (!PyCell_Check(o) || !PyCell_Get(o)) { - return; - } - - visitor.visitTopo(PyCell_Get(o)); - return; - } - } - - template static void walk( TypeOrPyobj obj, + VisibilityType visibility, const visitor_type& visitor ) { PyEnsureGilAcquired getTheGil; @@ -602,6 +730,7 @@ class CompilerVisibleObjectVisitor { // don't visit into constants if (isSimpleConstant(obj.pyobj())) { + //TODO: this just looks wrong return; } @@ -609,19 +738,10 @@ class CompilerVisibleObjectVisitor { if (argType) { visitor.visitHash(ShaHash(2)); visitor.visitTopo(argType); - - if (argType->isFunction()) { - // visit the function's closure - Function* funcT = (Function*)argType; - instance_ptr dataPtr = ((PyInstance*)obj.pyobj())->dataPtr(); - - walkInstance( - funcT->getClosureType(), - dataPtr, - visitor - ); - } - + visitor.visitInstance( + argType, + ((PyInstance*)obj.pyobj())->dataPtr() + ); return; } @@ -704,6 +824,7 @@ class CompilerVisibleObjectVisitor { # else visitor.visitTopo(co->co_lnotab); # endif + return; } @@ -734,18 +855,34 @@ class CompilerVisibleObjectVisitor { visitor.visitHash(ShaHash(1)); if (f->func_globals && PyDict_Check(f->func_globals)) { + bool shouldVisit = true; + + if (visibility == VisibilityType::Identity) { + std::string name = isPyObjectGloballyIdentifiableModuleDict( + f->func_globals + ); + if (name.size()) { + visitor.visitHash(ShaHash(2)); + visitor.visitName(name); + shouldVisit = false; + } + } - std::vector > dotAccesses; + if (shouldVisit) { + visitor.visitHash(ShaHash(2)); - Function::Overload::visitCompilerVisibleGlobals( - [&](std::string name, PyObject* val) { - if (!isSpecialIgnorableName(name)) { - visitor.visitNamedTopo(name, val); - } - }, - (PyCodeObject*)f->func_code, - f->func_globals - ); + std::vector > dotAccesses; + + FunctionOverload::visitCompilerVisibleGlobals( + [&](std::string name, PyObject* val) { + if (!isSpecialIgnorableName(name)) { + visitor.visitNamedTopo(name, val); + } + }, + (PyCodeObject*)f->func_code, + f->func_globals + ); + } } visitor.visitHash(ShaHash(0)); @@ -854,5 +991,8 @@ class CompilerVisibleObjectVisitor { visitor.visitTopo((PyObject*)obj.pyobj()->ob_type); } - std::unordered_map > mPastVisits; + std::map< + VisibilityType, + std::unordered_map > + > mPastVisits; }; diff --git a/typed_python/CompositeType.cpp b/typed_python/CompositeType.cpp index 0c16b901e..5f6144de9 100644 --- a/typed_python/CompositeType.cpp +++ b/typed_python/CompositeType.cpp @@ -16,61 +16,6 @@ #include "AllTypes.hpp" -bool CompositeType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - CompositeType* otherO = (CompositeType*)other; - - if (m_types.size() != otherO->m_types.size()) { - return false; - } - - for (long k = 0; k < m_types.size(); k++) { - if (!m_types[k]->isBinaryCompatibleWith(otherO->m_types[k])) { - return false; - } - } - - return true; -} - -bool CompositeType::_updateAfterForwardTypesChanged() { - bool is_default_constructible = true; - size_t size = 0; - - m_byte_offsets.clear(); - - for (auto t: m_types) { - m_byte_offsets.push_back(size); - size += t->bytecount(); - } - - for (auto t: m_types) { - if (!t->is_default_constructible()) { - is_default_constructible = false; - } - } - - m_serialize_typecodes.clear(); - m_serialize_typecodes_to_position.clear(); - for (int i = 0; i < m_types.size(); i++) { - m_serialize_typecodes.push_back(i); - m_serialize_typecodes_to_position[i] = i; - } - - bool anyChanged = ( - m_is_default_constructible != is_default_constructible || - size != m_size - ); - - m_size = size; - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - bool CompositeType::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (pyComparisonOp == Py_NE) { return !cmp(left, right, Py_EQ, suppressExceptions); @@ -159,53 +104,34 @@ void CompositeType::assign(instance_ptr self, instance_ptr other) { } } -bool NamedTuple::_updateAfterForwardTypesChanged() { - bool anyChanged = ((CompositeType*)this)->_updateAfterForwardTypesChanged(); +std::string NamedTuple::computeRecursiveNameConcrete(TypeStack& typeStack) { + std::string name = "NamedTuple("; - std::string oldName = m_name; - - if (m_is_recursive_forward) { - m_name = m_recursive_name; - m_stripped_name = ""; - } else { - m_name = "NamedTuple("; - m_stripped_name = ""; + if (m_types.size() != m_names.size()) { + throw std::logic_error("Names mismatched with types!"); + } - if (m_types.size() != m_names.size()) { - throw std::logic_error("Names mismatched with types!"); + for (long k = 0; k < m_types.size();k++) { + if (k) { + name += ", "; } - - for (long k = 0; k < m_types.size();k++) { - if (k) { - m_name += ", "; - } - m_name += m_names[k] + "=" + m_types[k]->name(true); - } - m_name += ")"; + name += m_names[k] + "=" + m_types[k]->computeRecursiveName(typeStack); } + name += ")"; - return anyChanged || (oldName != m_name); + return name; } -bool Tuple::_updateAfterForwardTypesChanged() { - bool anyChanged = ((CompositeType*)this)->_updateAfterForwardTypesChanged(); - - std::string oldName = m_name; +std::string Tuple::computeRecursiveNameConcrete(TypeStack& typeStack) { + std::string name = "Tuple("; - if (m_is_recursive_forward) { - m_name = m_recursive_name; - m_stripped_name = ""; - } else { - m_name = "Tuple("; - for (long k = 0; k < m_types.size();k++) { - if (k) { - m_name += ", "; - } - m_name += m_types[k]->name(true); + for (long k = 0; k < m_types.size();k++) { + if (k) { + name += ", "; } - m_name += ")"; - m_stripped_name = ""; + name += m_types[k]->computeRecursiveName(typeStack); } + name += ")"; - return anyChanged || m_name != oldName; + return name; } diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index ac635620e..1733baabb 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -22,6 +22,13 @@ class CompositeType : public Type { public: + // construct a non-forward uninitialized type + CompositeType(TypeCategory in_typeCategory) : + Type(in_typeCategory) + { + } + + // construct a forward-defined CompositeType CompositeType( TypeCategory in_typeCategory, const std::vector& types, @@ -31,11 +38,13 @@ class CompositeType : public Type { m_types(types), m_names(names) { + m_is_forward_defined = true; + for (long k = 0; k < names.size(); k++) { m_nameToIndex[names[k]] = k; } - endOfConstructorInitialization(); // finish initializing the type object. + recomputeName(); } template @@ -52,8 +61,6 @@ class CompositeType : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { for (auto& typePtr: m_types) { @@ -66,8 +73,6 @@ class CompositeType : public Type { _visitContainedTypes(visitor); } - bool _updateAfterForwardTypesChanged(); - instance_ptr eltPtr(instance_ptr self, int64_t ix) const { return self + m_byte_offsets[ix]; } @@ -228,26 +233,84 @@ class CompositeType : public Type { return m_nameToIndex; } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_types = ((CompositeType*)forwardDefinitionOfSelf)->m_types; + m_names = ((CompositeType*)forwardDefinitionOfSelf)->m_names; + m_nameToIndex = ((CompositeType*)forwardDefinitionOfSelf)->m_nameToIndex; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + for (long k = 0; k < m_types.size(); k++) { + auto it = groupMap.find(m_types[k]); + if (it != groupMap.end()) { + m_types[k] = it->second; + } + } + } + + void postInitializeConcrete() { + bool is_default_constructible = true; + size_t size = 0; + + m_byte_offsets.clear(); + + for (long k = 0; k < m_names.size(); k++) { + m_nameToIndex[m_names[k]] = k; + } + + for (auto t: m_types) { + m_byte_offsets.push_back(size); + size += t->bytecount(); + } + + for (auto t: m_types) { + if (!t->is_default_constructible()) { + is_default_constructible = false; + } + } + + m_serialize_typecodes.clear(); + m_serialize_typecodes_to_position.clear(); + for (int i = 0; i < m_types.size(); i++) { + m_serialize_typecodes.push_back(i); + m_serialize_typecodes_to_position[i] = i; + } + + m_size = size; + m_is_default_constructible = is_default_constructible; + } + protected: template - static subtype* MakeSubtype(const std::vector& types, const std::vector& names, subtype* knownType = nullptr) { + static subtype* MakeSubtype(const std::vector& types, const std::vector& names) { + bool anyForward = false; + + for (auto t: types) { + if (t->isForwardDefined()) { + anyForward = true; + } + } + + if (anyForward) { + return new subtype(types, names); + } + PyEnsureGilAcquired getTheGil; typedef std::pair, const std::vector > keytype; - static std::map m; + static std::map memo; - auto it = m.find(keytype(types, names)); - if (it == m.end()) { - it = m.insert( - std::make_pair( - keytype(types, names), - knownType ? knownType : new subtype(types, names) - ) - ).first; + auto it = memo.find(keytype(types, names)); + if (it != memo.end()) { + return it->second; } - return it->second; + subtype* res = new subtype(types, names); + subtype* concrete = (subtype*)res->forwardResolvesTo(); + + memo[keytype(types, names)] = concrete; + return concrete; } std::vector m_types; @@ -263,32 +326,45 @@ PyDoc_STRVAR(NamedTuple_doc, "NamedTuple(kw)(t) -> new typed named tuple with names and types from kw, initialized from tuple t\n" "\n" "Raises TypeError if types don't match.\n" - ); +); class NamedTuple : public CompositeType { public: + // construct a non-forward NamedTuple + NamedTuple() : CompositeType(TypeCategory::catNamedTuple) + { + } + NamedTuple(const std::vector& types, const std::vector& names) : CompositeType(TypeCategory::catNamedTuple, types, names) { assert(types.size() == names.size()); - - m_doc = NamedTuple_doc; - - endOfConstructorInitialization(); // finish initializing the type object. } - bool _updateAfterForwardTypesChanged(); + void initializeDuringDeserialization( + const std::vector& types, + const std::vector& names + ) { + m_names = names; + m_types = types; + } - void _updateTypeMemosAfterForwardResolution() { - NamedTuple::Make(m_types, m_names, this); + const char* docConcrete() { + return NamedTuple_doc; } - static NamedTuple* Make(const std::vector& types, const std::vector& names, NamedTuple* knownType = nullptr) { + static NamedTuple* Make(const std::vector& types, const std::vector& names) { if (names.size() != types.size()) { throw std::runtime_error("Names mismatched with types!"); } - return MakeSubtype(types, names, knownType); + return MakeSubtype(types, names); + } + + Type* cloneForForwardResolutionConcrete() { + return new NamedTuple(); } + + std::string computeRecursiveNameConcrete(TypeStack& typeStack); }; PyDoc_STRVAR(Tuple_doc, @@ -296,24 +372,37 @@ PyDoc_STRVAR(Tuple_doc, "Tuple(T1, T2, ...)(t) -> new typed tuple with types T1, T2, ..., initialized from tuple t\n" "\n" "Raises TypeError if types don't match.\n" - ); +); class Tuple : public CompositeType { public: + // construct a non-forward Tuple + Tuple() : CompositeType(TypeCategory::catTuple) + { + } + Tuple(const std::vector& types, const std::vector& names) : CompositeType(TypeCategory::catTuple, types, names) { - m_doc = Tuple_doc; - endOfConstructorInitialization(); // finish initializing the type object. } - bool _updateAfterForwardTypesChanged(); + void initializeDuringDeserialization( + const std::vector& types + ) { + m_types = types; + } + + const char* docConcrete() { + return Tuple_doc; + } - void _updateTypeMemosAfterForwardResolution() { - Tuple::Make(m_types, this); + static Tuple* Make(const std::vector& types) { + return MakeSubtype(types, std::vector()); } - static Tuple* Make(const std::vector& types, Tuple* knownType=nullptr) { - return MakeSubtype(types, std::vector(), knownType); + Type* cloneForForwardResolutionConcrete() { + return new Tuple(); } + + std::string computeRecursiveNameConcrete(TypeStack& typeStack); }; diff --git a/typed_python/ConcreteAlternativeType.cpp b/typed_python/ConcreteAlternativeType.cpp index 0cf8348f9..8560ecb9f 100644 --- a/typed_python/ConcreteAlternativeType.cpp +++ b/typed_python/ConcreteAlternativeType.cpp @@ -17,50 +17,6 @@ #include "AllTypes.hpp" #include "../typed_python/Format.hpp" -bool ConcreteAlternative::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() == TypeCategory::catConcreteAlternative) { - ConcreteAlternative* otherO = (ConcreteAlternative*)other; - - return otherO->m_alternative->isBinaryCompatibleWith(m_alternative) && - m_which == otherO->m_which; - } - - if (other->getTypeCategory() == TypeCategory::catAlternative) { - return m_alternative->isBinaryCompatibleWith(other); - } - - return false; -} - -bool ConcreteAlternative::_updateAfterForwardTypesChanged() { - if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { - throw std::runtime_error( - "invalid alternative index: " + - format(m_which) + " not in [0," + - format(m_alternative->subtypes().size()) + ")" - ); - } - - m_base = m_alternative; - - std::string name = m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; - size_t size = m_alternative->bytecount(); - bool is_default_constructible = m_alternative->subtypes()[m_which].second->is_default_constructible(); - - bool anyChanged = ( - m_name != name || - m_size != size || - m_is_default_constructible != is_default_constructible - ); - - m_name = name; - m_stripped_name = ""; - m_size = size; - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - void ConcreteAlternative::constructor(instance_ptr self) { assertForwardsResolvedSufficientlyToInstantiate(); @@ -72,33 +28,3 @@ void ConcreteAlternative::constructor(instance_ptr self) { }); } } - -// static -ConcreteAlternative* ConcreteAlternative::Make(Alternative* alt, int64_t which, ConcreteAlternative* knownType) { - PyEnsureGilAcquired getTheGil; - - typedef std::pair keytype; - - static std::map m; - - auto it = m.find(keytype(alt ,which)); - - if (it == m.end()) { - if (which < 0 || which >= alt->subtypes().size()) { - throw std::runtime_error( - "invalid alternative index: " + - format(which) + " not in [0," + - format(alt->subtypes().size()) + ")" - ); - } - - it = m.insert( - std::make_pair( - keytype(alt,which), - knownType ? knownType : new ConcreteAlternative(alt,which) - ) - ).first; - } - - return it->second; -} diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index c15cac2f9..10ca91da8 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -23,24 +23,75 @@ class ConcreteAlternative : public Type { public: typedef Alternative::layout layout; + ConcreteAlternative() : + Type(TypeCategory::catConcreteAlternative), + m_which(0), + m_alternative(nullptr) + { + } + ConcreteAlternative(Alternative* m_alternative, int64_t which) : Type(TypeCategory::catConcreteAlternative), m_alternative(m_alternative), m_which(which) { - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; + m_name = m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; + m_module_name = m_alternative->moduleName(); } - std::string nameWithModuleConcrete() { - if (m_alternative->moduleName().size() == 0) { - return m_name; + void initializeDuringDeserialization(int64_t which, Alternative* base) { + m_alternative = base; + m_base = base; + m_which = which; + } + + const char* docConcrete() { + return Alternative_doc; + } + + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { + throw std::runtime_error( + "invalid alternative index: " + + format(m_which) + " not in [0," + + format(m_alternative->subtypes().size()) + ")" + ); } - return m_alternative->moduleName() + "." + m_name; + return m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_base = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_base; + m_alternative = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_alternative; + m_which = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_which; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_alternative, groupMap); + updateTypeRefFromGroupMap(m_base, groupMap); } - std::string moduleNameConcrete() { - return m_alternative->moduleName(); + Type* cloneForForwardResolutionConcrete() { + return new ConcreteAlternative(); + } + + void postInitializeConcrete() { + if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { + throw std::runtime_error( + "invalid alternative index: " + + format(m_which) + " not in [0," + + format(m_alternative->subtypes().size()) + ")" + ); + } + + m_base = m_alternative; + size_t size = m_alternative->bytecount(); + bool is_default_constructible = m_alternative->subtypes()[m_which].second->is_default_constructible(); + + m_size = size; + m_is_default_constructible = is_default_constructible; } void deepcopyConcrete( @@ -55,12 +106,6 @@ class ConcreteAlternative : public Type { return m_alternative->deepBytecount(instance, alreadyVisited, outSlabs); } - void _updateTypeMemosAfterForwardResolution() { - ConcreteAlternative::Make(m_alternative, m_which, this); - } - - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { Type* t = m_alternative; @@ -82,8 +127,6 @@ class ConcreteAlternative : public Type { v.visitTopo(m_alternative); } - bool _updateAfterForwardTypesChanged(); - typed_python_hash_type hash(instance_ptr left) { return m_alternative->hash(left); } @@ -152,8 +195,6 @@ class ConcreteAlternative : public Type { return m_alternative->eltPtr(self); } - static ConcreteAlternative* Make(Alternative* alt, int64_t which, ConcreteAlternative* knownType=nullptr); - Type* elementType() const { return m_alternative->subtypes()[m_which].second; } diff --git a/typed_python/ConstDictType.cpp b/typed_python/ConstDictType.cpp index 4f3773ca1..57ebe4a08 100644 --- a/typed_python/ConstDictType.cpp +++ b/typed_python/ConstDictType.cpp @@ -16,62 +16,29 @@ #include "AllTypes.hpp" -bool ConstDictType::_updateAfterForwardTypesChanged() { - size_t old_bytes_per_key_value_pair = m_bytes_per_key_value_pair; - - m_size = sizeof(void*); - m_is_default_constructible = true; - m_bytes_per_key = m_key->bytecount(); - m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); - m_bytes_per_key_subtree_pair = m_key->bytecount() + this->bytecount(); - - std::string name = "ConstDict(" + m_key->name(true) + "->" + m_value->name(true) + ")"; - - if (m_is_recursive_forward) { - name = m_recursive_name; +// static +ConstDictType* ConstDictType::Make(Type* key, Type* value) { + if (key->isForwardDefined() || value->isForwardDefined()) { + return new ConstDictType(key, value); } - bool anyChanged = ( - name != m_name || - m_bytes_per_key_value_pair != old_bytes_per_key_value_pair - ); + PyEnsureGilAcquired getTheGil; - m_name = name; - m_stripped_name = ""; + static std::map, ConstDictType*> memo; - return anyChanged; -} + auto lookup_key = std::make_pair(key, value); -bool ConstDictType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; + auto it = memo.find(lookup_key); + if (it != memo.end()) { + return it->second; } - ConstDictType* otherO = (ConstDictType*)other; - - return m_key->isBinaryCompatibleWith(otherO->m_key) && - m_value->isBinaryCompatibleWith(otherO->m_value); -} - -// static -ConstDictType* ConstDictType::Make(Type* key, Type* value, ConstDictType* knownType) { - PyEnsureGilAcquired getTheGil; + ConstDictType* res = new ConstDictType(key, value); + ConstDictType* concrete = (ConstDictType*)res->forwardResolvesTo(); - static std::map, ConstDictType*> m; - - auto lookup_key = std::make_pair(key,value); - - auto it = m.find(lookup_key); - if (it == m.end()) { - it = m.insert( - std::make_pair( - lookup_key, - knownType ? knownType : new ConstDictType(key, value) - ) - ).first; - } + memo[lookup_key] = concrete; - return it->second; + return concrete; } void ConstDictType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 0b377f440..c186de523 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -40,13 +40,26 @@ class ConstDictType : public Type { typedef layout* layout_ptr; + ConstDictType() : Type(TypeCategory::catConstDict) + { + } + ConstDictType(Type* key, Type* value) : Type(TypeCategory::catConstDict), m_key(key), m_value(value) { - m_doc = ConstDictType_doc; - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; + recomputeName(); + } + + void initializeDuringDeserialization(Type* keyType, Type* valType) { + m_key = keyType; + m_value = valType; + } + + const char* docConcrete() { + return ConstDictType_doc; } template @@ -66,16 +79,46 @@ class ConstDictType : public Type { v.visitTopo(m_value); } - bool _updateAfterForwardTypesChanged(); + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "ConstDict(" + + m_key->computeRecursiveName(typeStack) + + ", " + + m_value->computeRecursiveName(typeStack) + + ")"; + } - bool isBinaryCompatibleWithConcrete(Type* other); + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_key = ((ConstDictType*)forwardDefinitionOfSelf)->m_key; + m_value = ((ConstDictType*)forwardDefinitionOfSelf)->m_value; + } - void _updateTypeMemosAfterForwardResolution() { - ConstDictType::Make(m_key, m_value, this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it_key = groupMap.find(m_key); + if (it_key != groupMap.end()) { + m_key = it_key->second; + } + + auto it_val = groupMap.find(m_value); + if (it_val != groupMap.end()) { + m_value = it_val->second; + } + } + + Type* cloneForForwardResolutionConcrete() { + return new ConstDictType(); + } + + void postInitializeConcrete() { + m_size = sizeof(void*); + m_is_default_constructible = true; } - static ConstDictType* Make(Type* key, Type* value, ConstDictType* knownType = nullptr); + void finalizeTypeConcrete() { + m_bytes_per_key = m_key->bytecount(); + m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); + } + static ConstDictType* Make(Type* key, Type* value); // hand 'visitor' each an instance_ptr for // each value. if it returns 'false', exit early. diff --git a/typed_python/DictType.cpp b/typed_python/DictType.cpp index b7591433b..e9b056c13 100644 --- a/typed_python/DictType.cpp +++ b/typed_python/DictType.cpp @@ -16,56 +16,29 @@ #include "AllTypes.hpp" -bool DictType::_updateAfterForwardTypesChanged() { - m_size = sizeof(void*); - m_is_default_constructible = true; - m_bytes_per_key = m_key->bytecount(); - m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); - - std::string name = "Dict(" + m_key->name(true) + "->" + m_value->name(true) + ")"; - - if (m_is_recursive_forward) { - name = m_recursive_name; +// static +DictType* DictType::Make(Type* key, Type* value) { + if (key->isForwardDefined() || value->isForwardDefined()) { + return new DictType(key, value); } - bool anyChanged = name != m_name; + PyEnsureGilAcquired getTheGil; - m_name = name; - m_stripped_name = ""; + static std::map, DictType*> memo; - return anyChanged; -} + auto lookup_key = std::make_pair(key, value); -bool DictType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; + auto it = memo.find(lookup_key); + if (it != memo.end()) { + return it->second; } - DictType* otherO = (DictType*)other; - - return m_key->isBinaryCompatibleWith(otherO->m_key) && - m_value->isBinaryCompatibleWith(otherO->m_value); -} + DictType* res = new DictType(key, value); + DictType* concrete = (DictType*)res->forwardResolvesTo(); -// static -DictType* DictType::Make(Type* key, Type* value, DictType* knownType) { - PyEnsureGilAcquired getTheGil; - - static std::map, DictType*> m; - - auto lookup_key = std::make_pair(key,value); - - auto it = m.find(lookup_key); - if (it == m.end()) { - it = m.insert( - std::make_pair( - lookup_key, - knownType ? knownType: new DictType(key, value) - ) - ).first; - } + memo[lookup_key] = concrete; - return it->second; + return concrete; } void DictType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index d6166c73b..ac757a90b 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -31,17 +31,28 @@ PyDoc_STRVAR(DictType_doc, class DictType : public Type { public: + // construct a non-forward uninitialized dict + DictType() : Type(TypeCategory::catDict) + { + } + + // declare a forward-defined Dict DictType(Type* key, Type* value) : Type(TypeCategory::catDict), m_key(key), m_value(value) { - m_doc = DictType_doc; - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; + recomputeName(); + } + + void initializeDuringDeserialization(Type* keyType, Type* valType) { + m_key = keyType; + m_value = valType; } - void _updateTypeMemosAfterForwardResolution() { - DictType::Make(m_key, m_value, this); + const char* docConcrete() { + return DictType_doc; } template @@ -61,11 +72,46 @@ class DictType : public Type { v.visitTopo(m_value); } - bool _updateAfterForwardTypesChanged(); + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "Dict(" + + m_key->computeRecursiveName(typeStack) + + ", " + + m_value->computeRecursiveName(typeStack) + + ")"; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_key = ((DictType*)forwardDefinitionOfSelf)->m_key; + m_value = ((DictType*)forwardDefinitionOfSelf)->m_value; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it_key = groupMap.find(m_key); + if (it_key != groupMap.end()) { + m_key = it_key->second; + } - bool isBinaryCompatibleWithConcrete(Type* other); + auto it_val = groupMap.find(m_value); + if (it_val != groupMap.end()) { + m_value = it_val->second; + } + } + + Type* cloneForForwardResolutionConcrete() { + return new DictType(); + } + + void postInitializeConcrete() { + m_size = sizeof(void*); + m_is_default_constructible = true; + } + + void finalizeTypeConcrete() { + m_bytes_per_key = m_key->bytecount(); + m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); + } - static DictType* Make(Type* key, Type* value, DictType* knownType=nullptr); + static DictType* Make(Type* key, Type* value); // hand 'visitor' each an instance_ptr for // each value. if it returns 'false', exit early. diff --git a/typed_python/EmbeddedMessageType.hpp b/typed_python/EmbeddedMessageType.hpp index c8686de78..6aec9c629 100644 --- a/typed_python/EmbeddedMessageType.hpp +++ b/typed_python/EmbeddedMessageType.hpp @@ -20,6 +20,17 @@ #include "NullSerializationContext.hpp" #include "SerializationBuffer.hpp" +PyDoc_STRVAR(EmbeddedMessage_doc, + "EmbeddedMessage: represents an embedded message in a serialization graph.\n\n" + "If you have a type T in a type graph and you replace it with EmbeddedMessage and \n" + "deserialize an instance of that type, instead of the actual instance you'll get a\n" + "block of bytes representing the embedded type. This can be used in some circumstances\n" + "to build tooling that can handle messages without knowing their full type. It cannot\n" + "be used in contexts where there may be embedded memos within the subgraph, so use with\n" + "care. This will probably get deprecated at some point.\n" +); + + class EmbeddedMessageType : public BytesType { public: EmbeddedMessageType() @@ -30,6 +41,10 @@ class EmbeddedMessageType : public BytesType { m_typeCategory = TypeCategory::catEmbeddedMessage; } + const char* docConcrete() { + return EmbeddedMessage_doc; + } + static EmbeddedMessageType* Make() { static EmbeddedMessageType* res = new EmbeddedMessageType(); return res; @@ -61,5 +76,6 @@ class EmbeddedMessageType : public BytesType { constructor((instance_ptr)self, outBuffer.size(), (const char*)outBuffer.buffer()); } -}; + void postInitializeConcrete() {} +}; diff --git a/typed_python/ForwardType.cpp b/typed_python/ForwardType.cpp new file mode 100644 index 000000000..df4291fb6 --- /dev/null +++ b/typed_python/ForwardType.cpp @@ -0,0 +1,172 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyInstance.hpp" +#include "ForwardType.hpp" + +/* static */ +Forward* Forward::MakeFromFunction(PyObject* funcObj) { + if (!PyFunction_Check(funcObj)) { + throw std::runtime_error( + "Forwards can only be made from functions which return a single variable lookup." + ); + } + + PyCodeObject* code = (PyCodeObject*)PyFunction_GetCode(funcObj); + PyObject* closure = PyFunction_GetClosure(funcObj); + + if (!PyTuple_Check(code->co_names)) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names is not a tuple"); + } + + if (closure) { + PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); + + if (PyTuple_Size(code->co_names)) { + throw std::runtime_error("Corrupt Forward lookup function: can't have more than one name"); + } + + if (!PyTuple_Check(coFreevars)) { + throw std::runtime_error("Corrupt Forward lookup function: co_freevars was not a tuple"); + } + + if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { + throw std::runtime_error("Corrupt Forward lookup function: co_freevars not the same size as closure"); + } + + if (PyTuple_Size(coFreevars) != 1) { + throw std::runtime_error("Corrupt Forward lookup function: can't have more than 1 free variable"); + } + + PyObject* varname = PyTuple_GetItem(coFreevars, 0); + if (!PyUnicode_Check(varname)) { + throw std::runtime_error("Corrupt Forward lookup function: closure varnames need to be strings"); + } + + std::string name = PyUnicode_AsUTF8(varname); + + if (!PyCell_Check(PyTuple_GetItem(closure, 0))) { + throw std::runtime_error("Corrupt Forward lookup function: closure was not a cell"); + } + + return new Forward(name, PyTuple_GetItem(closure, 0)); + } + + if (PyTuple_Size(code->co_names) != 1) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); + } + + if (!PyUnicode_Check(PyTuple_GetItem(code->co_names, 0))) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); + } + + std::string name = PyUnicode_AsUTF8(PyTuple_GetItem(code->co_names, 0)); + return new Forward(name, PyFunction_GetGlobals(funcObj)); +} + +Type* Forward::lambdaDefinition() { + if (!mCellOrDict) { + return nullptr; + } + + if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { + Type* res = PyInstance::unwrapTypeArgToTypePtr(PyCell_Get(mCellOrDict)); + + if (res == this) { + return nullptr; + } + + return res; + } + + if (PyDict_Check(mCellOrDict)) { + PyObject* item = PyDict_GetItemString(mCellOrDict, m_name.c_str()); + + if (!item) { + return nullptr; + } + + Type* res = PyInstance::unwrapTypeArgToTypePtr(item); + + if (res == this) { + return nullptr; + } + + return res; + } + + return nullptr; +} + +bool Forward::lambdaDefinitionPopulated() { + return lambdaDefinition(); +} + +void Forward::installDefinitionIfLambda() { + if (!mCellOrDict) { + return; + } + + if (!m_forward_resolves_to) { + return; + } + + if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { + if (PyType_Check(PyCell_Get(mCellOrDict))) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)PyCell_Get(mCellOrDict) + ); + + if (cellContents) { + // see if cellContents resolves + if (cellContents->isResolved() + && cellContents->forwardResolvesTo() + == m_forward_resolves_to) { + PyCell_Set( + mCellOrDict, + (PyObject*)PyInstance::typeObj( + m_forward_resolves_to + ) + ); + } + } + } + + return; + } + + if (PyDict_Check(mCellOrDict)) { + PyObject* curContents = PyDict_GetItemString(mCellOrDict, m_name.c_str()); + + if (curContents && PyType_Check(curContents)) { + Type* curContentsType = PyInstance::extractTypeFrom( + (PyTypeObject*)curContents + ); + + if (curContentsType && curContentsType->isResolved() && + curContentsType->forwardResolvesTo() + == m_forward_resolves_to) { + PyDict_SetItemString( + mCellOrDict, + m_name.c_str(), + (PyObject*)PyInstance::typeObj( + m_forward_resolves_to + ) + ); + } + } + } +} diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 7e861dfdd..124c43fd7 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2019 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,37 +23,23 @@ PyDoc_STRVAR(Forward_doc, "Forward(n) -> new forward type named n\n" "\n" "Forward types must be resolved before any types that contain them can be used.\n" - ); +); // forward types must be resolved (removed from the graph) before // any types that contain them can be used. class Forward : public Type { public: - Forward(std::string name, int index) : + Forward(std::string name, PyObject* cellOrDict=nullptr) : Type(TypeCategory::catForward), mTarget(nullptr), - mIndex(index) + mCellOrDict(incref(cellOrDict)) { m_name = name; - m_doc = Forward_doc; - - // deliberately don't invoke 'endOfConstructorInitialization' + m_is_forward_defined = true; } - std::string moduleNameConcrete() { - if (mTarget) { - return mTarget->moduleNameConcrete(); - } - - return ""; - } - - std::string nameWithModuleConcrete() { - if (mTarget) { - return mTarget->nameWithModule(); - } - - return m_name; + const char* docConcrete() { + return Forward_doc; } static Forward* Make() { @@ -61,106 +47,56 @@ class Forward : public Type { } static Forward* Make(std::string name) { - static std::atomic index; - - int64_t indexVal = index++; - - return new Forward(name, indexVal); + return new Forward(name); } - Type* define(Type* target) { - if (!target) { - throw std::runtime_error("Can't resolve a Forward to the nullptr"); - } - - while (target->getTypeCategory() == TypeCategory::catForward) { - if (((Forward*)target)->getTarget()) { - target = ((Forward*)target)->getTarget(); - } else { - break; - } - } + static Forward* MakeFromFunction(PyObject* funcObj); - if (target == this) { - throw std::runtime_error("Can't resolve a forward to itself!"); + Type* getTargetTransitive() { + if (!mTarget) { + return nullptr; } - if (target == mTarget) { + if (!mTarget->isForward()) { return mTarget; } - bool tgtIsForward = target->getTypeCategory() == TypeCategory::catForward; - - if (!tgtIsForward) { - // check the containment graph. If we are a forward that's - // directly contained by the target, then we shouldn't allow this - // definition because we'd be infinitely large. - auto& containedForwards = target->getContainedForwards(); - - if (containedForwards.find(this) != containedForwards.end()) { - throw std::runtime_error("Can't resolve forward " + m_name + " to " + - target->name() + " because it would create a type-containment cycle" + - " (specifically, the type would have an infinite bytecount because it 'contains' itself)." - ); - } + std::set seen; - bool thisIsRecursive = target->getReferencedForwards().find(this) != target->getReferencedForwards().end(); + Type* curFwd = this; - if (thisIsRecursive) { - target->setNameAndIndexForRecursiveType(m_name, mIndex); - target->_updateAfterForwardTypesChanged(); + while (curFwd->isForward()) { + if (seen.find(curFwd) != seen.end()) { + throw std::runtime_error("Forward cycle detected."); } - } - mTarget = target; - m_resolved = true; + seen.insert(curFwd); - // forward everyone looking at us to the new target - for (auto typePtr: m_referencing_us_indirectly) { - typePtr->forwardResolvedTo(this, target); - } + curFwd = ((Forward*)curFwd)->getTarget(); - bool anyChanged = true; - - while (anyChanged) { - anyChanged = false; - - for (auto typePtr: m_referencing_us_indirectly) { - if (typePtr->_updateAfterForwardTypesChanged()) { - anyChanged = true; - } + if (!curFwd) { + return nullptr; } } - std::set resolvedThisPass; + return curFwd; + } - for (auto typePtr: m_referencing_us_indirectly) { - if (typePtr->getReferencedForwards().size() == 0) { - typePtr->forwardTypesAreResolved(); - resolvedThisPass.insert(typePtr); - } + Type* define(Type* target) { + if (!target) { + throw std::runtime_error("Can't resolve a Forward to the nullptr"); } - // we need to order the nodes in a canonical way, which we do by - // taking the lowest-indexed 'Forward' and walking forward through the - // graph. The remaining nodes will all be non-recursive, and so we can - // visit them last, and in any order we like - - std::map forwardByIndex; - for (auto t: resolvedThisPass) { - int index = t->getRecursiveForwardIndex(); - - if (index >= 0) { - if (forwardByIndex.find(index) != forwardByIndex.end()) { - throw std::runtime_error("Somehow, a forward index got used twice?"); - } + if (mTarget) { + throw std::runtime_error("Forward is already resolved."); + } - forwardByIndex[index] = t; - } + if (target == this) { + asm("int3"); + throw std::runtime_error("Forward can't be resolved to itself."); } - m_name = mTarget->name(); - m_referencing_us_indirectly.clear(); + mTarget = target; return target; } @@ -182,22 +118,40 @@ class Forward : public Type { template void _visitReferencedTypes(const visitor_type& v) { - if (mTarget) + if (mTarget) { v(mTarget); - } - - void markIndirectForwardUse(Type* user) { - if (m_resolved) { - throw std::runtime_error("already resolved forward type " + name() + " can't be used like this."); } - if (user->getTypeCategory() == Type::TypeCategory::catForward) { - throw std::runtime_error("Makes no sense for a forward to be used by another forward"); + if (lambdaDefinitionPopulated()) { + v(lambdaDefinition()); } + } + + bool hasLambdaDefinition() { + return mCellOrDict != nullptr; + } + + PyObject* getCellOrDict() { + return mCellOrDict; + } + + void setName(std::string newName) { + m_name = newName; + } - m_referencing_us_indirectly.insert(user); + void setCellOrDict(PyObject* newCellOrDict) { + if (mCellOrDict) { + decref(mCellOrDict); + } + mCellOrDict = incref(newCellOrDict); } + bool lambdaDefinitionPopulated(); + + Type* lambdaDefinition(); + + void installDefinitionIfLambda(); + void constructor(instance_ptr self) { throw std::runtime_error("Forward types should never be explicity instantiated."); } @@ -224,16 +178,13 @@ class Forward : public Type { throw std::runtime_error("Forward types should never be explicity instantiated."); } - const std::set getReferencing() const { - return m_referencing_us_indirectly; - } + void postInitializeConcrete() {}; + private: Type* mTarget; - // when we're trying to determine how to hash a type graph, - // it's helpful when recursive types know that they were - // defined by a forward and the index of that forward - int64_t mIndex; - - std::set m_referencing_us_indirectly; + // if we are a recursive forward defined by a simple lambda function encoded by name, + // then this will contain either a PyCell from the closure, or a PyDict in which + // we are supposed to look to find a 'name'. + PyObject* mCellOrDict; }; diff --git a/typed_python/FunctionArg.hpp b/typed_python/FunctionArg.hpp new file mode 100644 index 000000000..4ed1e4ec7 --- /dev/null +++ b/typed_python/FunctionArg.hpp @@ -0,0 +1,180 @@ +/****************************************************************************** + Copyright 2017-2022 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Type.hpp" + +class FunctionArg { +public: + FunctionArg() : + m_typeFilter(nullptr), + m_defaultValue(nullptr) + { + } + + FunctionArg(std::string name, Type* typeFilterOrNull, PyObject* defaultValue, bool isStarArg, bool isKwarg) : + m_name(name), + m_typeFilter(typeFilterOrNull), + m_defaultValue(defaultValue), + m_isStarArg(isStarArg), + m_isKwarg(isKwarg) + { + assert(!(isStarArg && isKwarg)); + } + + std::string getName() const { + return m_name; + } + + PyObject* getDefaultValue() const { + return m_defaultValue; + } + + Type* getTypeFilter() const { + return m_typeFilter; + } + + bool getIsStarArg() const { + return m_isStarArg; + } + + bool getIsKwarg() const { + return m_isKwarg; + } + + bool getIsNormalArg() const { + return !m_isKwarg && !m_isStarArg; + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (m_typeFilter) { + visitor(m_typeFilter); + } + } + + bool operator<(const FunctionArg& other) const { + if (m_name < other.m_name) { + return true; + } + if (m_name > other.m_name) { + return false; + } + if (m_typeFilter < other.m_typeFilter) { + return true; + } + if (m_typeFilter > other.m_typeFilter) { + return false; + } + if (m_defaultValue < other.m_defaultValue) { + return true; + } + if (m_defaultValue > other.m_defaultValue) { + return false; + } + if (m_isStarArg < other.m_isStarArg) { + return true; + } + if (m_isStarArg > other.m_isStarArg) { + return false; + } + if (m_isKwarg < other.m_isKwarg) { + return true; + } + if (m_isKwarg > other.m_isKwarg) { + return false; + } + + return false; + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + + buffer.writeStringObject(0, m_name); + if (m_typeFilter) { + context.serializeNativeType(m_typeFilter, buffer, 1); + } + if (m_defaultValue) { + context.serializePythonObject(m_defaultValue, buffer, 2); + } + buffer.writeUnsignedVarintObject(3, m_isStarArg ? 1 : 0); + buffer.writeUnsignedVarintObject(4, m_isKwarg ? 1 : 0); + + buffer.writeEndCompound(); + } + + template + static FunctionArg deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + std::string name; + Type* typeFilterOrNull = nullptr; + PyObjectHolder defaultValue; + bool isStarArg = false; + bool isKwarg = false; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + assertWireTypesEqual(wireType, WireType::BYTES); + name = buffer.readStringObject(); + } + else if (fieldNumber == 1) { + typeFilterOrNull = context.deserializeNativeType(buffer, wireType); + } + else if (fieldNumber == 2) { + defaultValue.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 3) { + assertWireTypesEqual(wireType, WireType::VARINT); + isStarArg = buffer.readUnsignedVarint(); + } + else if (fieldNumber == 4) { + assertWireTypesEqual(wireType, WireType::VARINT); + isKwarg = buffer.readUnsignedVarint(); + } + }); + + return FunctionArg(name, typeFilterOrNull, defaultValue, isStarArg, isKwarg); + } + + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + v.visitName(m_name); + if (m_defaultValue) { + v.visitTopo((PyObject*)m_defaultValue); + } else { + v.visitHash(ShaHash()); + } + + if (m_typeFilter) { + v.visitTopo(m_typeFilter); + } else { + v.visitHash(ShaHash()); + } + + v.visitHash( + ShaHash((m_isStarArg ? 2 : 1) + (m_isKwarg ? 10: 11)) + ); + } + +private: + std::string m_name; + Type* m_typeFilter; + PyObjectHolder m_defaultValue; + bool m_isStarArg; + bool m_isKwarg; +}; diff --git a/typed_python/FunctionCallArgMapping.hpp b/typed_python/FunctionCallArgMapping.hpp index fd8b9662f..49acbbd07 100644 --- a/typed_python/FunctionCallArgMapping.hpp +++ b/typed_python/FunctionCallArgMapping.hpp @@ -31,23 +31,23 @@ class FunctionCallArgMapping { FunctionCallArgMapping& operator=(const FunctionCallArgMapping&) = delete; public: - class FunctionArg { + class LiveFunctionArg { public: - FunctionArg() : + LiveFunctionArg() : mInstance(), mValid(false), mRawPtr(nullptr) { } - FunctionArg(instance_ptr inData) : + LiveFunctionArg(instance_ptr inData) : mInstance(), mValid(true), mRawPtr(inData) { } - FunctionArg(Instance i) : + LiveFunctionArg(Instance i) : mInstance(i), mValid(true), mRawPtr(nullptr) @@ -75,7 +75,7 @@ class FunctionCallArgMapping { }; - FunctionCallArgMapping(const Function::Overload& overload) : + FunctionCallArgMapping(const FunctionOverload& overload) : mArgs(overload.getArgs()), mCurrentArgumentIx(0), mIsValid(true) @@ -375,7 +375,7 @@ class FunctionCallArgMapping { return res; } - FunctionArg extractArgWithType(int argIx, Type* argType) const { + LiveFunctionArg extractArgWithType(int argIx, Type* argType) const { if (mArgs[argIx].getIsNormalArg()) { try { Type* actualType = PyInstance::extractTypeFrom(mSingleValueArgs[argIx]->ob_type); @@ -384,15 +384,15 @@ class FunctionCallArgMapping { PyInstance* argAsPyInstance = ((PyInstance*)mSingleValueArgs[argIx]); if (argAsPyInstance->mTemporaryRefTo) { - return FunctionArg(argAsPyInstance->mTemporaryRefTo); + return LiveFunctionArg(argAsPyInstance->mTemporaryRefTo); } - return FunctionArg( + return LiveFunctionArg( argAsPyInstance->mContainingInstance ); } - return FunctionArg( + return LiveFunctionArg( Instance::createAndInitialize(argType, [&](instance_ptr p) { PyInstance::copyConstructFromPythonInstance( argType, p, mSingleValueArgs[argIx], ConversionLevel::Signature @@ -402,11 +402,11 @@ class FunctionCallArgMapping { } catch(PythonExceptionSet& s) { // failed to convert, but keep going PyErr_Clear(); - return FunctionArg(); + return LiveFunctionArg(); } catch(...) { // not a valid conversion - return FunctionArg(); + return LiveFunctionArg(); } } else if (mArgs[argIx].getIsStarArg()) { if (argType->getTypeCategory() != Type::TypeCategory::catTuple) { @@ -416,11 +416,11 @@ class FunctionCallArgMapping { Tuple* tup = (Tuple*)argType; if (mStarArgValues.size() != tup->getTypes().size()) { - return FunctionArg(); + return LiveFunctionArg(); } try { - return FunctionArg( + return LiveFunctionArg( Instance::createAndInitialize(tup, [&](instance_ptr p) { tup->constructor(p, [&](instance_ptr subElt, int tupArg) { PyInstance::copyConstructFromPythonInstance( @@ -433,11 +433,11 @@ class FunctionCallArgMapping { } catch(PythonExceptionSet& s) { // failed to convert, but keep going PyErr_Clear(); - return FunctionArg(); + return LiveFunctionArg(); } catch(...) { // not a valid conversion - return FunctionArg(); + return LiveFunctionArg(); } } else if (mArgs[argIx].getIsKwarg()) { if (argType->getTypeCategory() != Type::TypeCategory::catNamedTuple) { @@ -447,17 +447,17 @@ class FunctionCallArgMapping { NamedTuple* tup = (NamedTuple*)argType; if (mKwargValues.size() != tup->getTypes().size()) { - return FunctionArg(); + return LiveFunctionArg(); } for (long k = 0; k < mKwargValues.size(); k++) { if (mKwargValues[k].first != tup->getNames()[k]) { - return FunctionArg(); + return LiveFunctionArg(); } } try { - return FunctionArg( + return LiveFunctionArg( Instance::createAndInitialize(tup, [&](instance_ptr p) { tup->constructor(p, [&](instance_ptr subElt, int tupArg) { PyInstance::copyConstructFromPythonInstance( @@ -472,11 +472,11 @@ class FunctionCallArgMapping { } catch(PythonExceptionSet& s) { // failed to convert, but keep going PyErr_Clear(); - return FunctionArg(); + return LiveFunctionArg(); } catch(...) { // not a valid conversion - return FunctionArg(); + return LiveFunctionArg(); } } @@ -485,7 +485,7 @@ class FunctionCallArgMapping { private: - const std::vector& mArgs; + const std::vector& mArgs; std::vector mSingleValueArgs; std::vector mStarArgValues; diff --git a/typed_python/FunctionGlobal.cpp b/typed_python/FunctionGlobal.cpp new file mode 100644 index 000000000..d7aa45095 --- /dev/null +++ b/typed_python/FunctionGlobal.cpp @@ -0,0 +1,48 @@ +#include "FunctionGlobal.hpp" +#include "CompilerVisibleObjectVisitor.hpp" + + +/* static */ +std::pair FunctionGlobal::DottedGlobalsLookup( + PyObject* funcGlobals, + std::string dotSequence +) { + if (!PyDict_Check(funcGlobals)) { + throw std::runtime_error("Can't handle a non-dict function globals"); + } + + std::string shortGlobalName = dotSequence; + + size_t indexOfDot = shortGlobalName.find('.'); + if (indexOfDot != std::string::npos) { + shortGlobalName = shortGlobalName.substr(0, indexOfDot); + } + + PyObject* globalVal = PyDict_GetItemString(funcGlobals, shortGlobalName.c_str()); + + if (!globalVal) { + PyObject* builtins = PyDict_GetItemString(funcGlobals, "__builtins__"); + + if (builtins && PyDict_Check(builtins) && PyDict_GetItemString(builtins, shortGlobalName.c_str())) { + return DottedGlobalsLookup(builtins, dotSequence); + } + } + + std::string moduleName = CompilerVisibleObjectVisitor::isPyObjectGloballyIdentifiableModuleDict(funcGlobals); + if (moduleName.size()) { + return std::make_pair( + shortGlobalName, + FunctionGlobal::NamedModuleMember( + funcGlobals, + moduleName, + shortGlobalName + ) + ); + } + + // TODO: use the remainder of the dot sequence usefully + return std::make_pair( + shortGlobalName, + FunctionGlobal::GlobalInDict(funcGlobals, shortGlobalName) + ); +} diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp new file mode 100644 index 000000000..ecefef786 --- /dev/null +++ b/typed_python/FunctionGlobal.hpp @@ -0,0 +1,547 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + + +#include "PyObjSnapshot.hpp" + + +class FunctionGlobal { + enum class GlobalType { + Unbound = 1, // this global is not bound to anything - its a Name error + NamedModuleMember = 2, // this global is a member of a particular module + Constant = 3, // this global is bound to a constant that we resolved at some point + GlobalInDict = 4, // this global is an unresolved variable residing in some python dict + GlobalInCell = 5, // this global is an unresolved variable residing in some PyCell + }; + + FunctionGlobal( + GlobalType inKind, + PyObject* dictOrCell, + std::string name, + std::string moduleName, + Type* constant + ) : + mKind(inKind), + mModuleDictOrCell(incref(dictOrCell)), + mName(name), + mModuleName(moduleName), + mConstant(constant) + { + } + +public: + FunctionGlobal() : + mKind(GlobalType::Unbound), + mModuleDictOrCell(nullptr) + { + } + + FunctionGlobal withConstantsInternalized(const std::map& typeMap) { + if (!(isGlobalInDict() || isGlobalInCell())) { + return *this; + } + + Type* t = getValueAsType(); + + if (t) { + auto it = typeMap.find(t); + if (it != typeMap.end()) { + return FunctionGlobal::Constant(it->second); + } else { + return FunctionGlobal::Constant(t); + } + } + + return *this; + } + + std::string toString() { + if (isUnbound()) { + return "FunctionGlobal.Unbound()"; + } + + if (isNamedModuleMember()) { + return "FunctionGlobal.NamedModuleMember(" + mModuleName + ", " + mName + ")"; + } + + if (isGlobalInCell()) { + return "FunctionGlobal.GlobalInCell()"; + } + + if (isGlobalInDict()) { + return "FunctionGlobal.GlobalInDict()"; + } + + if (isConstant()) { + return "FunctionGlobal.Constant(type=" + mConstant->name() + ")"; + } + + throw std::runtime_error("Unknown FunctionGlobal Kind"); + } + + static FunctionGlobal Unbound() { + return FunctionGlobal(); + } + + static FunctionGlobal Constant(Type* constant) { + if (!constant) { + throw std::runtime_error("FunctionGlobal::Constant can't be null"); + } + + return FunctionGlobal( + GlobalType::Constant, + nullptr, + "", + "", + constant + ); + } + + // return an approriate FunctionGlobal given that we are indexing into 'funcGlobals' + // with a dot-variable sequence. Returns a pair of (dotSequence, FunctionGlobal) that + // we actually want to use + static std::pair DottedGlobalsLookup( + PyObject* funcGlobals, + std::string dotSequence + ); + + static FunctionGlobal NamedModuleMember(PyObject* moduleDict, std::string moduleName, std::string name) { + if (!PyDict_Check(moduleDict)) { + throw std::runtime_error("NamedModuleMember requires a Python dict object"); + } + + return FunctionGlobal( + GlobalType::NamedModuleMember, + moduleDict, + name, + moduleName, + nullptr + ); + } + + static FunctionGlobal GlobalInDict(PyObject* dict, std::string name) { + if (!PyDict_Check(dict)) { + throw std::runtime_error("GlobalInDict requires a Python dict object"); + } + + return FunctionGlobal( + GlobalType::GlobalInDict, + dict, + name, + "", + nullptr + ); + } + + static FunctionGlobal GlobalInCell(PyObject* cell) { + if (!PyCell_Check(cell)) { + throw std::runtime_error("GlobalInCell requires a Python cell object"); + } + + return FunctionGlobal( + GlobalType::GlobalInCell, + cell, + "", + "", + nullptr + ); + } + + bool isUnbound() const { + return mKind == GlobalType::Unbound; + } + + bool isNamedModuleMember() const { + return mKind == GlobalType::NamedModuleMember; + } + + bool isConstant() const { + return mKind == GlobalType::Constant; + } + + bool isGlobalInDict() const { + return mKind == GlobalType::GlobalInDict; + } + + bool isGlobalInCell() const { + return mKind == GlobalType::GlobalInCell; + } + + Type* getValueAsType() { + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* ref = extractGlobalRefFromDictOrCell(); + if (!ref) { + return nullptr; + } + return PyInstance::extractTypeFrom(ref); + } + + if (isUnbound()) { + return nullptr; + } + + if (isConstant()) { + return mConstant; + } + + throw std::runtime_error("Unknown global kind."); + } + + PyObject* getValueAsPyobj() { + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + return extractGlobalRefFromDictOrCell(); + } + + if (isUnbound()) { + return nullptr; + } + + if (isConstant()) { + return (PyObject*)PyInstance::typeObj(mConstant); + } + + throw std::runtime_error("Unknown global kind."); + } + + GlobalType getKind() const { + return mKind; + } + + Type* getConstant() const { + return mConstant; + } + + const std::string& getName() const { + return mName; + } + + const std::string& getModuleName() const { + return mModuleName; + } + + PyObject* getModuleDictOrCell() const { + return mModuleDictOrCell; + } + + PyObject* extractGlobalRefFromDictOrCell() const { + if (isGlobalInDict()) { + return PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); + } + + if (isGlobalInCell()) { + return PyCell_Get(mModuleDictOrCell); + } + + if (isNamedModuleMember()) { + return PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); + } + + return nullptr; + } + + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (isUnbound()) { + return; + } + + if (isConstant()) { + visitor(mConstant); + return; + } + + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* obj = extractGlobalRefFromDictOrCell(); + + if (obj) { + _visitReferencedTypesInPyobj(obj, visitor); + } + return; + } + } + + template + void _visitReferencedTypesInPyobj(PyObject* obj, visitor_type vis) { + if (!obj) { + return; + } + + if (!PyType_Check(obj)) { + return; + } + + Type* t = PyInstance::extractTypeFrom(obj); + + if (t) { + vis(t); + } + } + + template + void _visitCompilerVisibleInternals(const visitor_type& visitor) { + if (isUnbound()) { + return; + } + + if (isConstant()) { + visitor.visitTopo(mConstant); + return; + } + + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* obj = extractGlobalRefFromDictOrCell(); + + if (obj) { + _visitCompilerVisibleInternalsInPyobj(obj, visitor); + } + return; + } + } + + template + void _visitCompilerVisibleInternalsInPyobj(PyObject* obj, const visitor_type& visitor) { + if (!PyType_Check(obj)) { + return; + } + + Type* t = PyInstance::extractTypeFrom(obj); + + if (t && t->isForwardDefined()) { + if (t->isResolved()) { + visitor.visitTopo( + t->forwardResolvesTo() + ); + } + } else if (t) { + visitor.visitTopo(t); + } else { + visitor.visitTopo(obj); + } + } + + FunctionGlobal withUpdatedInternalTypePointers(const std::map& groupMap) { + Type* t = getValueAsType(); + + if (!t) { + return *this; + } + + auto it = groupMap.find(t); + + if (it != groupMap.end()) { + // we're actually a constant! + return FunctionGlobal::Constant( + it->second + ); + } + + return *this; + } + + + bool isUnresolved(bool insistForwardsResolved) { + if (isConstant()) { + return false; + } + if (isUnbound()) { + return false; + } + + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* obj = extractGlobalRefFromDictOrCell(); + + if (!obj) { + return true; + } + + if (insistForwardsResolved) { + Type* t = PyInstance::extractTypeFrom(obj); + if (t && t->isForwardDefined()) { + return true; + } + } + + return false; + } + + return true; + } + + void autoresolveGlobal( + const std::set& resolvedForwards + ) { + if (isGlobalInCell()) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)PyCell_Get(mModuleDictOrCell) + ); + + if (cellContents && resolvedForwards.find(cellContents) != resolvedForwards.end()) { + PyCell_Set( + mModuleDictOrCell, + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + + return; + } + + if (isGlobalInDict() || isNamedModuleMember()) { + PyObject* dictVal = PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); + + if (dictVal) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)dictVal + ); + + if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + PyDict_SetItemString( + mModuleDictOrCell, + mName.c_str(), + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + } + } + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + buffer.writeRegisterType(0, (uint64_t)mKind); + + if (mKind == GlobalType::NamedModuleMember) { + context.serializePythonObject(mModuleDictOrCell, buffer, 1); + buffer.writeStringObject(2, mName); + buffer.writeStringObject(3, mModuleName); + } else + if (mKind == GlobalType::Constant) { + context.serializeNativeType(mConstant, buffer, 4); + } else + if (mKind == GlobalType::GlobalInDict) { + context.serializePythonObject(mModuleDictOrCell, buffer, 1); + buffer.writeStringObject(2, mName); + } else + if (mKind == GlobalType::GlobalInCell) { + context.serializePythonObject(mModuleDictOrCell, buffer, 1); + } + + buffer.writeEndCompound(); + } + + template + static FunctionGlobal deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + uint64_t kind = 0; + std::string name, moduleName; + PyObjectHolder cellOrModule; + Type* constant; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + buffer.readRegisterType(&kind, wireType); + } else + if (fieldNumber == 1) { + cellOrModule.steal(context.deserializePythonObject(buffer, wireType)); + } else + if (fieldNumber == 2) { + name = buffer.readStringObject(); + } else + if (fieldNumber == 3) { + moduleName = buffer.readStringObject(); + } else + if (fieldNumber == 4) { + constant = context.deserializeNativeType(buffer, wireType); + } + }); + + if (kind == (int)GlobalType::Unbound) { + return FunctionGlobal::Unbound(); + } + if (kind == (int)GlobalType::NamedModuleMember) { + if (!cellOrModule || !name.size() || !moduleName.size()) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid NamedModuleMember"); + } + return FunctionGlobal::NamedModuleMember(cellOrModule, name, moduleName); + } + if (kind == (int)GlobalType::Constant) { + if (!constant) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid constant"); + } + return FunctionGlobal::Constant(constant); + } + if (kind == (int)GlobalType::GlobalInDict) { + if (!cellOrModule || !name.size()) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid GlobalInDict"); + } + return FunctionGlobal::GlobalInDict(cellOrModule, name); + } + if (kind == (int)GlobalType::GlobalInCell) { + if (!cellOrModule) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid cell"); + } + return FunctionGlobal::GlobalInCell((PyObject*)cellOrModule); + } + + throw std::runtime_error("Corrupt FunctionGlobal - invalid 'kind'"); + } + + bool operator<(const FunctionGlobal& g) const { + if (mKind < g.mKind) { + return true; + } + if (mKind > g.mKind) { + return false; + } + + if (mConstant < g.mConstant) { + return true; + } + if (mConstant > g.mConstant) { + return false; + } + + if (mName < g.mName) { + return true; + } + if (mName > g.mName) { + return false; + } + + if (mModuleDictOrCell < g.mModuleDictOrCell) { + return true; + } + if (mModuleDictOrCell > g.mModuleDictOrCell) { + return false; + } + + return false; + } + +private: + GlobalType mKind; + + Type* mConstant; + + std::string mName; + std::string mModuleName; + PyObject* mModuleDictOrCell; +}; diff --git a/typed_python/FunctionOverload.cpp b/typed_python/FunctionOverload.cpp new file mode 100644 index 000000000..e689742b9 --- /dev/null +++ b/typed_python/FunctionOverload.cpp @@ -0,0 +1,173 @@ +#include "FunctionOverload.hpp" + + +/* static */ +PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr closureData) { + if (mCachedFunctionObj) { + return incref(mCachedFunctionObj); + } + + std::set globalsInCells( + mFunctionClosureVarnames.begin(), + mFunctionClosureVarnames.end() + ); + + PyObjectStealer funcGlobals(PyDict_New()); + + for (auto& nameAndGlobal: mGlobals) { + if (globalsInCells.find(nameAndGlobal.first) == globalsInCells.end()) { + PyObject* val = nameAndGlobal.second.getValueAsPyobj(); + + if (val) { + PyDict_SetItemString( + (PyObject*)funcGlobals, + nameAndGlobal.first.c_str(), + val + ); + } + } + } + + if (mFunctionGlobalsInClosureVarnames.size()) { + if (mGlobals.size() || globalsInCells.size()) { + throw std::runtime_error( + "Invalid FunctionOverload. We can't convert this to a python function. " + "We are supposed to have our globals either encoded in the globals dict " + "assuming that they are resolvable in a way that can place them in the type " + "or they are supposed to be in our closure. We're not supposed to mix them." + ); + } + + if (!closureType->isTuple() + || ((Tuple*)closureType)->getTypes().size() != 1 + || !((Tuple*)closureType)->getTypes()[0]->isNamedTuple() + ) { + throw std::runtime_error("Invalid closure-enclosing-globals"); + } + + NamedTuple* interior = (NamedTuple*)((Tuple*)closureType)->getTypes()[0]; + if (!interior->getTypes().size() || interior->getNames().back() != " _globals") { + throw std::runtime_error("Invalid closure-enclosing-globals"); + } + + Instance i = ( + ClosureVariableBinding() + 0 + ClosureVariableBindingStep(" _globals") + ).extractValueOrContainingClosure(closureType, closureData); + + if (i.type() != PythonObjectOfType::AnyPyDict()) { + throw std::runtime_error("Invalid closure-enclosing-globals"); + } + + funcGlobals.set(PythonObjectOfType::getPyObj(i.data())); + } + + PyObject* res = PyFunction_New(mFunctionCode, (PyObject*)funcGlobals); + + if (!res) { + throw PythonExceptionSet(); + } + + if (mFunctionDefaults) { + if (PyFunction_SetDefaults(res, mFunctionDefaults) == -1) { + throw PythonExceptionSet(); + } + } + + if (mFunctionAnnotations) { + if (PyFunction_SetAnnotations(res, mFunctionAnnotations) == -1) { + throw PythonExceptionSet(); + } + } + + int closureVarCount = PyCode_GetNumFree((PyCodeObject*)mFunctionCode); + + if (mFunctionClosureVarnames.size() != closureVarCount) { + throw std::runtime_error( + "Invalid closure: wrong number of cells: wanted " + + format(closureVarCount) + + " but got " + + format(mFunctionClosureVarnames.size()) + ); + } + + if (closureVarCount) { + PyObjectStealer closureTup(PyTuple_New(closureVarCount)); + + for (long k = 0; k < closureVarCount; k++) { + std::string varname = mFunctionClosureVarnames[k]; + + if (varname == "__class__" && getMethodOf()) { + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New((PyObject*)PyInstance::typeObj( + ((HeldClass*)getMethodOf())->getClassType() + )) + ); + } else { + auto globalIt = mGlobals.find(varname); + if (globalIt != mGlobals.end()) { + PyObject* globalVal = globalIt->second.getValueAsPyobj(); + + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New(globalVal) + ); + } else { + auto bindingIt = mClosureBindings.find(varname); + if (bindingIt == mClosureBindings.end()) { + throw std::runtime_error("Closure variable " + varname + " not in globals or closure bindings."); + } + + ClosureVariableBinding binding = bindingIt->second; + + Instance bindingValue = binding.extractValueOrContainingClosure(closureType, closureData); + + if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catPyCell) { + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New( + ((PyCellType*)bindingValue.type())->getPyObj(bindingValue.data()) + ) + ); + } else { + // it's not a cell. We have to encode it as a PyCell for the interpreter + // to be able to handle it. + PyObjectHolder newCellContents; + + if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catTypedCell) { + newCellContents.steal( + PyInstance::extractPythonObject( + ((TypedCellType*)bindingValue.type())->get(bindingValue.data()), + ((TypedCellType*)bindingValue.type())->getHeldType() + ) + ); + } else { + newCellContents.steal( + PyInstance::fromInstance(bindingValue) + ); + } + + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New(newCellContents) + ); + } + } + } + } + + if (PyFunction_SetClosure(res, (PyObject*)closureTup) == -1) { + throw PythonExceptionSet(); + } + } + + if (closureType->bytecount() == 0) { + mCachedFunctionObj = incref(res); + } + + return res; +} diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp new file mode 100644 index 000000000..a71942d41 --- /dev/null +++ b/typed_python/FunctionOverload.hpp @@ -0,0 +1,1042 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Type.hpp" +#include "TypedCellType.hpp" +#include "ReprAccumulator.hpp" +#include "Format.hpp" +#include "SpecialModuleNames.hpp" +#include "PyInstance.hpp" +#include "ClosureVariableBinding.hpp" +#include "FunctionArg.hpp" +#include "CompiledSpecialization.hpp" +#include "FunctionGlobal.hpp" + + +class FunctionOverload { +public: + FunctionOverload() : + mFunctionCode(nullptr), + mFunctionDefaults(nullptr), + mFunctionAnnotations(nullptr), + mReturnType(nullptr), + mSignatureFunction(nullptr), + mCachedFunctionObj(nullptr), + mMethodOf(nullptr) + { + } + + FunctionOverload( + PyObject* pyFuncCode, + PyObject* pyFuncDefaults, + PyObject* pyFuncAnnotations, + const std::map& inGlobals, + const std::vector& pyFuncClosureVarnames, + const std::vector& globalsInClosureVarnames, + const std::map& closureBindings, + Type* returnType, + PyObject* pySignatureFunction, + const std::vector& args, + Type* methodOf + ) : + mFunctionCode(pyFuncCode), + mFunctionDefaults(pyFuncDefaults), + mFunctionAnnotations(pyFuncAnnotations), + mGlobals(inGlobals), + mFunctionClosureVarnames(pyFuncClosureVarnames), + mFunctionGlobalsInClosureVarnames(globalsInClosureVarnames), + mReturnType(returnType), + mSignatureFunction(pySignatureFunction), + mArgs(args), + mHasKwarg(false), + mHasStarArg(false), + mMinPositionalArgs(0), + mMaxPositionalArgs(-1), + mClosureBindings(closureBindings), + mCachedFunctionObj(nullptr), + mMethodOf(methodOf) + { + long argsWithDefaults = 0; + long argsDefinitelyConsuming = 0; + + for (auto arg: mArgs) { + if (arg.getIsStarArg()) { + mHasStarArg = true; + } + else if (arg.getIsKwarg()) { + mHasKwarg = true; + } + else if (arg.getDefaultValue()) { + argsWithDefaults++; + } else { + argsDefinitelyConsuming++; + } + } + + mMinPositionalArgs = argsDefinitelyConsuming; + if (!mHasStarArg) { + mMaxPositionalArgs = argsDefinitelyConsuming + argsWithDefaults; + } + + increfAllPyObjects(); + } + + FunctionOverload(const FunctionOverload& other) { + other.increfAllPyObjects(); + + mFunctionCode = other.mFunctionCode; + mGlobals = other.mGlobals; + mFunctionDefaults = other.mFunctionDefaults; + mFunctionAnnotations = other.mFunctionAnnotations; + mSignatureFunction = other.mSignatureFunction; + mMethodOf = other.mMethodOf; + + mFunctionClosureVarnames = other.mFunctionClosureVarnames; + mFunctionGlobalsInClosureVarnames = other.mFunctionGlobalsInClosureVarnames; + + mClosureBindings = other.mClosureBindings; + mReturnType = other.mReturnType; + mArgs = other.mArgs; + mCompiledSpecializations = other.mCompiledSpecializations; + + mHasStarArg = other.mHasStarArg; + mHasKwarg = other.mHasKwarg; + mMinPositionalArgs = other.mMinPositionalArgs; + mMaxPositionalArgs = other.mMaxPositionalArgs; + + mCachedFunctionObj = other.mCachedFunctionObj; + } + + ~FunctionOverload() { + decrefAllPyObjects(); + } + + FunctionOverload withShiftedFrontClosureBindings(long shiftAmount) const { + std::map bindings; + for (auto nameAndBinding: mClosureBindings) { + bindings[nameAndBinding.first] = nameAndBinding.second.withShiftedFrontBinding(shiftAmount); + } + + return FunctionOverload( + mFunctionCode, + mFunctionDefaults, + mFunctionAnnotations, + mGlobals, + mFunctionClosureVarnames, + mFunctionGlobalsInClosureVarnames, + bindings, + mReturnType, + mSignatureFunction, + mArgs, + mMethodOf + ); + } + + FunctionOverload withMethodOf(Type* methodOf) const { + return FunctionOverload( + mFunctionCode, + mFunctionDefaults, + mFunctionAnnotations, + mGlobals, + mFunctionClosureVarnames, + mFunctionGlobalsInClosureVarnames, + mClosureBindings, + mReturnType, + mSignatureFunction, + mArgs, + methodOf + ); + } + + FunctionOverload withClosureBindings(const std::map &bindings) const { + return FunctionOverload( + mFunctionCode, + mFunctionDefaults, + mFunctionAnnotations, + mGlobals, + mFunctionClosureVarnames, + mFunctionGlobalsInClosureVarnames, + bindings, + mReturnType, + mSignatureFunction, + mArgs, + mMethodOf + ); + } + + std::string toString() const { + std::ostringstream str; + + str << "("; + + if (mMethodOf) { + str << "method of " << mMethodOf->name() << ", "; + } + + for (long k = 0; k < mArgs.size(); k++) { + if (k) { + str << ", "; + } + + if (mArgs[k].getIsStarArg()) { + str << "*"; + } + + if (mArgs[k].getIsKwarg()) { + str << "**"; + } + + str << mArgs[k].getName(); + + if (mArgs[k].getDefaultValue()) { + str << "=..."; + } + + if (mArgs[k].getTypeFilter()) { + str << ": " << mArgs[k].getTypeFilter()->name(); + } + } + + str << ")"; + + if (mReturnType) { + str << " -> " << mReturnType->name(); + } + + return str.str(); + } + + // return the FunctionArg* that a positional argument would map to, or 'nullptr' if + // it wouldn't + const FunctionArg* argForPositionalArgument(long argIx) const { + if (argIx >= mArgs.size()) { + return nullptr; + } + + if (mArgs[argIx].getIsStarArg() || mArgs[argIx].getIsKwarg()) { + return nullptr; + } + + return &mArgs[argIx]; + } + + // can we possibly match 'argCount' positional arguments? + bool couldMatchPositionalCount(long argCount) const { + return argCount >= mMinPositionalArgs && argCount < mMaxPositionalArgs; + } + + Type* getReturnType() const { + return mReturnType; + } + + const std::vector& getFunctionClosureVarnames() const { + return mFunctionClosureVarnames; + } + + const std::vector& getFunctionGlobalsInClosureVarnames() const { + return mFunctionGlobalsInClosureVarnames; + } + + const std::vector& getArgs() const { + return mArgs; + } + + void finalizeTypeConcrete() { + // ensure that the Function object we represent is in sync with + // our actual return and argument types. It would be better to not have + // any direct references to the python interpreter in this class and + // rebuild the concrete function object on demand later... + if (mReturnType) { + if (mFunctionAnnotations + && PyDict_Check(mFunctionAnnotations) + ) { + PyDict_SetItemString( + mFunctionAnnotations, + "return", + (PyObject*)PyInstance::typeObj(mReturnType) + ); + } + } + for (auto& a: mArgs) { + Type* typeFilter = a.getTypeFilter(); + + if (typeFilter + && mFunctionAnnotations + && PyDict_Check(mFunctionAnnotations) + ) { + PyDict_SetItemString( + mFunctionAnnotations, + a.getName().c_str(), + (PyObject*)PyInstance::typeObj(a.getTypeFilter()) + ); + } + } + } + + void internalizeConstants(const std::map& typeMap) { + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second = nameAndGlobal.second.withConstantsInternalized(typeMap); + } + } + + void updateInternalTypePointers(const std::map& groupMap) { + if (mReturnType) { + Type::updateTypeRefFromGroupMap(mReturnType, groupMap); + } + + for (auto& a: mArgs) { + a._visitReferencedTypes([&](Type*& typePtr) { + Type::updateTypeRefFromGroupMap(typePtr, groupMap); + }); + } + + for (auto& varnameAndBinding: mClosureBindings) { + varnameAndBinding.second._visitReferencedTypes([&](Type*& typePtr) { + Type::updateTypeRefFromGroupMap(typePtr, groupMap); + }); + } + + if (mMethodOf) { + Type::updateTypeRefFromGroupMap(mMethodOf, groupMap); + } + + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second = nameAndGlobal.second.withUpdatedInternalTypePointers(groupMap); + } + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + // we need to keep mArgs and mReturnType in sync with + // mAnnotations, so if we change one of our types we + // need to update the resulting dictionary as well. + if (mReturnType) { + visitor(mReturnType); + } + for (auto& a: mArgs) { + a._visitReferencedTypes(visitor); + } + + for (auto& varnameAndBinding: mClosureBindings) { + varnameAndBinding.second._visitReferencedTypes(visitor); + } + + if (mMethodOf) { + visitor(mMethodOf); + } + + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second._visitReferencedTypes(visitor); + } + } + + template + void _visitCompilerVisibleInternals(const visitor_type& visitor) { + visitor.visitTopo(mFunctionCode); + + if (mFunctionAnnotations) { + visitor.visitHash(ShaHash(2)); + + if (PyDict_CheckExact(mFunctionAnnotations)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(mFunctionAnnotations, &pos, &key, &value)) { + visitor.visitTopo(key); + visitor.visitTopo(value); + } + } + + visitor.visitHash(ShaHash(2)); + } + + if (mSignatureFunction) { + visitor.visitHash(ShaHash(1)); + visitor.visitTopo(mSignatureFunction); + } else { + visitor.visitHash(ShaHash(0)); + } + + if (mFunctionDefaults) { + visitor.visitHash(ShaHash(2)); + + if (PyDict_CheckExact(mFunctionDefaults)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(mFunctionDefaults, &pos, &key, &value)) { + visitor.visitTopo(key); + visitor.visitTopo(value); + } + } + + visitor.visitHash(ShaHash(2)); + } + + visitor.visitHash(ShaHash(mGlobals.size())); + + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second._visitCompilerVisibleInternals(visitor); + } + + if (mReturnType) { + visitor.visitTopo(mReturnType); + } else { + visitor.visitHash(ShaHash()); + } + + if (mMethodOf) { + visitor.visitTopo(mMethodOf); + } else { + visitor.visitHash(ShaHash()); + } + + visitor.visitHash(ShaHash(mClosureBindings.size())); + for (auto nameAndClosure: mClosureBindings) { + visitor.visitName(nameAndClosure.first); + nameAndClosure.second._visitCompilerVisibleInternals(visitor); + } + + visitor.visitHash(ShaHash(mArgs.size())); + + for (auto a: mArgs) { + a._visitCompilerVisibleInternals(visitor); + } + + visitor.visitHash(ShaHash(1)); + visitor.visitHash(ShaHash(mGlobals.size())); + + for (auto& nameAndGlobal: mGlobals) { + visitor.visitName(nameAndGlobal.first); + nameAndGlobal.second._visitCompilerVisibleInternals(visitor); + } + + visitor.visitHash(ShaHash(1)); + } + + template + static void visitCompilerVisibleGlobals( + const visitor_type& visitor, + PyCodeObject* code, + PyObject* globals + ) { + std::vector > dotAccesses; + + extractDottedGlobalAccessesFromCode(code, dotAccesses); + + auto visitSequence = [&](const std::vector& sequence) { + PyObjectHolder curObj; + std::string curName; + + for (PyObject* name: sequence) { + std::string nameStr = PyUnicode_AsUTF8(name); + + if (isSpecialIgnorableName(nameStr)) { + break; + } + + if (!curObj) { + curName = nameStr; + } else { + curName = curName + "." + nameStr; + } + + if (!curObj) { + // this is a lookup in the global dict + curObj.set(PyDict_GetItem(globals, name)); + + if (!curObj) { + // this is an invalid global lookup, which is OK. no need to hash anything. + PyErr_Clear(); + return; + } + } else { + // we're looking up an attribute of this object. We only want to look into modules. + if (PyModule_CheckExact(curObj) && PyObject_HasAttr(curObj, name)) { + PyObjectStealer moduleMember(PyObject_GetAttr(curObj, name)); + + curObj.steal(PyObject_GetAttr(curObj, name)); + if (!curObj) { + // this is an invalid module member lookup. We can just bail. + PyErr_Clear(); + return; + } + } else { + break; + } + } + } + + // also visit at the end of the sequence + if (curObj) { + visitor(curName, (PyObject*)curObj); + } + }; + + for (auto& sequence: dotAccesses) { + visitSequence(sequence); + } + } + + template + void _visitContainedTypes(const visitor_type& visitor) { + } + + const std::vector& getCompiledSpecializations() const { + return mCompiledSpecializations; + } + + void addCompiledSpecialization(compiled_code_entrypoint e, Type* returnType, const std::vector& argTypes) { + CompiledSpecialization newSpec = CompiledSpecialization(e,returnType,argTypes); + + for (auto& spec: mCompiledSpecializations) { + if (spec == newSpec) { + return; + } + } + + mCompiledSpecializations.push_back(newSpec); + } + + void touchCompiledSpecializations() { + //force the memory for the compiled specializations to move. + std::vector other = mCompiledSpecializations; + std::swap(mCompiledSpecializations, other); + } + + bool operator<(const FunctionOverload& other) const { + if (mFunctionCode < other.mFunctionCode) { return true; } + if (mFunctionCode > other.mFunctionCode) { return false; } + + if (mGlobals < other.mGlobals) { return true; } + if (mGlobals > other.mGlobals) { return false; } + + if (mClosureBindings < other.mClosureBindings) { return true; } + if (mClosureBindings > other.mClosureBindings) { return false; } + + if (mReturnType < other.mReturnType) { return true; } + if (mReturnType > other.mReturnType) { return false; } + + if (mArgs < other.mArgs) { return true; } + if (mArgs > other.mArgs) { return false; } + + if (mMethodOf > other.mMethodOf) { return true; } + if (mMethodOf < other.mMethodOf) { return false; } + + return false; + } + + const std::map& getClosureVariableBindings() const { + return mClosureBindings; + } + + PyObject* getFunctionCode() const { + return mFunctionCode; + } + + PyObject* getFunctionDefaults() const { + return mFunctionDefaults; + } + + PyObject* getFunctionAnnotations() const { + return mFunctionAnnotations; + } + + Type* getMethodOf() const { + return mMethodOf; + } + + PyObject* getSignatureFunction() const { + return mSignatureFunction; + } + + const std::map& getGlobals() const { + return mGlobals; + } + + std::map& getGlobals() { + return mGlobals; + } + + /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. + + In cases where we write something like 'x.y.z' the compiler shouldn't have a reference to 'x', + just to whatever 'x.y.z' refers to. + + This transformation just figures out what the dotting sequences are. + */ + static void extractDottedGlobalAccessesFromCode(PyCodeObject* code, std::vector >& outSequences) { + uint8_t* bytes; + Py_ssize_t bytecount; + + static PyObject* moduleHashName = PyUnicode_FromString("__module_hash__"); + outSequences.push_back(std::vector({moduleHashName})); + + PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); + + long opcodeCount = bytecount / 2; + + // opcodes are encoded in the low byte + auto opcodeFor = [&](int i) { return bytes[i * 2]; }; + + // opcode targets are encoded in the high byte + auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; + + const uint8_t LOAD_ATTR = 106; + const uint8_t LOAD_GLOBAL = 116; + const uint8_t DELETE_GLOBAL = 98; + const uint8_t STORE_GLOBAL = 97; + const uint8_t LOAD_METHOD = 160; + + + std::vector curDotSequence; + for (long ix = 0; ix < opcodeCount; ix++) { + // if we're loading an attr on an existing sequence, just make it bigger + if ((opcodeFor(ix) == LOAD_ATTR || opcodeFor(ix) == LOAD_METHOD) && curDotSequence.size()) { + curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); + } else if (curDotSequence.size()) { + // any other operation should flush the buffer + outSequences.push_back(curDotSequence); + curDotSequence.clear(); + } + + // if we're loading a global, we start a new sequence + if (opcodeFor(ix) == LOAD_GLOBAL) { + curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); + } else if ( + opcodeFor(ix) == STORE_GLOBAL + || opcodeFor(ix) == DELETE_GLOBAL + ) { + outSequences.push_back({PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))}); + } + } + + // flush the buffer if we have something + if (curDotSequence.size()) { + outSequences.push_back(curDotSequence); + } + + // recurse into sub code objects + iterate(code->co_consts, [&](PyObject* o) { + if (PyCode_Check(o)) { + extractDottedGlobalAccessesFromCode((PyCodeObject*)o, outSequences); + } + }); + } + + // get a list of all "global accesses" contained in 'code'. This will pull out every + // case where we have an opcodes that access a global variable by name + static void extractGlobalAccessesFromCode(PyCodeObject* code, std::set& outAccesses) { + uint8_t* bytes; + Py_ssize_t bytecount; + + PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); + + long opcodeCount = bytecount / 2; + + // opcodes are encoded in the low byte + auto opcodeFor = [&](int i) { return bytes[i * 2]; }; + + // opcode targets are encoded in the high byte + auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; + + const uint8_t LOAD_GLOBAL = 116; + const uint8_t DELETE_GLOBAL = 98; + const uint8_t STORE_GLOBAL = 97; + + for (long ix = 0; ix < opcodeCount; ix++) { + // if we're loading a global, we start a new sequence + if (opcodeFor(ix) == LOAD_GLOBAL) { + PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); + if (!PyUnicode_Check(name)) { + throw std::runtime_error("Function had a non-string object in co_names"); + } + outAccesses.insert(PyUnicode_AsUTF8(name)); + } else if ( + opcodeFor(ix) == STORE_GLOBAL + || opcodeFor(ix) == DELETE_GLOBAL + ) { + PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); + if (!PyUnicode_Check(name)) { + throw std::runtime_error("Function had a non-string object in co_names"); + } + outAccesses.insert(PyUnicode_AsUTF8(name)); + } + } + + // recurse into sub code objects + iterate(code->co_consts, [&](PyObject* o) { + if (PyCode_Check(o)) { + extractGlobalAccessesFromCode((PyCodeObject*)o, outAccesses); + } + }); + + outAccesses.insert("__module_hash__"); + } + + void extractGlobalAccessesFromCodeIncludingCells( + std::set& outNames + ) const { + for (auto nameAndGlobal: mGlobals) { + outNames.insert(nameAndGlobal.first); + } + + outNames.insert("__module_hash__"); + + extractGlobalAccessesFromCode((PyCodeObject*)mFunctionCode, outNames); + } + + bool symbolIsUnresolved(std::string name, bool insistForwardsResolved) { + if (mClosureBindings.find(name) != mClosureBindings.end()) { + return false; + } + + auto it = mGlobals.find(name); + + if (it == mGlobals.end()) { + return true; + } + + return it->second.isUnresolved(insistForwardsResolved); + } + + void autoresolveGlobal( + std::string name, + const std::set& resolvedForwards + ) { + auto it = mGlobals.find(name); + + if (it != mGlobals.end()) { + it->second.autoresolveGlobal(resolvedForwards); + } + } + + void autoresolveGlobals(const std::set& resolvedForwards) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__") { + autoresolveGlobal(nameStr, resolvedForwards); + } + } + } + + bool hasUnresolvedSymbols(bool insistForwardsResolved) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistForwardsResolved)) { + return true; + } + } + + return false; + } + + std::string firstUnresolvedSymbol(bool insistForwardsResolved) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistForwardsResolved)) { + return nameStr; + } + } + + return ""; + } + + static void buildInitialGlobalsDict( + std::map& outGlobals, + PyObject* inFuncGlobals, + PyCodeObject* inFuncCode + ) { + std::set allNamesString; + extractGlobalAccessesFromCode(inFuncCode, allNamesString); + + for (auto nameStr: allNamesString) { + if (nameStr != "__module_hash__" && outGlobals.find(nameStr) != outGlobals.end()) { + throw std::runtime_error( + "Somehow we already a closure binding for " + nameStr + + " and somehow we want to register a global binding?" + ); + } + + std::pair ref = FunctionGlobal::DottedGlobalsLookup( + inFuncGlobals, + nameStr + ); + + outGlobals[ref.first] = ref.second; + } + } + + // create a new function object for this closure (or cache it + // if we have no closure) + PyObject* buildFunctionObj(Type* closureType, instance_ptr closureData); + + FunctionOverload& operator=(const FunctionOverload& other) { + other.increfAllPyObjects(); + decrefAllPyObjects(); + + mFunctionCode = other.mFunctionCode; + mGlobals = other.mGlobals; + mFunctionDefaults = other.mFunctionDefaults; + mFunctionAnnotations = other.mFunctionAnnotations; + mSignatureFunction = other.mSignatureFunction; + + mMethodOf = other.mMethodOf; + + mFunctionClosureVarnames = other.mFunctionClosureVarnames; + mFunctionGlobalsInClosureVarnames = other.mFunctionGlobalsInClosureVarnames; + + mClosureBindings = other.mClosureBindings; + mReturnType = other.mReturnType; + mArgs = other.mArgs; + mCompiledSpecializations = other.mCompiledSpecializations; + + mHasStarArg = other.mHasStarArg; + mHasKwarg = other.mHasKwarg; + mMinPositionalArgs = other.mMinPositionalArgs; + mMaxPositionalArgs = other.mMaxPositionalArgs; + + mCachedFunctionObj = other.mCachedFunctionObj; + + return *this; + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + + context.serializePythonObject(mFunctionCode, buffer, 0); + + if (mFunctionDefaults) { + context.serializePythonObject(mFunctionDefaults, buffer, 2); + } + + if (mFunctionAnnotations) { + context.serializePythonObject(mFunctionAnnotations, buffer, 3); + } + + buffer.writeBeginCompound(4); + int stringIx = 0; + for (auto varname: mFunctionClosureVarnames) { + buffer.writeStringObject(stringIx++, varname); + } + buffer.writeEndCompound(); + + buffer.writeBeginCompound(11); + stringIx = 0; + for (auto varname: mFunctionGlobalsInClosureVarnames) { + buffer.writeStringObject(stringIx++, varname); + } + buffer.writeEndCompound(); + + buffer.writeBeginCompound(5); + int varIx = 0; + for (auto& nameAndGlobal: mGlobals) { + buffer.writeStringObject(varIx++, nameAndGlobal.first); + nameAndGlobal.second.serialize(context, buffer, varIx++); + } + buffer.writeEndCompound(); + + buffer.writeBeginCompound(6); + int closureBindingIx = 0; + for (auto nameAndBinding: mClosureBindings) { + buffer.writeStringObject(closureBindingIx++, nameAndBinding.first); + nameAndBinding.second.serialize(context, buffer, closureBindingIx++); + } + buffer.writeEndCompound(); + + if (mReturnType) { + context.serializeNativeType(mReturnType, buffer, 7); + } + + buffer.writeBeginCompound(8); + + int argIx = 0; + for (auto& arg: mArgs) { + arg.serialize(context, buffer, argIx++); + } + + buffer.writeEndCompound(); + + if (mSignatureFunction) { + context.serializePythonObject(mSignatureFunction, buffer, 9); + } + + if (mMethodOf) { + context.serializeNativeType(mMethodOf, buffer, 10); + } + + buffer.writeEndCompound(); + } + + template + static FunctionOverload deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + PyObjectHolder functionCode; + PyObjectHolder functionAnnotations; + PyObjectHolder functionDefaults; + PyObjectHolder functionSignature; + std::vector closureVarnames; + std::vector functionGlobalsInClosureVarnames; + std::map functionGlobals; + std::map closureBindings; + Type* returnType = nullptr; + Type* methodOf = nullptr; + std::vector args; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + functionCode.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 2) { + functionDefaults.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 3) { + functionAnnotations.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 4) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + assertWireTypesEqual(wireType, WireType::BYTES); + closureVarnames.push_back(buffer.readStringObject()); + }); + } + else if (fieldNumber == 11) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + assertWireTypesEqual(wireType, WireType::BYTES); + functionGlobalsInClosureVarnames.push_back(buffer.readStringObject()); + }); + } + else if (fieldNumber == 5) { + std::string last; + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber % 2 == 0) { + assertWireTypesEqual(wireType, WireType::BYTES); + last = buffer.readStringObject(); + } else { + if (last == "") { + throw std::runtime_error("Corrupt Function closure encountered"); + } + functionGlobals[last] = FunctionGlobal::deserialize(context, buffer, wireType); + last = ""; + } + }); + } + else if (fieldNumber == 6) { + std::string last; + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber % 2 == 0) { + assertWireTypesEqual(wireType, WireType::BYTES); + last = buffer.readStringObject(); + } else { + closureBindings[last] = ClosureVariableBinding::deserialize(context, buffer, wireType); + } + }); + } + else if (fieldNumber == 7) { + returnType = context.deserializeNativeType(buffer, wireType); + } + else if (fieldNumber == 8) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + args.push_back(FunctionArg::deserialize(context, buffer, wireType)); + }); + } + else if (fieldNumber == 9) { + functionSignature.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 10) { + methodOf = context.deserializeNativeType(buffer, wireType); + } + }); + + return FunctionOverload( + functionCode, + functionDefaults, + functionAnnotations, + functionGlobals, + closureVarnames, + functionGlobalsInClosureVarnames, + closureBindings, + returnType, + functionSignature, + args, + methodOf + ); + } + + void increfAllPyObjects() const { + incref(mFunctionCode); + incref(mFunctionDefaults); + incref(mFunctionAnnotations); + incref(mSignatureFunction); + } + + void decrefAllPyObjects() { + decref(mFunctionCode); + decref(mFunctionDefaults); + decref(mFunctionAnnotations); + decref(mSignatureFunction); + } + +private: + // every global we access not through our closure + std::map mGlobals; + + PyObject* mFunctionCode; + + // the order (by name) of the variables in the __closure__ of the original + // function. This is the order that the python code will expect. + std::vector mFunctionClosureVarnames; + + // a list of variables which are globals, but which we are reading from the + // closure because the 'globals' dictionary is in fact a free python object + // that can't be pinned down in the type + std::vector mFunctionGlobalsInClosureVarnames; + + PyObject* mFunctionDefaults; + + PyObject* mFunctionAnnotations; + + PyObject* mSignatureFunction; + + // note that we are deliberately leaking this value because Overloads get + // stashed in static std::map memos anyways. + mutable PyObject* mCachedFunctionObj; + + std::map mClosureBindings; + + Type* mReturnType; + + // if we are a method of a class, what class? + Type* mMethodOf; + + std::vector mArgs; + + // in compiled code, the closure arguments get passed in front of the + // actual function arguments + std::vector mCompiledSpecializations; + + bool mHasStarArg; + bool mHasKwarg; + size_t mMinPositionalArgs; + size_t mMaxPositionalArgs; +}; diff --git a/typed_python/FunctionType.cpp b/typed_python/FunctionType.cpp deleted file mode 100644 index 60a99672f..000000000 --- a/typed_python/FunctionType.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include "PyInstance.hpp" -#include "FunctionType.hpp" - -Instance ClosureVariableBinding::extractValueOrContainingClosure(Type* closureType, instance_ptr data) { - for (long stepIx = 0; stepIx < size(); stepIx++) { - ClosureVariableBindingStep step = (*this)[stepIx]; - - if (step.isFunction()) { - closureType = (Type*)step.getFunction(); - } else - if (step.isNamedField()) { - if (closureType->isNamedTuple()) { - NamedTuple* tupType = (NamedTuple*)closureType; - auto it = tupType->getNameToIndex().find(step.getNamedField()); - if (it == tupType->getNameToIndex().end()) { - throw std::runtime_error( - "Invalid closure: expected NamedTuple to have field " + - step.getNamedField() + " but it doesn't." - ); - } - - closureType = tupType->getTypes()[it->second]; - data = data + tupType->getOffsets()[it->second]; - } else - if (closureType->getTypeCategory() == Type::TypeCategory::catClass) { - Class* clsType = (Class*)closureType; - int index = clsType->getMemberIndex(step.getNamedField().c_str()); - if (index == -1) { - throw std::runtime_error("Can't find a field " + step.getNamedField() + " in class " + clsType->name()); - } - - if (!clsType->checkInitializationFlag(data, index)) { - throw std::runtime_error("Closure field " + step.getNamedField() + " is not populated."); - } - - closureType = clsType->getMemberType(index); - data = clsType->eltPtr(data, index); - } else { - throw std::runtime_error( - "Invalid closure: expected to find a Class or a NamedTuple." - ); - } - } else - if (step.isIndexedField()) { - if (!closureType->isComposite()) { - throw std::runtime_error("Invalid closure: expected a NamedTuple or Tuple but got " + closureType->name()); - } - - CompositeType* tupType = (CompositeType*)closureType; - - if (step.getIndexedField() < 0 || step.getIndexedField() >= tupType->getTypes().size()) { - throw std::runtime_error( - "Invalid closure: index " + format(step.getIndexedField()) + " is out of bounds in closure " + tupType->name() - ); - } - - closureType = tupType->getTypes()[step.getIndexedField()]; - data = data + tupType->getOffsets()[step.getIndexedField()]; - } else - if (step.isCellAccess()) { - if (!(closureType->getTypeCategory() == Type::TypeCategory::catPyCell || - closureType->getTypeCategory() == Type::TypeCategory::catTypedCell)) { - throw std::runtime_error( - "Invalid closure: expected a cell, but got " - + Type::categoryToString(closureType->getTypeCategory()) - ); - } - - if (stepIx + 1 == size()) { - // do nothing, because this function grabs the containing - // closure - } else { - if (closureType->getTypeCategory() == Type::TypeCategory::catPyCell) { - throw std::runtime_error("Corrupt closure encountered: a PyCell should always be the last step"); - } - - // it's a typed closure - data = ((TypedCellType*)closureType)->get(data); - closureType = ((TypedCellType*)closureType)->getHeldType(); - } - } else { - throw std::runtime_error("Corrupt closure variable binding enountered."); - } - } - - return Instance(data, closureType); -} - - -/* static */ -PyObject* Function::Overload::buildFunctionObj(Type* closureType, instance_ptr closureData) const { - if (mCachedFunctionObj) { - return incref(mCachedFunctionObj); - } - - PyObject* res = PyFunction_New(mFunctionCode, mFunctionGlobals); - - if (!res) { - throw PythonExceptionSet(); - } - - if (mFunctionDefaults) { - if (PyFunction_SetDefaults(res, mFunctionDefaults) == -1) { - throw PythonExceptionSet(); - } - } - - if (mFunctionAnnotations) { - if (PyFunction_SetAnnotations(res, mFunctionAnnotations) == -1) { - throw PythonExceptionSet(); - } - } - - int closureVarCount = PyCode_GetNumFree((PyCodeObject*)mFunctionCode); - - if (mFunctionClosureVarnames.size() != closureVarCount) { - throw std::runtime_error( - "Invalid closure: wrong number of cells: wanted " + - format(closureVarCount) + - " but got " + - format(mFunctionClosureVarnames.size()) - ); - } - - if (closureVarCount) { - PyObjectStealer closureTup(PyTuple_New(closureVarCount)); - - for (long k = 0; k < closureVarCount; k++) { - std::string varname = mFunctionClosureVarnames[k]; - - auto globalIt = mFunctionGlobalsInCells.find(varname); - if (globalIt != mFunctionGlobalsInCells.end()) { - if (varname == "__class__" && getMethodOf()) { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - PyCell_New((PyObject*)PyInstance::typeObj( - ((HeldClass*)getMethodOf())->getClassType() - )) - ); - } else { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - incref(globalIt->second) - ); - } - } else { - auto bindingIt = mClosureBindings.find(varname); - if (bindingIt == mClosureBindings.end()) { - throw std::runtime_error("Closure variable " + varname + " not in globals or closure bindings."); - } - - ClosureVariableBinding binding = bindingIt->second; - - Instance bindingValue = binding.extractValueOrContainingClosure(closureType, closureData); - - if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catPyCell) { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - incref( - ((PyCellType*)bindingValue.type())->getPyObj(bindingValue.data()) - ) - ); - } else { - // it's not a cell. We have to encode it as a PyCell for the interpreter - // to be able to handle it. - PyObjectHolder newCellContents; - - if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catTypedCell) { - newCellContents.steal( - PyInstance::extractPythonObject( - ((TypedCellType*)bindingValue.type())->get(bindingValue.data()), - ((TypedCellType*)bindingValue.type())->getHeldType() - ) - ); - } else { - newCellContents.steal( - PyInstance::fromInstance(bindingValue) - ); - } - - PyTuple_SetItem( - (PyObject*)closureTup, - k, - PyCell_New(newCellContents) - ); - } - } - } - - if (PyFunction_SetClosure(res, (PyObject*)closureTup) == -1) { - throw PythonExceptionSet(); - } - } - - if (closureType->bytecount() == 0) { - mCachedFunctionObj = incref(res); - } - - return res; -} diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 239733a73..7741baeaa 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2022 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,1550 +22,212 @@ #include "Format.hpp" #include "SpecialModuleNames.hpp" #include "PyInstance.hpp" +#include "ClosureVariableBinding.hpp" +#include "FunctionArg.hpp" +#include "FunctionOverload.hpp" class Function; -class ClosureVariableBindingStep { - enum class BindingType { - FUNCTION = 1, - NAMED_FIELD = 2, - INDEXED_FIELD = 3, - ACCESS_CELL = 4 - }; - - ClosureVariableBindingStep() : - mKind(BindingType::ACCESS_CELL), - mIndexedFieldToAccess(0), - mFunctionToBind(nullptr) - {} - -public: - ClosureVariableBindingStep(Type* bindFunction) : - mKind(BindingType::FUNCTION), - mIndexedFieldToAccess(0), - mFunctionToBind(bindFunction) - {} - - ClosureVariableBindingStep(std::string fieldAccess) : - mKind(BindingType::NAMED_FIELD), - mIndexedFieldToAccess(0), - mFunctionToBind(nullptr), - mNamedFieldToAccess(fieldAccess) - {} - - ClosureVariableBindingStep(int elementAccess) : - mKind(BindingType::INDEXED_FIELD), - mFunctionToBind(nullptr), - mIndexedFieldToAccess(elementAccess) - {} - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - if (isFunction()) { - v.visitHash(ShaHash(1)); - v.visitTopo(getFunction()); - } else - if (isNamedField()) { - v.visitHash(ShaHash(2) + ShaHash(getNamedField())); - } else - if (isIndexedField()) { - v.visitHash(ShaHash(3) + ShaHash(getIndexedField())); - } else - if (isCellAccess()) { - v.visitHash(ShaHash(4)); - } else { - v.visitErr("Invalid ClosureVariableBindingStep found"); - } - } - - static ClosureVariableBindingStep AccessCell() { - ClosureVariableBindingStep step; - return step; - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - if (mKind == BindingType::FUNCTION) { - visitor(mFunctionToBind); - } - } - - bool isFunction() const { - return mKind == BindingType::FUNCTION; - } - - bool isNamedField() const { - return mKind == BindingType::NAMED_FIELD; - } - - bool isIndexedField() const { - return mKind == BindingType::INDEXED_FIELD; - } - - bool isCellAccess() const { - return mKind == BindingType::ACCESS_CELL; - } - - Type* getFunction() const { - if (!isFunction()) { - throw std::runtime_error("Binding is not a function"); - } - - return mFunctionToBind; - } - - std::string getNamedField() const { - if (!isNamedField()) { - throw std::runtime_error("Binding is not a named field bindng"); - } - - return mNamedFieldToAccess; - } - - int getIndexedField() const { - if (!isIndexedField()) { - throw std::runtime_error("Binding is not an index field bindng"); - } - - return mIndexedFieldToAccess; - } - - bool operator<(const ClosureVariableBindingStep& step) const { - if (mKind < step.mKind) { - return true; - } - - if (mKind > step.mKind) { - return false; - } - - if (mKind == BindingType::ACCESS_CELL) { - return false; - } - - if (mKind == BindingType::FUNCTION) { - return mFunctionToBind < step.mFunctionToBind; - } - - if (mKind == BindingType::NAMED_FIELD) { - return mNamedFieldToAccess < step.mNamedFieldToAccess; - } - - if (mKind == BindingType::INDEXED_FIELD) { - return mIndexedFieldToAccess < step.mIndexedFieldToAccess; - } - - return false; - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - - if (mKind == BindingType::ACCESS_CELL) { - buffer.writeUnsignedVarintObject(0, 0); - } - else if (mKind == BindingType::FUNCTION) { - buffer.writeUnsignedVarintObject(0, 1); - context.serializeNativeType(mFunctionToBind, buffer, 1); - } - else if (mKind == BindingType::NAMED_FIELD) { - buffer.writeUnsignedVarintObject(0, 2); - buffer.writeStringObject(1, mNamedFieldToAccess); - } - else if (mKind == BindingType::INDEXED_FIELD) { - buffer.writeUnsignedVarintObject(0, 3); - buffer.writeUnsignedVarintObject(1, mIndexedFieldToAccess); - } - - buffer.writeEndCompound(); - } - - template - static ClosureVariableBindingStep deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - ClosureVariableBindingStep out; - - int whichBinding = -1; - - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber == 0) { - assertWireTypesEqual(wireType, WireType::VARINT); - whichBinding = buffer.readUnsignedVarint(); - - if (whichBinding == 0) { - out = ClosureVariableBindingStep::AccessCell(); - } - } - else if (fieldNumber == 1) { - if (whichBinding == -1) { - throw std::runtime_error("Corrupt ClosureVariableBindingStep"); - } - if (whichBinding == 1) { - out = ClosureVariableBindingStep( - context.deserializeNativeType(buffer, wireType) - ); - } - else if (whichBinding == 2) { - assertWireTypesEqual(wireType, WireType::BYTES); - out = ClosureVariableBindingStep(buffer.readStringObject()); - } - else if (whichBinding == 3) { - assertWireTypesEqual(wireType, WireType::VARINT); - out = ClosureVariableBindingStep(buffer.readUnsignedVarint()); - } - } else { - throw std::runtime_error("Corrupt ClosureVariableBindingStep"); - } - }); - - return out; - } - - -private: - BindingType mKind; - - // this can be a Function or a Forward that will become a function - Type* mFunctionToBind; - - std::string mNamedFieldToAccess; - - int mIndexedFieldToAccess; -}; - - -class ClosureVariableBinding { -public: - ClosureVariableBinding() {} - - ClosureVariableBinding(const std::vector& steps) : - mSteps(new std::vector(steps)) - {} - - ClosureVariableBinding(const std::vector& steps, ClosureVariableBindingStep step) : - mSteps(new std::vector(steps)) - { - mSteps->push_back(step); - } - - ClosureVariableBinding(const ClosureVariableBinding& other) : mSteps(other.mSteps) - {} - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitHash(ShaHash(mSteps->size())); - - for (auto step: *mSteps) { - step._visitCompilerVisibleInternals(v); - } - } - - ClosureVariableBinding& operator=(const ClosureVariableBinding& other) { - mSteps = other.mSteps; - return *this; - } - - ClosureVariableBinding operator+(ClosureVariableBindingStep step) { - if (mSteps) { - return ClosureVariableBinding(*mSteps, step); - } - - return ClosureVariableBinding(std::vector(), step); - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - for (long stepIx = 0; stepIx < size(); stepIx++) { - (*this)[stepIx].serialize(context, buffer, stepIx); - } - buffer.writeEndCompound(); - } - - template - static ClosureVariableBinding deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - std::vector steps; - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - steps.push_back(ClosureVariableBindingStep::deserialize(context, buffer, wireType)); - }); - - return ClosureVariableBinding(steps); - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - if (!mSteps) { - return; - } - - for (auto& step: *mSteps) { - step._visitReferencedTypes(visitor); - } - } - - ClosureVariableBinding withShiftedFrontBinding(long amount) const { - if (!size()) { - throw std::runtime_error("Empty Binding can't be shifted."); - } - - if (!(*this)[0].isIndexedField()) { - throw std::runtime_error("Shifting the first binding only makes sense if it's an indexed lookup"); - } - - std::vector steps; - steps.push_back(ClosureVariableBindingStep((*this)[0].getIndexedField() + amount)); - - for (long k = 1; k < size(); k++) { - steps.push_back((*this)[k]); - } - - return ClosureVariableBinding(steps); - } - - size_t size() const { - if (mSteps) { - return mSteps->size(); - } - - return 0; - } - - bool operator<(const ClosureVariableBinding& other) const { - if (size() < other.size()) { - return true; - } - if (size() > other.size()) { - return false; - } - if (!size()) { - return false; - } - - return *mSteps < *other.mSteps; - } - - ClosureVariableBindingStep operator[](int i) const { - if (i < 0 || i >= size()) { - throw std::runtime_error("ClosureVariableBinding index out of bounds"); - } - - return (*mSteps)[i]; - } - - Instance extractValueOrContainingClosure(Type* closureType, instance_ptr data); - -private: - std::shared_ptr > mSteps; -}; - - -inline ClosureVariableBinding operator+(const ClosureVariableBindingStep& step, const ClosureVariableBinding& binding) { - std::vector steps; - steps.push_back(step); - for (long k = 0; k < binding.size(); k++) { - steps.push_back(binding[k]); - } - return ClosureVariableBinding(steps); -} - -PyDoc_STRVAR(Function_doc, - "Function(f) -> typed function\n" - "\n" - "Converts function f to a typed function.\n" - ); - -class Function : public Type { -public: - class FunctionArg { - public: - FunctionArg(std::string name, Type* typeFilterOrNull, PyObject* defaultValue, bool isStarArg, bool isKwarg) : - m_name(name), - m_typeFilter(typeFilterOrNull), - m_defaultValue(defaultValue), - m_isStarArg(isStarArg), - m_isKwarg(isKwarg) - { - assert(!(isStarArg && isKwarg)); - } - - std::string getName() const { - return m_name; - } - - PyObject* getDefaultValue() const { - return m_defaultValue; - } - - Type* getTypeFilter() const { - return m_typeFilter; - } - - bool getIsStarArg() const { - return m_isStarArg; - } - - bool getIsKwarg() const { - return m_isKwarg; - } - - bool getIsNormalArg() const { - return !m_isKwarg && !m_isStarArg; - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - if (m_typeFilter) { - visitor(m_typeFilter); - } - } - - bool operator<(const FunctionArg& other) const { - if (m_name < other.m_name) { - return true; - } - if (m_name > other.m_name) { - return false; - } - if (m_typeFilter < other.m_typeFilter) { - return true; - } - if (m_typeFilter > other.m_typeFilter) { - return false; - } - if (m_defaultValue < other.m_defaultValue) { - return true; - } - if (m_defaultValue > other.m_defaultValue) { - return false; - } - if (m_isStarArg < other.m_isStarArg) { - return true; - } - if (m_isStarArg > other.m_isStarArg) { - return false; - } - if (m_isKwarg < other.m_isKwarg) { - return true; - } - if (m_isKwarg > other.m_isKwarg) { - return false; - } - - return false; - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - - buffer.writeStringObject(0, m_name); - if (m_typeFilter) { - context.serializeNativeType(m_typeFilter, buffer, 1); - } - if (m_defaultValue) { - context.serializePythonObject(m_defaultValue, buffer, 2); - } - buffer.writeUnsignedVarintObject(3, m_isStarArg ? 1 : 0); - buffer.writeUnsignedVarintObject(4, m_isKwarg ? 1 : 0); - - buffer.writeEndCompound(); - } - - template - static FunctionArg deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - std::string name; - Type* typeFilterOrNull = nullptr; - PyObjectHolder defaultValue; - bool isStarArg = false; - bool isKwarg = false; - - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber == 0) { - assertWireTypesEqual(wireType, WireType::BYTES); - name = buffer.readStringObject(); - } - else if (fieldNumber == 1) { - typeFilterOrNull = context.deserializeNativeType(buffer, wireType); - } - else if (fieldNumber == 2) { - defaultValue.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 3) { - assertWireTypesEqual(wireType, WireType::VARINT); - isStarArg = buffer.readUnsignedVarint(); - } - else if (fieldNumber == 4) { - assertWireTypesEqual(wireType, WireType::VARINT); - isKwarg = buffer.readUnsignedVarint(); - } - }); - - return FunctionArg(name, typeFilterOrNull, defaultValue, isStarArg, isKwarg); - } - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitName(m_name); - if (m_defaultValue) { - v.visitTopo((PyObject*)m_defaultValue); - } else { - v.visitHash(ShaHash()); - } - - if (m_typeFilter) { - v.visitTopo(m_typeFilter); - } else { - v.visitHash(ShaHash()); - } - - v.visitHash( - ShaHash((m_isStarArg ? 2 : 1) + (m_isKwarg ? 10: 11)) - ); - } - - private: - std::string m_name; - Type* m_typeFilter; - PyObjectHolder m_defaultValue; - bool m_isStarArg; - bool m_isKwarg; - }; - - class CompiledSpecialization { - public: - CompiledSpecialization( - compiled_code_entrypoint funcPtr, - Type* returnType, - const std::vector& argTypes - ) : - mFuncPtr(funcPtr), - mReturnType(returnType), - mArgTypes(argTypes) - {} - - compiled_code_entrypoint getFuncPtr() const { - return mFuncPtr; - } - - Type* getReturnType() const { - return mReturnType; - } - - const std::vector& getArgTypes() const { - return mArgTypes; - } - - bool operator==(const CompiledSpecialization& other) const { - return mFuncPtr == other.mFuncPtr - && mReturnType == other.mReturnType - && mArgTypes == other.mArgTypes - ; - } - - private: - compiled_code_entrypoint mFuncPtr; - Type* mReturnType; - std::vector mArgTypes; - }; - - class Overload { - public: - Overload( - PyObject* pyFuncCode, - PyObject* pyFuncGlobals, - PyObject* pyFuncDefaults, - PyObject* pyFuncAnnotations, - const std::map& pyFuncGlobalsInCells, - const std::vector& pyFuncClosureVarnames, - const std::map closureBindings, - Type* returnType, - PyObject* pySignatureFunction, - const std::vector& args, - Type* methodOf - ) : - mFunctionCode(pyFuncCode), - mFunctionGlobals(pyFuncGlobals), - mFunctionDefaults(pyFuncDefaults), - mFunctionAnnotations(pyFuncAnnotations), - mFunctionGlobalsInCells(pyFuncGlobalsInCells), - mFunctionClosureVarnames(pyFuncClosureVarnames), - mReturnType(returnType), - mSignatureFunction(pySignatureFunction), - mArgs(args), - mHasKwarg(false), - mHasStarArg(false), - mMinPositionalArgs(0), - mMaxPositionalArgs(-1), - mClosureBindings(closureBindings), - mCachedFunctionObj(nullptr), - mMethodOf(methodOf) - { - long argsWithDefaults = 0; - long argsDefinitelyConsuming = 0; - - for (auto arg: mArgs) { - if (arg.getIsStarArg()) { - mHasStarArg = true; - } - else if (arg.getIsKwarg()) { - mHasKwarg = true; - } - else if (arg.getDefaultValue()) { - argsWithDefaults++; - } else { - argsDefinitelyConsuming++; - } - } - - mMinPositionalArgs = argsDefinitelyConsuming; - if (!mHasStarArg) { - mMaxPositionalArgs = argsDefinitelyConsuming + argsWithDefaults; - } - - increfAllPyObjects(); - } - - Overload(const Overload& other) { - other.increfAllPyObjects(); - - mFunctionCode = other.mFunctionCode; - mFunctionGlobals = other.mFunctionGlobals; - mFunctionDefaults = other.mFunctionDefaults; - mFunctionAnnotations = other.mFunctionAnnotations; - mSignatureFunction = other.mSignatureFunction; - mMethodOf = other.mMethodOf; - - mFunctionClosureVarnames = other.mFunctionClosureVarnames; - - mClosureBindings = other.mClosureBindings; - mReturnType = other.mReturnType; - mArgs = other.mArgs; - mCompiledSpecializations = other.mCompiledSpecializations; - - mHasStarArg = other.mHasStarArg; - mHasKwarg = other.mHasKwarg; - mMinPositionalArgs = other.mMinPositionalArgs; - mMaxPositionalArgs = other.mMaxPositionalArgs; - - mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; - - mCachedFunctionObj = other.mCachedFunctionObj; - } - - ~Overload() { - decrefAllPyObjects(); - } - - Overload withShiftedFrontClosureBindings(long shiftAmount) const { - std::map bindings; - for (auto nameAndBinding: mClosureBindings) { - bindings[nameAndBinding.first] = nameAndBinding.second.withShiftedFrontBinding(shiftAmount); - } - - return Overload( - mFunctionCode, - mFunctionGlobals, - mFunctionDefaults, - mFunctionAnnotations, - mFunctionGlobalsInCells, - mFunctionClosureVarnames, - bindings, - mReturnType, - mSignatureFunction, - mArgs, - mMethodOf - ); - } - - Overload withMethodOf(Type* methodOf) const { - return Overload( - mFunctionCode, - mFunctionGlobals, - mFunctionDefaults, - mFunctionAnnotations, - mFunctionGlobalsInCells, - mFunctionClosureVarnames, - mClosureBindings, - mReturnType, - mSignatureFunction, - mArgs, - methodOf - ); - } - - Overload withClosureBindings(const std::map &bindings) const { - return Overload( - mFunctionCode, - mFunctionGlobals, - mFunctionDefaults, - mFunctionAnnotations, - mFunctionGlobalsInCells, - mFunctionClosureVarnames, - bindings, - mReturnType, - mSignatureFunction, - mArgs, - mMethodOf - ); - } - - std::string toString() const { - std::ostringstream str; - - str << "("; - - for (long k = 0; k < mArgs.size(); k++) { - if (k) { - str << ", "; - } - - if (mArgs[k].getIsStarArg()) { - str << "*"; - } - - if (mArgs[k].getIsKwarg()) { - str << "**"; - } - - str << mArgs[k].getName(); - - if (mArgs[k].getDefaultValue()) { - str << "=..."; - } - - if (mArgs[k].getTypeFilter()) { - str << ": " << mArgs[k].getTypeFilter()->name(); - } - } - - str << ")"; - - if (mReturnType) { - str << " -> " << mReturnType->name(); - } - - return str.str(); - } - - // return the FunctionArg* that a positional argument would map to, or 'nullptr' if - // it wouldn't - const FunctionArg* argForPositionalArgument(long argIx) const { - if (argIx >= mArgs.size()) { - return nullptr; - } - - if (mArgs[argIx].getIsStarArg() || mArgs[argIx].getIsKwarg()) { - return nullptr; - } - - return &mArgs[argIx]; - } - - // can we possibly match 'argCount' positional arguments? - bool couldMatchPositionalCount(long argCount) const { - return argCount >= mMinPositionalArgs && argCount < mMaxPositionalArgs; - } - - Type* getReturnType() const { - return mReturnType; - } - - const std::vector& getFunctionClosureVarnames() const { - return mFunctionClosureVarnames; - } - - const std::vector& getArgs() const { - return mArgs; - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - // we need to keep mArgs and mReturnType in sync with - // mAnnotations, so if we change one of our types we - // need to update the resulting dictionary as well. - if (mReturnType) { - Type* retType = mReturnType; - - visitor(mReturnType); - - if (mReturnType != retType - && mFunctionAnnotations - && PyDict_Check(mFunctionAnnotations) - ) { - PyDict_SetItemString( - mFunctionAnnotations, - "return", - (PyObject*)PyInstance::typeObj(mReturnType) - ); - } - } - for (auto& a: mArgs) { - Type* typeFilter = a.getTypeFilter(); - - a._visitReferencedTypes(visitor); - - if (a.getTypeFilter() != typeFilter - && mFunctionAnnotations - && PyDict_Check(mFunctionAnnotations) - ) { - PyDict_SetItemString( - mFunctionAnnotations, - a.getName().c_str(), - (PyObject*)PyInstance::typeObj(a.getTypeFilter()) - ); - } - } - - for (auto& varnameAndBinding: mClosureBindings) { - varnameAndBinding.second._visitReferencedTypes(visitor); - } - if (mMethodOf) { - visitor(mMethodOf); - } - } - - template - void _visitCompilerVisibleInternals(const visitor_type& visitor) { - visitor.visitTopo(mFunctionCode); - - if (mFunctionAnnotations) { - visitor.visitHash(ShaHash(2)); - - if (PyDict_CheckExact(mFunctionAnnotations)) { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(mFunctionAnnotations, &pos, &key, &value)) { - visitor.visitTopo(key); - visitor.visitTopo(value); - } - } - - visitor.visitHash(ShaHash(2)); - } - - if (mSignatureFunction) { - visitor.visitHash(ShaHash(1)); - visitor.visitTopo(mSignatureFunction); - } else { - visitor.visitHash(ShaHash(0)); - } - - if (mFunctionDefaults) { - visitor.visitHash(ShaHash(2)); - - if (PyDict_CheckExact(mFunctionDefaults)) { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(mFunctionDefaults, &pos, &key, &value)) { - visitor.visitTopo(key); - visitor.visitTopo(value); - } - } - - visitor.visitHash(ShaHash(2)); - } - - visitor.visitHash(ShaHash(mFunctionGlobalsInCells.size())); - - for (auto nameAndGlobal: mFunctionGlobalsInCells) { - if (!PyCell_Check(nameAndGlobal.second)) { - throw std::runtime_error( - "A global in mFunctionGlobalsInCells is somehow not a cell" - ); - } - - visitor.visitNamedTopo( - nameAndGlobal.first, - nameAndGlobal.second - ); - } - - if (mReturnType) { - visitor.visitTopo(mReturnType); - } else { - visitor.visitHash(ShaHash()); - } - - if (mMethodOf) { - visitor.visitTopo(mMethodOf); - } else { - visitor.visitHash(ShaHash()); - } - - visitor.visitHash(ShaHash(mClosureBindings.size())); - for (auto nameAndClosure: mClosureBindings) { - visitor.visitName(nameAndClosure.first); - nameAndClosure.second._visitCompilerVisibleInternals(visitor); - } - - visitor.visitHash(ShaHash(mArgs.size())); - - for (auto a: mArgs) { - a._visitCompilerVisibleInternals(visitor); - } - - visitor.visitHash(ShaHash(1)); - - _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - visitor.visitNamedTopo(name, val); - }); - - visitor.visitHash(ShaHash(1)); - } - - - template - static void visitCompilerVisibleGlobals( - const visitor_type& visitor, - PyCodeObject* code, - PyObject* globals - ) { - std::vector > dotAccesses; - - extractDottedGlobalAccessesFromCode(code, dotAccesses); - - auto visitSequence = [&](const std::vector& sequence) { - PyObjectHolder curObj; - std::string curName; - - for (PyObject* name: sequence) { - std::string nameStr = PyUnicode_AsUTF8(name); +PyDoc_STRVAR(Function_doc, + "Function(f) -> typed function\n" + "\n" + "Converts function f to a typed function.\n" + ); - if (isSpecialIgnorableName(nameStr)) { - break; - } +class Function : public Type { +public: + Function() : Type(catFunction) + { + } - if (!curObj) { - curName = nameStr; - } else { - curName = curName + "." + nameStr; - } + const char* docConcrete() { + return Function_doc; + } - if (!curObj) { - // this is a lookup in the global dict - curObj.set(PyDict_GetItem(globals, name)); - - if (!curObj) { - // this is an invalid global lookup, which is OK. no need to hash anything. - PyErr_Clear(); - return; - } - } else { - // we're looking up an attribute of this object. We only want to look into modules. - if (PyModule_CheckExact(curObj) && PyObject_HasAttr(curObj, name)) { - PyObjectStealer moduleMember(PyObject_GetAttr(curObj, name)); - - curObj.steal(PyObject_GetAttr(curObj, name)); - if (!curObj) { - // this is an invalid module member lookup. We can just bail. - PyErr_Clear(); - return; - } - } else { - break; - } - } - } + Function(std::string inName, + std::string qualname, + std::string moduleName, + const std::vector& overloads, + Type* closureType, + bool isEntrypoint, + bool isNocompile + ) : + Type(catFunction), + mOverloads(overloads), + mIsEntrypoint(isEntrypoint), + mIsNocompile(isNocompile), + mRootName(inName), + mQualname(qualname), + mClosureType(closureType) + { + m_is_forward_defined = true; + m_name = mRootName; + m_module_name = moduleName; + } - // also visit at the end of the sequence - if (curObj) { - visitor(curName, (PyObject*)curObj); - } - }; + void initializeDuringDeserialization( + std::string inName, + std::string qualname, + std::string moduleName, + std::vector overloads, + Type* closureType, + bool isEntrypoint, + bool isNocompile + ) { + mRootName = inName; + mQualname = qualname; + m_module_name = moduleName; + mOverloads = overloads; + mClosureType = closureType; + mIsEntrypoint = isEntrypoint; + mIsNocompile = isNocompile; + } - for (auto& sequence: dotAccesses) { - visitSequence(sequence); + // does this function have any of its globals that are not + // resolved to actual values. if 'insistForwardsResolved' then we + // return 'true' if the symbol resolves to a forward defined + // type - the symbol is only considered resolved when there are no + // visible forwards + std::string firstUnresolvedSymbol(bool insistResolved) { + for (auto& o: mOverloads) { + std::string res = o.firstUnresolvedSymbol(insistResolved); + if (res.size()) { + return res; } } - template - void _visitCompilerVisibleGlobals(const visitor_type& visitor) { - visitCompilerVisibleGlobals(visitor, (PyCodeObject*)mFunctionCode, mFunctionGlobals); - } - - template - void _visitContainedTypes(const visitor_type& visitor) { - } - - const std::vector& getCompiledSpecializations() const { - return mCompiledSpecializations; - } - - void addCompiledSpecialization(compiled_code_entrypoint e, Type* returnType, const std::vector& argTypes) { - CompiledSpecialization newSpec = CompiledSpecialization(e,returnType,argTypes); + return ""; + } - for (auto& spec: mCompiledSpecializations) { - if (spec == newSpec) { - return; - } + bool hasUnresolvedSymbols(bool insistResolved) { + for (auto& o: mOverloads) { + if (o.hasUnresolvedSymbols(insistResolved)) { + return true; } - - mCompiledSpecializations.push_back(newSpec); - } - - void touchCompiledSpecializations() { - //force the memory for the compiled specializations to move. - std::vector other = mCompiledSpecializations; - std::swap(mCompiledSpecializations, other); } - bool operator<(const Overload& other) const { - if (mFunctionCode < other.mFunctionCode) { return true; } - if (mFunctionCode > other.mFunctionCode) { return false; } - - if (mFunctionGlobals < other.mFunctionGlobals) { return true; } - if (mFunctionGlobals > other.mFunctionGlobals) { return false; } - - if (mFunctionGlobalsInCells < other.mFunctionGlobalsInCells) { return true; } - if (mFunctionGlobalsInCells > other.mFunctionGlobalsInCells) { return false; } - - if (mClosureBindings < other.mClosureBindings) { return true; } - if (mClosureBindings > other.mClosureBindings) { return false; } - - if (mReturnType < other.mReturnType) { return true; } - if (mReturnType > other.mReturnType) { return false; } - - if (mArgs < other.mArgs) { return true; } - if (mArgs > other.mArgs) { return false; } - - if (mMethodOf > other.mMethodOf) { return true; } - if (mMethodOf < other.mMethodOf) { return false; } - - return false; - } + return false; + } - const std::map& getClosureVariableBindings() const { - return mClosureBindings; + void autoresolveGlobals(const std::set& resolvedForwards) { + for (auto& o: mOverloads) { + o.autoresolveGlobals(resolvedForwards); } + } - PyObject* getFunctionCode() const { - return mFunctionCode; + void finalizeTypeConcrete() { + for (auto& o: mOverloads) { + o.finalizeTypeConcrete(); } + } - PyObject* getFunctionDefaults() const { - return mFunctionDefaults; - } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return mRootName; + } - PyObject* getFunctionAnnotations() const { - return mFunctionAnnotations; - } + void postInitializeConcrete() { + m_size = mClosureType->bytecount(); + m_is_default_constructible = mClosureType->is_default_constructible(); + } - Type* getMethodOf() const { - return mMethodOf; - } + void initializeFromConcrete(Type* forwardDef) { + Function* fwdFunc = (Function*)forwardDef; - PyObject* getSignatureFunction() const { - return mSignatureFunction; - } + mClosureType = fwdFunc->mClosureType; + mOverloads = fwdFunc->mOverloads; + mIsEntrypoint = fwdFunc->mIsEntrypoint; + mIsNocompile = fwdFunc->mIsNocompile; + mRootName = fwdFunc->mRootName; + mQualname = fwdFunc->mQualname; + m_module_name = fwdFunc->m_module_name; + } - PyObject* getFunctionGlobals() const { - return mFunctionGlobals; - } + Type* cloneForForwardResolutionConcrete() { + return new Function(); + } - const std::map getFunctionGlobalsInCells() const { - return mFunctionGlobalsInCells; + // replace any direct references to PyObject we're holding internally + // with PyObjSnapshot references instead + void internalizeConstants(const std::map& typeMap) { + for (auto& o: mOverloads) { + o.internalizeConstants(typeMap); } + } - /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. - - In cases where we write something like 'x.y.z' the compiler shouldn't have a reference to 'x', - just to whatever 'x.y.z' refers to. - - This transformation just figures out what the dotting sequences are. - */ - static void extractDottedGlobalAccessesFromCode(PyCodeObject* code, std::vector >& outSequences) { - uint8_t* bytes; - Py_ssize_t bytecount; - - static PyObject* moduleHashName = PyUnicode_FromString("__module_hash__"); - outSequences.push_back(std::vector({moduleHashName})); - - PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); - - long opcodeCount = bytecount / 2; - - // opcodes are encoded in the low byte - auto opcodeFor = [&](int i) { return bytes[i * 2]; }; - - // opcode targets are encoded in the high byte - auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; - - const uint8_t LOAD_ATTR = 106; - const uint8_t LOAD_GLOBAL = 116; - const uint8_t DELETE_GLOBAL = 98; - const uint8_t STORE_GLOBAL = 97; - const uint8_t LOAD_METHOD = 160; - - - std::vector curDotSequence; - for (long ix = 0; ix < opcodeCount; ix++) { - // if we're loading an attr on an existing sequence, just make it bigger - if ((opcodeFor(ix) == LOAD_ATTR || opcodeFor(ix) == LOAD_METHOD) && curDotSequence.size()) { - curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); - } else if (curDotSequence.size()) { - // any other operation should flush the buffer - outSequences.push_back(curDotSequence); - curDotSequence.clear(); - } - - // if we're loading a global, we start a new sequence - if (opcodeFor(ix) == LOAD_GLOBAL) { - curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); - } else if ( - opcodeFor(ix) == STORE_GLOBAL - || opcodeFor(ix) == DELETE_GLOBAL - ) { - outSequences.push_back({PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))}); - } - } - - // flush the buffer if we have something - if (curDotSequence.size()) { - outSequences.push_back(curDotSequence); - } + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(mClosureType, groupMap); - // recurse into sub code objects - iterate(code->co_consts, [&](PyObject* o) { - if (PyCode_Check(o)) { - extractDottedGlobalAccessesFromCode((PyCodeObject*)o, outSequences); - } - }); + for (auto& o: mOverloads) { + o.updateInternalTypePointers(groupMap); } + } - static void extractGlobalAccessesFromCode(PyCodeObject* code, std::set& outAccesses) { - uint8_t* bytes; - Py_ssize_t bytecount; - - PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); - - long opcodeCount = bytecount / 2; - - // opcodes are encoded in the low byte - auto opcodeFor = [&](int i) { return bytes[i * 2]; }; - - // opcode targets are encoded in the high byte - auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; - - const uint8_t LOAD_GLOBAL = 116; - const uint8_t DELETE_GLOBAL = 98; - const uint8_t STORE_GLOBAL = 97; - - for (long ix = 0; ix < opcodeCount; ix++) { - // if we're loading a global, we start a new sequence - if (opcodeFor(ix) == LOAD_GLOBAL) { - PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); - if (!PyUnicode_Check(name)) { - throw std::runtime_error("Function had a non-string object in co_names"); - } - outAccesses.insert(PyUnicode_AsUTF8(name)); - } else if ( - opcodeFor(ix) == STORE_GLOBAL - || opcodeFor(ix) == DELETE_GLOBAL - ) { - PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); - if (!PyUnicode_Check(name)) { - throw std::runtime_error("Function had a non-string object in co_names"); - } - outAccesses.insert(PyUnicode_AsUTF8(name)); - } - } - - // recurse into sub code objects - iterate(code->co_consts, [&](PyObject* o) { - if (PyCode_Check(o)) { - extractGlobalAccessesFromCode((PyCodeObject*)o, outAccesses); - } - }); + static Function* Make( + std::string inName, + std::string qualname, + std::string moduleName, + std::vector overloads, + Type* closureType, + bool isEntrypoint, + bool isNocompile + ) { + bool anyFwd = false; - outAccesses.insert("__module_hash__"); + if (closureType->isForwardDefined()) { + anyFwd = true; } - static void extractNamesFromCode(PyCodeObject* code, std::set& outNames) { - iterate(code->co_names, [&](PyObject* o) { outNames.insert(o); }); - iterate(code->co_freevars, [&](PyObject* o) { outNames.insert(o); }); - - iterate(code->co_consts, [&](PyObject* o) { - if (PyCode_Check(o)) { - extractNamesFromCode((PyCodeObject*)o, outNames); + for (auto& o: overloads) { + o._visitReferencedTypes([&](Type* t) { + if (t->isForwardDefined()) { + anyFwd = true; } }); - static PyObject* moduleHashName = PyUnicode_FromString("__module_hash__"); - outNames.insert(moduleHashName); - } - - void setGlobals(PyObject* globals) { - decref(mFunctionGlobals); - mFunctionGlobals = incref(globals); - } - - PyObject* getUsedGlobals() const { - // restrict the globals to contain only the values it references. - PyObject* result = PyDict_New(); - - std::set allNames; - extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); - - std::set allNamesString; - - for (auto name: allNames) { - if (PyUnicode_Check(name)) { - std::string nameStr = PyUnicode_AsUTF8(name); - - if (mClosureBindings.find(nameStr) == mClosureBindings.end()) { - allNamesString.insert(nameStr); - } - } - } - - // iterate mFunctionGlobals, keeping any where the name is in allNames - // note we split on '.' and take the first part so that if a module - // like lxml.etree is included, and we use 'lxml', we'll take the reference - // to etree as well. This ensures that submodules in anonymously serialized - // code can get pulled along. - PyObject *key, *value; - Py_ssize_t pos = 0; - - // place them in sorted order - std::map > toCopy; - - while (PyDict_Next(mFunctionGlobals, &pos, &key, &value)) { - if (PyUnicode_Check(key)) { - std::string globalName = PyUnicode_AsUTF8(key); - std::string shortGlobalName = globalName; - - size_t indexOfDot = shortGlobalName.find('.'); - if (indexOfDot != std::string::npos) { - shortGlobalName = shortGlobalName.substr(0, indexOfDot); - } - - if (allNamesString.find(shortGlobalName) != allNamesString.end()) { - toCopy[globalName] = std::make_pair(key, value); - } - } - } - - for (auto& nameandKV: toCopy) { - if (PyDict_SetItem(result, nameandKV.second.first, nameandKV.second.second)) { - throw PythonExceptionSet(); - } - } - - PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); - if (builtins) { - PyDict_SetItemString(result, "__builtins__", builtins); - } - - return result; - } - - // create a new function object for this closure (or cache it - // if we have no closure) - PyObject* buildFunctionObj(Type* closureType, instance_ptr closureData) const; - - Overload& operator=(const Overload& other) { - other.increfAllPyObjects(); - decrefAllPyObjects(); - - mFunctionCode = other.mFunctionCode; - mFunctionGlobals = other.mFunctionGlobals; - mFunctionDefaults = other.mFunctionDefaults; - mFunctionAnnotations = other.mFunctionAnnotations; - mSignatureFunction = other.mSignatureFunction; - - mMethodOf = other.mMethodOf; - - mFunctionClosureVarnames = other.mFunctionClosureVarnames; - - mClosureBindings = other.mClosureBindings; - mReturnType = other.mReturnType; - mArgs = other.mArgs; - mCompiledSpecializations = other.mCompiledSpecializations; - - mHasStarArg = other.mHasStarArg; - mHasKwarg = other.mHasKwarg; - mMinPositionalArgs = other.mMinPositionalArgs; - mMaxPositionalArgs = other.mMaxPositionalArgs; - mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; - - mCachedFunctionObj = other.mCachedFunctionObj; - - return *this; - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - - context.serializePythonObject(mFunctionCode, buffer, 0); - - if (mFunctionDefaults) { - context.serializePythonObject(mFunctionDefaults, buffer, 2); - } - - if (mFunctionAnnotations) { - context.serializePythonObject(mFunctionAnnotations, buffer, 3); - } - - buffer.writeBeginCompound(4); - int stringIx = 0; - for (auto varname: mFunctionClosureVarnames) { - buffer.writeStringObject(stringIx++, varname); - } - buffer.writeEndCompound(); - - buffer.writeBeginCompound(5); - int varIx = 0; - for (auto nameAndCell: mFunctionGlobalsInCells) { - buffer.writeStringObject(varIx++, nameAndCell.first); - context.serializePythonObject(nameAndCell.second, buffer, varIx++); - } - buffer.writeEndCompound(); - - buffer.writeBeginCompound(6); - int closureBindingIx = 0; - for (auto nameAndBinding: mClosureBindings) { - buffer.writeStringObject(closureBindingIx++, nameAndBinding.first); - nameAndBinding.second.serialize(context, buffer, closureBindingIx++); - } - buffer.writeEndCompound(); - - if (mReturnType) { - context.serializeNativeType(mReturnType, buffer, 7); - } - - buffer.writeBeginCompound(8); - - int argIx = 0; - for (auto& arg: mArgs) { - arg.serialize(context, buffer, argIx++); - } - - buffer.writeEndCompound(); - - if (mSignatureFunction) { - context.serializePythonObject(mSignatureFunction, buffer, 9); + if (o.hasUnresolvedSymbols(true)) { + anyFwd = true; } - - if (mMethodOf) { - context.serializeNativeType(mMethodOf, buffer, 10); - } - - buffer.writeEndCompound(); } - template - static Overload deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - PyObjectHolder functionCode; - PyObjectHolder functionGlobals; - PyObjectHolder functionAnnotations; - PyObjectHolder functionDefaults; - PyObjectHolder functionSignature; - std::vector closureVarnames; - std::map functionGlobalsInCells; - std::map functionGlobalsInCellsRaw; - std::map closureBindings; - Type* returnType = nullptr; - Type* methodOf = nullptr; - std::vector args; - - functionGlobals.steal(PyDict_New()); - - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber == 0) { - functionCode.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 2) { - functionDefaults.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 3) { - functionAnnotations.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 4) { - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - assertWireTypesEqual(wireType, WireType::BYTES); - closureVarnames.push_back(buffer.readStringObject()); - }); - } - else if (fieldNumber == 5) { - std::string last; - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber % 2 == 0) { - assertWireTypesEqual(wireType, WireType::BYTES); - last = buffer.readStringObject(); - } else { - if (last == "") { - throw std::runtime_error("Corrupt Function closure encountered"); - } - functionGlobalsInCells[last].steal(context.deserializePythonObject(buffer, wireType)); - functionGlobalsInCellsRaw[last] = functionGlobalsInCells[last]; - last = ""; - } - }); - } - else if (fieldNumber == 6) { - std::string last; - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber % 2 == 0) { - assertWireTypesEqual(wireType, WireType::BYTES); - last = buffer.readStringObject(); - } else { - closureBindings[last] = ClosureVariableBinding::deserialize(context, buffer, wireType); - } - }); - } - else if (fieldNumber == 7) { - returnType = context.deserializeNativeType(buffer, wireType); - } - else if (fieldNumber == 8) { - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - args.push_back(FunctionArg::deserialize(context, buffer, wireType)); - }); - } - else if (fieldNumber == 9) { - functionSignature.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 10) { - methodOf = context.deserializeNativeType(buffer, wireType); - } - }); - - return Overload( - functionCode, - functionGlobals, - functionDefaults, - functionAnnotations, - functionGlobalsInCellsRaw, - closureVarnames, - closureBindings, - returnType, - functionSignature, - args, - methodOf + if (anyFwd) { + return new Function( + inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile ); } - void increfAllPyObjects() const { - incref(mFunctionCode); - incref(mFunctionGlobals); - incref(mFunctionDefaults); - incref(mFunctionAnnotations); - incref(mSignatureFunction); - - for (auto nameAndOther: mFunctionGlobalsInCells) { - incref(nameAndOther.second); - } - } - - void decrefAllPyObjects() { - decref(mFunctionCode); - decref(mFunctionGlobals); - decref(mFunctionDefaults); - decref(mFunctionAnnotations); - decref(mSignatureFunction); - - for (auto nameAndOther: mFunctionGlobalsInCells) { - decref(nameAndOther.second); - } - } - - private: - PyObject* mFunctionCode; - - PyObject* mFunctionGlobals; - - // globals that are stored in cells. This happens when class objects - // are defined inside of function scopes. We assume that anything in their - // closure is global (and therefore constant) but it may not be defined yet, - // so we can't just pull the value out and stick it in the function closure - // itself. Each value in the map is guaranteed to be a 'cell' object. - std::map mFunctionGlobalsInCells; - - // the order (by name) of the variables in the __closure__ of the original - // function. This is the order that the python code will expect. - std::vector mFunctionClosureVarnames; - - PyObject* mFunctionDefaults; - - PyObject* mFunctionAnnotations; - - PyObject* mSignatureFunction; - - // note that we are deliberately leaking this value because Overloads get - // stashed in static std::map memos anyways. - mutable PyObject* mCachedFunctionObj; - - std::map mClosureBindings; - - Type* mReturnType; - - // if we are a method of a class, what class? Used to - Type* mMethodOf; - - std::vector mArgs; - - // in compiled code, the closure arguments get passed in front of the - // actual function arguments - std::vector mCompiledSpecializations; - - bool mHasStarArg; - bool mHasKwarg; - size_t mMinPositionalArgs; - size_t mMaxPositionalArgs; - }; - - Function(std::string inName, - std::string qualname, - std::string moduleName, - const std::vector& overloads, - Type* closureType, - bool isEntrypoint, - bool isNocompile - ) : - Type(catFunction), - mOverloads(overloads), - mIsEntrypoint(isEntrypoint), - mIsNocompile(isNocompile), - mRootName(inName), - mQualname(qualname), - mModulename(moduleName) - { - m_is_simple = false; - m_doc = Function_doc; - - mClosureType = closureType; - - _updateAfterForwardTypesChanged(); - endOfConstructorInitialization(); // finish initializing the type object. - } + PyEnsureGilAcquired getTheGil; - std::string moduleNameConcrete() { - if (mModulename.size() == 0) { - return "builtins"; - } + typedef std::tuple< + const std::string, + const std::string, + const std::string, + const std::vector, + Type*, + bool, + bool + > keytype; + + // allocate and leak the memo or else we will crash at process unload time + static std::map *memo = new std::map(); + + keytype key( + inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile + ); - return mModulename; - } + auto it = memo->find(key); - std::string nameWithModuleConcrete() { - if (mModulename.size() == 0) { - return mRootName; + if (it != memo->end()) { + return it->second; } - return mModulename + "." + mRootName; - } - - bool _updateAfterForwardTypesChanged() { - m_name = mRootName; - m_stripped_name = ""; - - m_size = mClosureType->bytecount(); + Function* res = new Function( + inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile + ); - m_is_default_constructible = mClosureType->is_default_constructible(); + Function* concrete = (Function*)res->forwardResolvesTo(); - return false; + (*memo)[key] = concrete; + return concrete; } template @@ -1579,7 +241,7 @@ class Function : public Type { v.visitName(m_name); v.visitName(mRootName); v.visitName(mQualname); - v.visitName(mModulename); + v.visitName(m_module_name); v.visitTopo(mClosureType); @@ -1590,24 +252,6 @@ class Function : public Type { } } - static Function* Make(std::string inName, std::string qualname, std::string moduleName, const std::vector& overloads, Type* closureType, bool isEntrypoint, bool isNocompile) { - PyEnsureGilAcquired getTheGil; - - typedef std::tuple, Type*, bool, bool> keytype; - - static std::map *m = new std::map(); - - auto it = m->find(keytype(inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile)); - if (it == m->end()) { - it = m->insert(std::pair( - keytype(inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile), - new Function(inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile) - )).first; - } - - return it->second; - } - template void _visitContainedTypes(const visitor_type& visitor) { visitor(mClosureType); @@ -1633,7 +277,7 @@ class Function : public Type { types.push_back(t); } - std::vector overloads(f1->mOverloads); + std::vector overloads(f1->mOverloads); for (auto o: f2->mOverloads) { overloads.push_back(o.withShiftedFrontClosureBindings(((Tuple*)f1->getClosureType())->getTypes().size())); } @@ -1641,7 +285,7 @@ class Function : public Type { return Function::Make( f1->mRootName, f1->mQualname, - f1->mModulename, + f1->m_module_name, overloads, Tuple::Make(types), f1->isEntrypoint() || f2->isEntrypoint(), @@ -1658,8 +302,6 @@ class Function : public Type { return mClosureType->cmp(left, right, pyComparisonOp, suppressExceptions); } - - void deepcopyConcrete( instance_ptr dest, instance_ptr src, @@ -1732,7 +374,11 @@ class Function : public Type { return mClosureType->isPOD(); } - const std::vector& getOverloads() const { + const std::vector& getOverloads() const { + return mOverloads; + } + + std::vector& getOverloads() { return mOverloads; } @@ -1780,7 +426,7 @@ class Function : public Type { return this; } - std::vector overloads; + std::vector overloads; for (auto& o: mOverloads) { overloads.push_back(o.withMethodOf(methodOf)); } @@ -1789,11 +435,24 @@ class Function : public Type { } Function* withEntrypoint(bool isEntrypoint) const { - return Function::Make(mRootName, mQualname, mModulename, mOverloads, mClosureType, isEntrypoint, mIsNocompile); + Function* f = Function::Make( + mRootName, mQualname, m_module_name, mOverloads, mClosureType, isEntrypoint, mIsNocompile + ); + + if (f->isForwardDefined() && !isForwardDefined()) { + return (Function*)f->forwardResolvesTo(); + } + return f; } Function* withNocompile(bool isNocompile) const { - return Function::Make(mRootName, mQualname, mModulename, mOverloads, mClosureType, mIsEntrypoint, isNocompile); + Function* f = Function::Make( + mRootName, mQualname, m_module_name, mOverloads, mClosureType, mIsEntrypoint, isNocompile + ); + if (f->isForwardDefined() && !isForwardDefined()) { + return (Function*)f->forwardResolvesTo(); + } + return f; } Type* getClosureType() const { @@ -1851,29 +510,29 @@ class Function : public Type { return true; } - //Function types can be instantiated even if forwards are not resolved - //in their annotations. - void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { - mClosureType->assertForwardsResolvedSufficientlyToInstantiate(); - } + // //Function types can be instantiated even if forwards are not resolved + // //in their annotations. + // void assertForwardsResolvedSufficientlyToInstantiateConcrete() { + // mClosureType->assertForwardsResolvedSufficientlyToInstantiate(); + // } Function* replaceClosure(Type* closureType) const { - return Function::Make(mRootName, mQualname, mModulename, mOverloads, closureType, mIsEntrypoint, mIsNocompile); + return Function::Make(mRootName, mQualname, m_module_name, mOverloads, closureType, mIsEntrypoint, mIsNocompile); } - Function* replaceOverloads(const std::vector& overloads) const { - return Function::Make(mRootName, mQualname, mModulename, overloads, mClosureType, mIsEntrypoint, mIsNocompile); + Function* replaceOverloads(const std::vector& overloads) const { + return Function::Make(mRootName, mQualname, m_module_name, overloads, mClosureType, mIsEntrypoint, mIsNocompile); } Function* replaceOverloadVariableBindings(long index, const std::map& bindings) { - std::vector overloads(mOverloads); + std::vector overloads(mOverloads); if (index < 0 || index >= mOverloads.size()) { throw std::runtime_error("Invalid index to replaceOverloadVariableBindings"); } overloads[index] = overloads[index].withClosureBindings(bindings); - return Function::Make(mRootName, mQualname, mModulename, overloads, mClosureType, mIsEntrypoint, mIsNocompile); + return Function::Make(mRootName, mQualname, m_module_name, overloads, mClosureType, mIsEntrypoint, mIsNocompile); } std::string qualname() const { @@ -1884,12 +543,8 @@ class Function : public Type { return m_name; } - std::string moduleName() const { - return mModulename; - } - private: - std::vector mOverloads; + std::vector mOverloads; Type* mClosureType; @@ -1897,5 +552,5 @@ class Function : public Type { bool mIsNocompile; - std::string mRootName, mQualname, mModulename; + std::string mRootName, mQualname; }; diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index d522cd098..85f2a917e 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -30,27 +30,6 @@ MemberDefinition::MemberDefinition( mDefaultValueAsPyobj = PyInstance::extractPythonObject(mDefaultValue); } -bool HeldClass::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - HeldClass* otherO = (HeldClass*)other; - - if (m_members.size() != otherO->m_members.size()) { - return false; - } - - for (long k = 0; k < m_members.size(); k++) { - if (m_members[k].getName() != otherO->m_members[k].getName() || - !m_members[k].getType()->isBinaryCompatibleWith(otherO->m_members[k].getType())) { - return false; - } - } - - return true; -} - void HeldClass::updateBytesOfInitBits() { bool anyMembersWithInitializers = false; for (auto& m: m_members) { @@ -66,26 +45,6 @@ void HeldClass::updateBytesOfInitBits() { } } -bool HeldClass::_updateAfterForwardTypesChanged() { - m_byte_offsets.clear(); - - updateBytesOfInitBits(); - - size_t size = mBytesOfInitializationBits; - - for (auto t: m_members) { - m_byte_offsets.push_back(size); - size += t.getType()->bytecount(); - } - - bool anyChanged = size != m_size; - - m_is_default_constructible = m_memberFunctions.find("__init__") == m_memberFunctions.end(); - m_size = size; - - return anyChanged; -} - bool HeldClass::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (hasAnyComparisonOperators()) { auto it = getMemberFunctions().find(Class::pyComparisonOpToMethodName(pyComparisonOp)); @@ -441,9 +400,9 @@ HeldClass* HeldClass::Make( const std::map& memberFunctions, const std::map& staticFunctions, const std::map& propertyFunctions, + // TODO: are we accidentally leaking forwards out of this? Probably. const std::map& classMembers, - const std::map& classMethods, - bool isNew + const std::map& classMethods ) { //we only allow one base class to have members because we want native code to be //able to just find those values in subclasses without hitting the vtable. @@ -466,8 +425,7 @@ HeldClass* HeldClass::Make( for (auto nameAndCM: classMembers) { auto it = base->getOwnClassMembers().find(nameAndCM.first); - if (it != base->getOwnClassMembers().end() - && nameAndCM.first != "__typed_python_template__") { + if (it != base->getOwnClassMembers().end()) { // check that they're the same object if (it->second != nameAndCM.second && nameAndCM.first != "__qualname__" && nameAndCM.first != "__module__" @@ -496,6 +454,7 @@ HeldClass* HeldClass::Make( throw PythonExceptionSet(); } + // this is a forward-defined HeldClass HeldClass* result = new HeldClass( "Held(" + inName + ")", bases, @@ -505,16 +464,65 @@ HeldClass* HeldClass::Make( staticFunctions, propertyFunctions, classMembers, - classMethods, - isNew + classMethods ); - // we do these outside of the constructor so that if they throw we - // don't destroy the HeldClass type object (and just leak it instead) because - // we need to ensure we never delete Type objects. - result->initializeMRO(); + Class* clsType = new Class(inName, result); - result->endOfConstructorInitialization(); + // initialize the corresonding Class type + result->setClassType(clsType); + + // check if the forward has any references to forward types. If not, we can + // (and should) resolve it immediately, since our contract is that we only produce forwards + // if we refer to forwards. + bool anyForward = false; + + for (auto h: bases) { + if (h->isForwardDefined()) { + anyForward = true; + } + } + for (auto& m: members) { + if (m.getType() && m.getType()->isForwardDefined()) { + // TODO: could ehte member _instance_ be forward? + anyForward = true; + } + } + + for (auto& nameAndT: memberFunctions) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { + anyForward = true; + } + } + for (auto& nameAndT: staticFunctions) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { + anyForward = true; + } + } + for (auto& nameAndT: propertyFunctions) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { + anyForward = true; + } + } + + // TODO: a classMember could contain a forward reference! + // for (auto& nameAndT: classMembers) { + // if (nameAndT.second->isForwardDefined()) { + // anyForward = true; + // } + // } + + for (auto& nameAndT: classMethods) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { + anyForward = true; + } + } + + if (!anyForward) { + return (HeldClass*)result->forwardResolvesTo(); + } + // we have a forward ref - we have to go through the normal duplicate-and-intern forward + // construction process for this to work. return result; } diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 18ba217a9..ec6687ad1 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -70,9 +70,14 @@ class HashConstCharPtr { } }; + // models a single class member definition class MemberDefinition { public: + MemberDefinition() : mType(nullptr) + { + } + MemberDefinition( const std::string& inName, Type* inType, @@ -337,23 +342,48 @@ class VTable { Type* mClassType; }; + +PyDoc_STRVAR(HeldClass_doc, + "HeldClass: subclass Class and decorate with @Held to produce 'held' TP classes.\n" + "\n" + "These behave like TP classes except that their storage is owned not by the heap but\n" + "by the container of the reference. This means that they are faster (because they don't\n" + "have a refcount) but have different semantics. In particular if H is a held class and\n" + "you write\n\n" + " h1 = H()\n" + " h2 = h1\n" + "\nThen h1 and h2 will refer to different instances of the class.\n" +); + //a class held directly inside of another object class HeldClass : public Type { public: + HeldClass() : + Type(catHeldClass), + m_vtable(new VTable(this)), + m_classType(nullptr), + m_is_final(false), + m_hasComparisonOperators(false), + m_hasGetAttributeMagicMethod(false), + m_hasGetAttrMagicMethod(false), + m_hasSetAttrMagicMethod(false), + m_hasDelAttrMagicMethod(false), + m_hasDelMagicMethod(false), + m_hasConvertFromMagicMethod(false), + m_refToType(nullptr) + { + } + HeldClass(std::string inName, - const std::vector& baseClasses, - bool isFinal, - const std::vector& members, - const std::map& memberFunctions, - const std::map& staticFunctions, - const std::map& propertyFunctions, - const std::map& classMembers, - const std::map& classMethods, - // set to True if this is the first time the class is being created - // and we need to make copies of all the function objects so that - // they know who their methods are. - bool isNew - ) : + const std::vector& baseClasses, + bool isFinal, + const std::vector& members, + const std::map& memberFunctions, + const std::map& staticFunctions, + const std::map& propertyFunctions, + const std::map& classMembers, + const std::map& classMethods + ) : Type(catHeldClass), m_vtable(new VTable(this)), m_bases(baseClasses), @@ -374,21 +404,134 @@ class HeldClass : public Type { m_hasConvertFromMagicMethod(false), m_refToType(nullptr) { + m_is_forward_defined = true; + m_name = inName; - if (isNew) { - for (auto& nameAndF: m_own_memberFunctions) { - nameAndF.second = nameAndF.second->withMethodOf(this); - } + for (auto& nameAndF: m_own_memberFunctions) { + nameAndF.second = nameAndF.second->withMethodOf(this); + } - for (auto& nameAndF: m_own_classMethods) { - nameAndF.second = nameAndF.second->withMethodOf(this); - } + for (auto& nameAndF: m_own_classMethods) { + nameAndF.second = nameAndF.second->withMethodOf(this); + } - for (auto& nameAndF: m_own_staticFunctions) { - nameAndF.second = nameAndF.second->withMethodOf(this); - } + for (auto& nameAndF: m_own_staticFunctions) { + nameAndF.second = nameAndF.second->withMethodOf(this); + } + + initializeMRO(); + } + + void initializeDuringDeserialization( + std::string inName, + const std::vector& bases, + bool isFinal, + const std::vector& members, + const std::map& memberFunctions, + const std::map& staticFunctions, + const std::map& propertyFunctions, + const std::map& classMembers, + const std::map& classMethods, + Class* clsType + ) { + m_name = inName; + m_bases = bases; + m_is_final = isFinal; + m_own_members = members; + m_own_memberFunctions = memberFunctions; + m_own_staticFunctions = staticFunctions; + m_own_propertyFunctions = propertyFunctions; + m_own_classMembers = classMembers; + m_own_classMethods = classMethods; + m_classType = clsType; + } + + void initializeDuringDeserialization( + std::string inName, + const std::vector& bases, + bool isFinal, + const std::vector& members, + const std::map& memberFunctions, + const std::map& staticFunctions, + const std::map& propertyFunctions, + const std::map& classMembers, + const std::map& classMethods, + + const std::vector& own_members, + const std::map& own_memberFunctions, + const std::map& own_staticFunctions, + const std::map& own_propertyFunctions, + const std::map& own_classMembers, + const std::map& own_classMethods, + + Class* clsType + ) { + m_name = inName; + m_bases = bases; + m_is_final = isFinal; + m_own_members = own_members; + m_own_memberFunctions = own_memberFunctions; + m_own_staticFunctions = own_staticFunctions; + m_own_propertyFunctions = own_propertyFunctions; + m_own_classMembers = own_classMembers; + m_own_classMethods = own_classMethods; + + m_members = members; + m_memberFunctions = memberFunctions; + m_staticFunctions = staticFunctions; + m_propertyFunctions = propertyFunctions; + m_classMembers = classMembers; + m_classMethods = classMethods; + + m_classType = clsType; + } + + const char* docConcrete() { + return HeldClass_doc; + } + + void postInitializeConcrete() { + // ensure our bases have their MROs initialized as well + for (auto h: m_bases) { + h->initializeMRO(); } + + initializeMRO(); + + size_t size = mBytesOfInitializationBits; + + for (auto t: m_members) { + m_byte_offsets.push_back(size); + size += t.getType()->bytecount(); + } + + m_size = size; + } + + void initializeFromConcrete(Type* forwardDef) { + HeldClass* fwdCls = (HeldClass*)forwardDef; + + m_name = fwdCls->m_name; + m_classType = fwdCls->m_classType; + m_bases = fwdCls->m_bases; + m_is_final = fwdCls->m_is_final; + m_own_members = fwdCls->m_own_members; + m_own_memberFunctions = fwdCls->m_own_memberFunctions; + m_own_staticFunctions = fwdCls->m_own_staticFunctions; + m_own_propertyFunctions = fwdCls->m_own_propertyFunctions; + m_own_classMembers = fwdCls->m_own_classMembers; + m_own_classMethods = fwdCls->m_own_classMethods; + } + + Type* cloneForForwardResolutionConcrete() { + return new HeldClass(); + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + _visitReferencedTypes([&](Type*& typePtr) { + updateTypeRefFromGroupMap(typePtr, groupMap); + }); } template @@ -437,8 +580,6 @@ class HeldClass : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { for (auto& o: m_members) { @@ -448,62 +589,45 @@ class HeldClass : public Type { template void _visitReferencedTypes(const visitor_type& visitor) { - for (auto& b: m_bases) { - Type* baseT = b; - visitor(baseT); - if (b != baseT) { - throw std::runtime_error("Somehow, we modified the base type of a HeldClass?"); - } + if (m_classType) { + adaptTypeVisitor(visitor, m_classType); + } + + for (long k = 0; k < m_bases.size(); k++) { + adaptTypeVisitor(visitor, m_bases[k]); } + for (auto& o: m_own_members) { // this is expected to actually modify the type // if its a forward. - visitor(o.getType()); + adaptTypeVisitor(visitor, o.getType()); } for (auto& o: m_own_memberFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_own_staticFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_own_classMethods) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_own_propertyFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_members) { - // this is expected to actually modify the type - // if its a forward. - visitor(o.getType()); + adaptTypeVisitor(visitor, o.getType()); } for (auto& o: m_memberFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_staticFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_propertyFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } } - bool _updateAfterForwardTypesChanged(); - static HeldClass* Make( std::string inName, const std::vector& bases, @@ -513,8 +637,7 @@ class HeldClass : public Type { const std::map& staticFunctions, const std::map& propertyFunctions, const std::map& classMembers, - const std::map& classMethods, - bool isNew + const std::map& classMethods ); // this gets called by Class. These types are always produced in pairs. @@ -549,8 +672,7 @@ class HeldClass : public Type { m_own_staticFunctions, m_own_propertyFunctions, m_own_classMembers, - m_own_classMethods, - true + m_own_classMethods ); } @@ -869,10 +991,20 @@ class HeldClass : public Type { // this function can be called multiple times, which happens after the class is // deserialized, since we may be rebuilding the function globals. void mergeOwnFunctionsIntoInheritanceTree() { + if (m_classMembers.size() + || m_memberFunctions.size() + || m_classMethods.size() + || m_propertyFunctions.size() + || m_staticFunctions.size() + ) { + return; + } + m_classMembers.clear(); m_memberFunctions.clear(); m_classMethods.clear(); m_propertyFunctions.clear(); + m_staticFunctions.clear(); for (HeldClass* base: m_mro) { for (auto nameAndObj: base->m_own_classMembers) { @@ -889,8 +1021,13 @@ class HeldClass : public Type { } void initializeMRO() { + if (m_mro.size()) { + return; + } + _computeMroSequence(); + for (size_t i = 0; i < m_mro.size(); i++) { m_ancestor_to_mro_index[m_mro[i]] = i; } @@ -901,7 +1038,16 @@ class HeldClass : public Type { } // build our own method resolution table directly from our parents. - mergeOwnFunctionsIntoInheritanceTree(); + // we only have to do this if we're forward defined since the type + // resolution process will just copy the functions over when it resolves + // them + if (m_is_forward_defined) { + mergeOwnFunctionsIntoInheritanceTree(); + } + + if (m_is_forward_defined) { + return; + } std::set membersSoFar; @@ -938,6 +1084,10 @@ class HeldClass : public Type { } for (HeldClass* ancestor: m_mro) { + if (ancestor->isForwardDefined()) { + throw std::runtime_error("Makes no sense to be a subclass of a forward"); + } + ancestor->m_implementors.insert(this); } @@ -956,6 +1106,10 @@ class HeldClass : public Type { } setMagicMethodExistConstants(); + + m_is_default_constructible = ( + m_memberFunctions.find("__init__") == m_memberFunctions.end() + ); } void updateBytesOfInitBits(); @@ -1068,6 +1222,7 @@ class HeldClass : public Type { ClassDispatchTable* dispatchTableAs(HeldClass* interface) { int64_t offset = getMroIndex(interface); if (offset < 0) { + asm("int3"); throw std::runtime_error("Interface is not an ancestor."); } diff --git a/typed_python/Instance.cpp b/typed_python/Instance.cpp index 234eebfaa..c9e9e4694 100644 --- a/typed_python/Instance.cpp +++ b/typed_python/Instance.cpp @@ -40,6 +40,25 @@ Instance Instance::create(Type*t, instance_ptr data) { t->copy_constructor(tgt, data); }); } + +Instance Instance::create(PyObject* inst) { + if (PyType_Check(inst)) { + return Instance::createAndInitialize( + PythonObjectOfType::AnyPyType(), + [&](instance_ptr i) { + PythonObjectOfType::AnyPyType()->initializeFromPyObject(i, inst); + } + ); + } + + return Instance::createAndInitialize( + PythonObjectOfType::AnyPyObject(), + [&](instance_ptr i) { + PythonObjectOfType::AnyPyObject()->initializeFromPyObject(i, inst); + } + ); +} + Instance Instance::create(Type*t) { t->assertForwardsResolvedSufficientlyToInstantiate(); @@ -59,6 +78,33 @@ Instance::Instance(const Instance& other) : mLayout(other.mLayout) { } } +Instance::Instance(const InstanceRef& other) { + mLayout = nullptr; + + Type* t = other.type(); + instance_ptr p = other.data(); + + if (t->isNone()) { + return; + } + + t->assertForwardsResolvedSufficientlyToInstantiate(); + + layout* l = (layout*)tp_malloc(sizeof(layout) + t->bytecount()); + + try { + t->copy_constructor(l->data, p); + } catch(...) { + tp_free(l); + throw; + } + + l->refcount = 1; + l->type = t; + + mLayout = l; +} + Instance::Instance(instance_ptr p, Type* t) : mLayout(nullptr) { if (t->isNone()) { return; @@ -147,3 +193,34 @@ typed_python_hash_type Instance::hash() const { return mLayout->type->hash(mLayout->data); } + +PyObject* Instance::extractPyobj() const { + return ref().extractPyobj(); +} + +Type* Instance::extractType(bool includePrimitives) const { + return ref().extractType(includePrimitives); +} + +PyObject* InstanceRef::extractPyobj() const { + if (type()->isPythonObjectOfType()) { + return PyObjectHandleTypeBase::getPyObj(data()); + } + + return nullptr; +} + +Type* InstanceRef::extractType(bool includePrimitives) const { + if ( + type()->isPythonObjectOfType() && + PyType_Check( + PyObjectHandleTypeBase::getPyObj(data()) + ) + ) { + PyObject* obj = PyObjectHandleTypeBase::getPyObj(data()); + + return PyInstance::extractTypeFrom((PyTypeObject*)obj, includePrimitives); + } + + return nullptr; +} diff --git a/typed_python/Instance.hpp b/typed_python/Instance.hpp index f12b15421..8dc41bcd8 100644 --- a/typed_python/Instance.hpp +++ b/typed_python/Instance.hpp @@ -53,6 +53,23 @@ class InstanceRef { return mData; } + bool operator<(const InstanceRef& other) const { + if (mType < other.mType) { return true; } + if (mType > other.mType) { return false; } + if (mData < other.mData) { return true; } + return false; + } + + bool operator==(const InstanceRef& other) const { + return mType == other.mType && mData == other.mData; + } + + // if we wrap a Type* as a python object, return it. Otherwise nullptr. + Type* extractType(bool includePrimitives=false) const; + + // if we wrap a PyObject return it. Otherwise nullptr. + PyObject* extractPyobj() const; + private: instance_ptr mData; Type* mType; @@ -81,6 +98,8 @@ class Instance { static Instance create(Type*t, instance_ptr data); + static Instance create(PyObject* o); + static Instance create(Type*t); Instance(); @@ -89,11 +108,23 @@ class Instance { Instance(instance_ptr p, Type* t); + Instance(const InstanceRef& ref); + + InstanceRef ref() const { + return InstanceRef(data(), type()); + } + template static Instance createAndInitialize(Type* t, const initializer_type& initFun) { return Instance(t, initFun); } + Instance clone() const { + return Instance::createAndInitialize(type(), [&](instance_ptr newSelf) { + type()->copy_constructor(newSelf, data()); + }); + } + template Instance(Type* t, const initializer_type& initFun) : mLayout(nullptr) { if (t->isNone()) { @@ -134,6 +165,12 @@ class Instance { return mLayout ? mLayout->type : noneType; } + // if we wrap a Type* as a python object, return it. Otherwise nullptr. + Type* extractType(bool includePrimitives=false) const; + + // if we wrap a PyObject return it. Otherwise nullptr. + PyObject* extractPyobj() const; + template T& cast() { return *(T*)data(); @@ -159,3 +196,16 @@ class Instance { // if the nullptr, then this is the None object. layout* mLayout; }; + +namespace std { + template<> + struct hash { + typedef InstanceRef argument_type; + typedef std::size_t result_type; + + result_type operator()(argument_type const& s) const noexcept { + return std::hash()((void*)s.data()) ^ std::hash()((void*)s.type()); + } + }; +} + diff --git a/typed_python/ModuleRepresentationCopyContext.hpp b/typed_python/ModuleRepresentationCopyContext.hpp index a560c65ab..61b236480 100644 --- a/typed_python/ModuleRepresentationCopyContext.hpp +++ b/typed_python/ModuleRepresentationCopyContext.hpp @@ -207,13 +207,13 @@ class ModuleRepresentationCopyContext { Function* f = (Function*)t; - std::vector overloads; + std::vector overloads; for (auto& o: f->getOverloads()) { - std::vector args; + std::vector args; for (auto& a: o.getArgs()) { args.push_back( - Function::FunctionArg( + FunctionArg( a.getName(), copyType(a.getTypeFilter()), copyObj(a.getDefaultValue()), @@ -224,13 +224,13 @@ class ModuleRepresentationCopyContext { } overloads.push_back( - Function::Overload( + FunctionOverload( o.getFunctionCode(), - copyObj(o.getFunctionGlobals()), copyObj(o.getFunctionDefaults()), copyObj(o.getFunctionAnnotations()), - o.getFunctionGlobalsInCells(), + o.getGlobals(), o.getFunctionClosureVarnames(), + o.getFunctionGlobalsInClosureVarnames(), o.getClosureVariableBindings(), copyType(o.getReturnType()), o.getSignatureFunction(), @@ -306,10 +306,7 @@ class ModuleRepresentationCopyContext { staticFunctions, propertyFunctions, classMembers, - classMethods, - // we want to ensure the class has appropriate - // methodOf for all of its function types - true + classMethods ); forwardC->define(outC); @@ -657,12 +654,11 @@ class ModuleRepresentationCopyContext { } } - reachable.insert(PyObjectHandle(o.getFunctionGlobals())); - - for (auto nameAndObj: o.getFunctionGlobalsInCells()) { - if (nameAndObj.second) { - reachable.insert(PyObjectHandle(nameAndObj.second)); - } + for (auto nameAndGlobal: o.getGlobals()) { + throw std::runtime_error("ModuleRepresentationCopyContext doesn't know what to do with new globals yet"); + // if (nameAndObj.second) { + // reachable.insert(PyObjectHandle(nameAndObj.second)); + // } } if (o.getFunctionAnnotations()) { diff --git a/typed_python/MutuallyRecursiveTypeGroup.cpp b/typed_python/MutuallyRecursiveTypeGroup.cpp index 8396ad601..431fc2fad 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.cpp +++ b/typed_python/MutuallyRecursiveTypeGroup.cpp @@ -9,7 +9,7 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + WITHOUT WARRANTIES OR CONDITIONS OF ANY visibility, either express or implied. See the License for the specific language governing permissions and limitations under the License. ******************************************************************************/ @@ -18,22 +18,30 @@ #include "MutuallyRecursiveTypeGroup.hpp" -MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup() : +MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup(VisibilityType visibility) : mAnyPyObjectsIncorrectlyOrdered(false), - mIsDeserializerGroup(false) + mIsDeserializerGroup(false), + mVisibilityType(visibility) { } -MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup(ShaHash hash) : +MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup( + ShaHash hash, + VisibilityType visibility +) : mAnyPyObjectsIncorrectlyOrdered(false), mIntendedHash(hash), + mVisibilityType(visibility), mIsDeserializerGroup(true) { } -MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::DeserializerGroup(ShaHash hash) { - return new MutuallyRecursiveTypeGroup(hash); +MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::DeserializerGroup( + ShaHash hash, + VisibilityType k +) { + return new MutuallyRecursiveTypeGroup(hash, k); } @@ -77,11 +85,13 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { // and then copy them in MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( - mIndexToObject.begin()->second + mIndexToObject.begin()->second, + mVisibilityType ); MutuallyRecursiveTypeGroup* canonical = MutuallyRecursiveTypeGroup::groupAndIndexFor( - mIndexToObject.begin()->second + mIndexToObject.begin()->second, + mVisibilityType ).first; if (mIndexToObject.size() != canonical->mIndexToObject.size()) { @@ -91,6 +101,8 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { + "\n\nand hashed it, we ended up with this group\n\n" + canonical->repr() + "\n\nwhich has a different number of elements" + + "\n\nmVisibilityType = " + visibilityTypeToStr(mVisibilityType) + + "\ncanonical->mVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) ); } @@ -105,6 +117,8 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { + "\n\nand hashed it, we ended up with this group\n\n" + canonical->repr() + "\n\nwhich has a different hash!" + + "\n\nmVisibilityType = " + visibilityTypeToStr(mVisibilityType) + + "\ncanonical->mVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) ); } @@ -118,8 +132,15 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { std::lock_guard lock(mMutex); // if this intended hash doesn't match anything, just copy it in - if (mIntendedHashToGroup.find(mIntendedHash) == mIntendedHashToGroup.end()) { - mIntendedHashToGroup[mIntendedHash] = this; + if (mIntendedHashToGroup[mVisibilityType].find(mIntendedHash) == mIntendedHashToGroup[mVisibilityType].end()) { + mIntendedHashToGroup[mVisibilityType][mIntendedHash] = this; + } + } + + // now internalize any types in here + for (auto indexAndTopo: mIndexToObject) { + if (indexAndTopo.second.type()) { + indexAndTopo.second.type()->internalize(); } } } @@ -135,24 +156,28 @@ void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { std::lock_guard lock(mMutex); // see if this official hash has been seen yet - if (mHashToGroup.find(mHash) == mHashToGroup.end()) { + if (mHashToGroup[mVisibilityType].find(mHash) == mHashToGroup[mVisibilityType].end()) { // this is the official MRTG for this hash, so we install ourselves // as the official group and register ourselves in the various type memos. // this group will come up in the reverse lookup now. - mHashToGroup[mHash] = this; + mHashToGroup[mVisibilityType][mHash] = this; } for (auto& typeAndOrder: mObjectToIndex) { - auto it = mTypeGroups.find(typeAndOrder.first); + auto it = mTypeGroups[mVisibilityType].find(typeAndOrder.first); - if (it == mTypeGroups.end()) { + if (it == mTypeGroups[mVisibilityType].end()) { // this type has never been installed - go ahead and add it. It's possible // there are multiple copies of this type in the system - mTypeGroups[typeAndOrder.first] = std::make_pair(this, typeAndOrder.second); + mTypeGroups[mVisibilityType][typeAndOrder.first] = std::make_pair(this, typeAndOrder.second); - if (typeAndOrder.first.type()) { + if (typeAndOrder.first.type() && mVisibilityType == VisibilityType::Identity) { Type* t = typeAndOrder.first.type(); - t->setRecursiveTypeGroup(this, typeAndOrder.second); + t->setRecursiveTypeGroup( + this, + typeAndOrder.second, + ShaHash(2) + this->hash() + ShaHash(typeAndOrder.second) + ); } } } @@ -161,9 +186,14 @@ void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { } //static -void MutuallyRecursiveTypeGroup::visibleFrom(TypeOrPyobj root, std::vector& outReachable) { +void MutuallyRecursiveTypeGroup::visibleFrom( + TypeOrPyobj root, + std::vector& outReachable, + VisibilityType visibility +) { CompilerVisibleObjectVisitor::singleton().visit( root, + visibility, [&](ShaHash h) {}, [&](const std::string& s) {}, [&](TypeOrPyobj t) { outReachable.push_back(t); }, @@ -188,16 +218,16 @@ void MutuallyRecursiveTypeGroup::setIndexToObject(int32_t index, TypeOrPyobj obj } } -bool MutuallyRecursiveTypeGroup::objectIsUnassigned(TypeOrPyobj obj) { +bool MutuallyRecursiveTypeGroup::objectIsUnassigned(TypeOrPyobj obj, VisibilityType visibility) { // we can check if we're installed without hitting the lock - if (obj.type() && obj.type()->hasTypeGroup()) { + if (visibility == VisibilityType::Identity && obj.type() && obj.type()->hasTypeGroup()) { return false; } std::lock_guard lock(mMutex); // if we've already installed this group into 'mTypeGroups' - if (mTypeGroups.find(obj) != mTypeGroups.end()) { + if (mTypeGroups[visibility].find(obj) != mTypeGroups[visibility].end()) { return false; } @@ -227,7 +257,8 @@ std::string MutuallyRecursiveTypeGroup::repr(bool deep) { std::string levelPrefix(level, ' '); - s << levelPrefix << "group with hash " << group->hash().digestAsHexString() << ":\n"; + s << levelPrefix << "group with hash " << group->hash().digestAsHexString() << " and " + << "visibility=" << visibilityTypeToStr(mVisibilityType) << ":\n"; // sort lexically and then by level, so that // we can tell what's going on when we have a discrepancy @@ -243,20 +274,22 @@ std::string MutuallyRecursiveTypeGroup::repr(bool deep) { s << levelPrefix << " " << ix << " -> " << obj.name() << "\n"; if (!deep) { - s << CompilerVisibleObjectVisitor::recordWalkAsString(obj) << "\n"; + s << CompilerVisibleObjectVisitor::recordWalkAsString( + obj, mVisibilityType + ) << "\n"; } if (deep) { std::vector visible; - visibleFrom(obj, visible); + visibleFrom(obj, visible, mVisibilityType); for (auto v: visible) { MutuallyRecursiveTypeGroup* subgroup; int ixInSubgroup; - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(v); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(v, mVisibilityType); std::tie(subgroup, ixInSubgroup) = - MutuallyRecursiveTypeGroup::groupAndIndexFor(v); + MutuallyRecursiveTypeGroup::groupAndIndexFor(v, mVisibilityType); if (subgroup) { if (seen.find(subgroup) == seen.end()) { @@ -287,7 +320,10 @@ std::string MutuallyRecursiveTypeGroup::repr(bool deep) { // were unable to uniquely pick it. // static -std::pair MutuallyRecursiveTypeGroup::computeRoot(const std::set& topos) { +std::pair MutuallyRecursiveTypeGroup::computeRoot( + const std::set& topos, + VisibilityType visibility +) { // we look at python objects in this function, so we need to be holding the GIL PyEnsureGilAcquired getTheGil; @@ -387,13 +423,14 @@ std::pair MutuallyRecursiveTypeGroup::computeRoot(const std:: CompilerVisibleObjectVisitor::singleton().visit( t, + visibility, [&](ShaHash h) { newHash += h; }, [&](const std::string& s) { newHash += ShaHash(s); }, [&](TypeOrPyobj t) { if (curHashes.find(t) != curHashes.end()) { newHash += curHashes[t]; } else { - newHash += shaHash(t); + newHash += shaHash(t, visibility); } }, [&](const std::string& s, TypeOrPyobj t) { @@ -402,7 +439,7 @@ std::pair MutuallyRecursiveTypeGroup::computeRoot(const std:: if (curHashes.find(t) != curHashes.end()) { newHash += curHashes[t]; } else { - newHash += shaHash(t); + newHash += shaHash(t, visibility); } }, [&](const std::string& err) {} @@ -435,7 +472,10 @@ std::pair MutuallyRecursiveTypeGroup::computeRoot(const std:: } // static -void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(const std::set& topos) { +void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup( + const std::set& topos, + VisibilityType visibility +) { // check to see if any of our topos has been installed in a type group already. If so, // then they ALL should have been installed in a type group. This can happen if two threads // are computing type groups at the same time: imagine a cycle of many objects - both @@ -447,7 +487,7 @@ void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(const std::setmAnyPyObjectsIncorrectlyOrdered) = computeRoot(topos); + std::tie(root, group->mAnyPyObjectsIncorrectlyOrdered) = computeRoot(topos, visibility); std::map ordering; @@ -493,6 +533,7 @@ void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(const std::set lock(mMutex); - auto it = mHashToGroup.find(hash); - if (it == mHashToGroup.end()) { + auto it = mHashToGroup[visibility].find(hash); + if (it == mHashToGroup[visibility].end()) { return nullptr; } return it->second; } -MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::getGroupFromIntendedHash(ShaHash hash) { +MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::getGroupFromIntendedHash(ShaHash hash, VisibilityType visibility) { std::lock_guard lock(mMutex); - auto it = mIntendedHashToGroup.find(hash); - if (it != mIntendedHashToGroup.end()) { + auto it = mIntendedHashToGroup[visibility].find(hash); + if (it != mIntendedHashToGroup[visibility].end()) { return it->second; } - auto it2 = mHashToGroup.find(hash); - if (it2 != mHashToGroup.end()) { + auto it2 = mHashToGroup[visibility].find(hash); + if (it2 != mHashToGroup[visibility].end()) { return it2->second; } @@ -555,7 +596,7 @@ void MutuallyRecursiveTypeGroup::computeHash() { thread_local static MutuallyRecursiveTypeGroup* currentlyHashing = nullptr; if (currentlyHashing) { - CompilerVisibleObjectVisitor::singleton().checkForInstability(); + CompilerVisibleObjectVisitor::singleton().checkForInstability(mVisibilityType); std::ostringstream errMsg; @@ -615,17 +656,20 @@ ShaHash MutuallyRecursiveTypeGroup::computeTopoHash(TypeOrPyobj toHash) { return; } - auto groupAndIx = groupAndIndexFor(o); + auto groupAndIx = groupAndIndexFor(o, mVisibilityType); if (!groupAndIx.first || groupAndIx.second == -1) { - CompilerVisibleObjectVisitor::singleton().checkForInstability(); + CompilerVisibleObjectVisitor::singleton().checkForInstability(mVisibilityType); throw std::runtime_error( "All reachable objects should be in this group or hashed already. " "Instead, this object is not hashed and not local: " + o.name() + (groupAndIx.first ? "\n\nhas a group but not an index\n\n":"") + - + "\n\nwithin\n\n" + toHash.name() + "\n\nwalk is\n" + CompilerVisibleObjectVisitor::recordWalkAsString(toHash) - + "\n\nwalk of not hashed is\n" + CompilerVisibleObjectVisitor::recordWalkAsString(o) + + "\n\nwithin\n\n" + toHash.name() + "\n\nwalk is\n" + + CompilerVisibleObjectVisitor::recordWalkAsString(toHash, mVisibilityType) + + "\n\nwalk of not hashed is\n" + + CompilerVisibleObjectVisitor::recordWalkAsString(o, mVisibilityType) + + "\nVT = " + visibilityTypeToStr(mVisibilityType) ); } @@ -635,6 +679,7 @@ ShaHash MutuallyRecursiveTypeGroup::computeTopoHash(TypeOrPyobj toHash) { // walk one layer deep into our objects CompilerVisibleObjectVisitor::singleton().visit( toHash, + mVisibilityType, [&](ShaHash h) { res += ShaHash(1) + h; }, [&](std::string o) { res += ShaHash(2) + ShaHash(o); }, [&](TypeOrPyobj o) { @@ -654,7 +699,9 @@ ShaHash MutuallyRecursiveTypeGroup::computeTopoHash(TypeOrPyobj toHash) { // compiler is concerned) class MutuallyRecursiveTypeGroupSearch { public: - MutuallyRecursiveTypeGroupSearch() {} + MutuallyRecursiveTypeGroupSearch(VisibilityType visibility) : + mVisibilityType(visibility) + {} // this object is new and needs its own group void pushGroup(TypeOrPyobj o) { @@ -681,15 +728,16 @@ class MutuallyRecursiveTypeGroupSearch { // now recurse into the subtypes CompilerVisibleObjectVisitor::singleton().visit( o, + mVisibilityType, [&](ShaHash h) {}, [&](std::string o) {}, [&](TypeOrPyobj o) { - if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o)) { + if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o, mVisibilityType)) { mGroupOutboundEdges.back()->push_back(o); } }, [&](std::string name, TypeOrPyobj o) { - if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o)) { + if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o, mVisibilityType)) { mGroupOutboundEdges.back()->push_back(o); } }, @@ -706,7 +754,7 @@ class MutuallyRecursiveTypeGroupSearch { void doOneStep() { if (mGroupOutboundEdges.back()->size() == 0) { // this group is finished.... - MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(*mGroups.back()); + MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(*mGroups.back(), mVisibilityType); for (auto gElt: *mGroups.back()) { mInAGroup.erase(gElt); @@ -717,9 +765,10 @@ class MutuallyRecursiveTypeGroupSearch { } else { // pop an outbound edge and see where does it go? TypeOrPyobj o = mGroupOutboundEdges.back()->back(); + mGroupOutboundEdges.back()->pop_back(); - if (!MutuallyRecursiveTypeGroup::objectIsUnassigned(o)) { + if (!MutuallyRecursiveTypeGroup::objectIsUnassigned(o, mVisibilityType)) { return; } @@ -760,14 +809,16 @@ class MutuallyRecursiveTypeGroupSearch { // for each group, the set of things it reaches out to std::vector > > mGroupOutboundEdges; + + VisibilityType mVisibilityType; }; // static -void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root) { +void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root, VisibilityType visibility) { // we do things with pyobj refcounts, so we need to hold the gil. PyEnsureGilAcquired getTheGil; - MutuallyRecursiveTypeGroupSearch groupFinder; + MutuallyRecursiveTypeGroupSearch groupFinder(visibility); static thread_local int count = 0; count++; @@ -788,12 +839,12 @@ void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root) { count--; } -ShaHash MutuallyRecursiveTypeGroup::shaHash(TypeOrPyobj o) { +ShaHash MutuallyRecursiveTypeGroup::shaHash(TypeOrPyobj o, VisibilityType visibility) { if (o.pyobj() && CompilerVisibleObjectVisitor::isSimpleConstant(o.pyobj())) { return shaHashOfSimplePyObjectConstant(o.pyobj()); } - auto groupAndIx = MutuallyRecursiveTypeGroup::groupAndIndexFor(o); + auto groupAndIx = MutuallyRecursiveTypeGroup::groupAndIndexFor(o, visibility); if (!groupAndIx.first) { return ShaHash(); @@ -803,16 +854,21 @@ ShaHash MutuallyRecursiveTypeGroup::shaHash(TypeOrPyobj o) { } // static -std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor(PyObject* o) { - return groupAndIndexFor(TypeOrPyobj::withoutIntern(o)); +std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor( + PyObject* o, VisibilityType visibility +) { + return groupAndIndexFor(TypeOrPyobj::withoutIntern(o), visibility); } // static -std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor(TypeOrPyobj o) { +std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor( + TypeOrPyobj o, + VisibilityType visibility +) { std::lock_guard lock(mMutex); - auto it = mTypeGroups.find(o); - if (it != mTypeGroups.end()) { + auto it = mTypeGroups[visibility].find(o); + if (it != mTypeGroups[visibility].end()) { return std::pair( it->second.first, it->second.first->indexOfObjectInThisGroup(o) ); @@ -1004,13 +1060,22 @@ ShaHash MutuallyRecursiveTypeGroup::shaHashOfSimplePyObjectConstant(PyObject* h) } //static -std::unordered_map > MutuallyRecursiveTypeGroup::mTypeGroups; +std::map< + VisibilityType, + std::unordered_map > +> MutuallyRecursiveTypeGroup::mTypeGroups; //static -std::map MutuallyRecursiveTypeGroup::mHashToGroup; +std::map< + VisibilityType, + std::map +> MutuallyRecursiveTypeGroup::mHashToGroup; //static -std::map MutuallyRecursiveTypeGroup::mIntendedHashToGroup; +std::map< + VisibilityType, + std::map +> MutuallyRecursiveTypeGroup::mIntendedHashToGroup; //static std::recursive_mutex MutuallyRecursiveTypeGroup::mMutex; diff --git a/typed_python/MutuallyRecursiveTypeGroup.hpp b/typed_python/MutuallyRecursiveTypeGroup.hpp index ec0af210b..8ea020c80 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.hpp +++ b/typed_python/MutuallyRecursiveTypeGroup.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2020 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,17 +18,47 @@ #include "TypeOrPyobj.hpp" #include "ShaHash.hpp" +#include "VisibilityType.hpp" + #include #include class MutuallyRecursiveTypeGroupSearch; +/***** +MutuallyRecursiveTypeGroup (or MRTG) provides infrstructure for hashing groups of python +type and module objects. The primary use cases for this infrastructure are for identifying +truly identical Type objects that may have been constructed separately (either via deserialization +or directly via the API) and for hashing Type and function objects so that the compiler cache +can formally identify that a function will compile to the same code. + +We have multiple angles at which we can look at an object. The simplest is 'identity', where +we wish to associate a hash with an object so that we are guaranteed that two objects +with the same hash will behave identically (not including pointer identity). For instance, +two instances of the same function object visible at module-level scope in a globally visible +module would be 'identical', even if the module object changes. + +The more stringent hash is the 'compiler' hash, in which two objects that have the same +hash would look identical to the compiler now and at all points in the future. In this model, +we need to deeply walk into the object in the same manner the compiler would. For this to +work, we make the same assumption that we make in the main compiler: once we have finished +the module import process, all of the objects defined in the module are fully defined. +In particular this means that the identify of module members will not chnage (essentially, +that the module's dict is immutable), and that class members of classes defined in that module +will not change (the class dicts are immutable). Mutable objects such as module-level lists +may change during runtime, and so we don't consider them part of the compiler hash. + +In order to hash python objects, we have to find groups of mutually recursive objects. As +an example, if 'f' and 'g' are mutually recursive functions, then if you change either of +their code objects, then both of their compiler hashes should change. To this end, we define +the MutuallyRecursiveTypeGroup (MRTG), which builds the DAG of object cycles. The MRTGs +are defined either as Identity or Compiler groups, depending on the use case. Objects +found in MRTGs are memoized in the program and won't ever be released, so be careful what +you hash. + +******/ -// represents a group of Type* and PyObject* instances that can 'see' each other according -// to the compiler. These instances need to be serialized and hashed as a group. The hash -// of the instances depends on the hash of the group, and the hash of the group depends on -// the set of instances contained inside of it. class MutuallyRecursiveTypeGroup { public: //return an empty 'builder' group. Builder groups can be built up and then @@ -41,7 +71,7 @@ class MutuallyRecursiveTypeGroup { //Regardless, you're guaranteed that after you finalize this group, there will be only //one copy of the types or singleton objects it contains. The types you put in may be //discarded, so don't keep a reference to them. - static MutuallyRecursiveTypeGroup* DeserializerGroup(ShaHash intendedHash); + static MutuallyRecursiveTypeGroup* DeserializerGroup(ShaHash intendedHash, VisibilityType visibility); bool isDeserializerGroup() const { return mIsDeserializerGroup; @@ -61,6 +91,10 @@ class MutuallyRecursiveTypeGroup { return mHash; } + VisibilityType visibilityType() const { + return mVisibilityType; + } + const std::map& getIndexToObject() const { return mIndexToObject; } @@ -68,44 +102,44 @@ class MutuallyRecursiveTypeGroup { std::string repr(bool deep=false); // if we have an official MRTG for this hash, return it. - static MutuallyRecursiveTypeGroup* getGroupFromHash(ShaHash h); + static MutuallyRecursiveTypeGroup* getGroupFromHash(ShaHash h, VisibilityType k); // if we have an MRTG with this 'intended' hash, return that. - static MutuallyRecursiveTypeGroup* getGroupFromIntendedHash(ShaHash h); + static MutuallyRecursiveTypeGroup* getGroupFromIntendedHash(ShaHash h, VisibilityType k); // construct an MRTG on this. After this call, this object will be in the type memo // and calls to 'groupAndIndexFor' and shaHash will succeed - static void ensureRecursiveTypeGroup(TypeOrPyobj root); + static void ensureRecursiveTypeGroup(TypeOrPyobj root, VisibilityType k); // return the current group head, assuming one has been created. If you haven't called // ensureRecursiveTypeGroup, you may get a nullptr for the group. Note that creating // a TypeOrPyobj will leak a reference to 'o', so don't do it on any old object. - static std::pair groupAndIndexFor(TypeOrPyobj o); + static std::pair groupAndIndexFor(TypeOrPyobj o, VisibilityType k); // lookup an object without increffing it - static std::pair groupAndIndexFor(PyObject* o); + static std::pair groupAndIndexFor(PyObject* o, VisibilityType k); // return the current sha-hash for this object, or ShaHash() if its not in a type // group yet. - static ShaHash shaHash(TypeOrPyobj t); + static ShaHash shaHash(TypeOrPyobj t, VisibilityType k); // ensure this object is in a type group and then return the set of reachable instances - static void visibleFrom(TypeOrPyobj root, std::vector& outReachable); + static void visibleFrom(TypeOrPyobj root, std::vector& outReachable, VisibilityType k); private: - MutuallyRecursiveTypeGroup(); + MutuallyRecursiveTypeGroup(VisibilityType visibility); - MutuallyRecursiveTypeGroup(ShaHash hash); + MutuallyRecursiveTypeGroup(ShaHash hash, VisibilityType visibility); static ShaHash shaHashOfSimplePyObjectConstant(PyObject* h); - static bool objectIsUnassigned(TypeOrPyobj obj); + static bool objectIsUnassigned(TypeOrPyobj obj, VisibilityType visibility); ShaHash computeTopoHash(TypeOrPyobj t); void _computeHashAndInstall(); - static void buildCompilerRecursiveGroup(const std::set& types); + static void buildCompilerRecursiveGroup(const std::set& types, VisibilityType visibility); static void installTypeHash(Type* t); @@ -113,7 +147,10 @@ class MutuallyRecursiveTypeGroup { // given a collection of TypeOrPyobj, figure out which one is the 'root' for the group // this must be stable regardless of the ordering. - static std::pair computeRoot(const std::set& types); + static std::pair computeRoot( + const std::set& types, + VisibilityType visibility + ); // static mutex that guards all the relevant data structures. Changes to all static // structures below must be made while holding the mutex. You may not hold the mutex @@ -121,11 +158,17 @@ class MutuallyRecursiveTypeGroup { static std::recursive_mutex mMutex; // for each python object that's in a type group, the group and index within the group - static std::unordered_map > mTypeGroups; + static std::map< + VisibilityType, + std::unordered_map > + > mTypeGroups; - static std::map mIntendedHashToGroup; + static std::map< + VisibilityType, + std::map + > mIntendedHashToGroup; - static std::map mHashToGroup; + static std::map > mHashToGroup; void computeHash(); @@ -133,6 +176,9 @@ class MutuallyRecursiveTypeGroup { // if not, we can't modify it and it may or may not be officially installed. bool mIsDeserializerGroup; + // what kind of group are we + VisibilityType mVisibilityType; + // the sha hash of this group within this codebase ShaHash mHash; diff --git a/typed_python/NoneType.hpp b/typed_python/NoneType.hpp index 4d5751f42..78be0443a 100644 --- a/typed_python/NoneType.hpp +++ b/typed_python/NoneType.hpp @@ -25,16 +25,6 @@ class NoneType : public Type { m_name = "None"; m_size = 0; m_is_default_constructible = true; - - endOfConstructorInitialization(); // finish initializing the type object. - } - - bool isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; } template @@ -97,4 +87,6 @@ class NoneType : public Type { void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { stream << "None"; } + + void postInitializeConcrete() {} }; diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index db1593dfa..2c83a7d58 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -16,79 +16,6 @@ #include "AllTypes.hpp" -bool OneOfType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != TypeCategory::catOneOf) { - return false; - } - - OneOfType* otherO = (OneOfType*)other; - - if (m_types.size() != otherO->m_types.size()) { - return false; - } - - for (long k = 0; k < m_types.size(); k++) { - if (!m_types[k]->isBinaryCompatibleWith(otherO->m_types[k])) { - return false; - } - } - - return true; -} - -bool OneOfType::_updateAfterForwardTypesChanged() { - size_t size = computeBytecount(); - std::string name = computeName(); - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool is_default_constructible = false; - - for (auto typePtr: m_types) { - if (typePtr->is_default_constructible()) { - is_default_constructible = true; - break; - } - } - - bool anyChanged = ( - size != m_size || - name != m_name || - is_default_constructible != m_is_default_constructible - ); - - m_size = size; - m_stripped_name = ""; - m_name = name; - m_stripped_name = ""; - - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - -std::string OneOfType::computeName() const { - std::string res = "OneOf("; - - bool first = true; - - for (auto t: m_types) { - if (first) { - first = false; - } else { - res += ", "; - } - - res += t->name(true); - } - - res += ")"; - - return res; -} - void OneOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { m_types[*((uint8_t*)self)]->repr(self+1, stream, isStr); } @@ -156,34 +83,119 @@ void OneOfType::assign(instance_ptr self, instance_ptr other) { } // static -OneOfType* OneOfType::Make(const std::vector& types, OneOfType* knownType) { - std::vector flat_typelist; - std::set seen; - - //make sure we only get each type once and don't have any other 'OneOfType' in there... - std::function)> visit = [&](const std::vector& subvec) { - for (auto t: subvec) { - if (t->getTypeCategory() == catOneOf) { - visit( ((OneOfType*)t)->getTypes() ); - } else if (seen.find(t) == seen.end()) { - flat_typelist.push_back(t); - seen.insert(t); - } +OneOfType* OneOfType::Make(const std::vector& types) { + bool anyForward = false; + + for (auto t: types) { + if (t->isForwardDefined()) { + anyForward = true; } - }; + } - visit(types); + if (anyForward) { + return new OneOfType(types); + } PyEnsureGilAcquired getTheGil; typedef const std::vector keytype; + static std::map memo; - static std::map m; + auto it = memo.find(types); + if (it != memo.end()) { + return it->second; + } - auto it = m.find(flat_typelist); - if (it == m.end()) { - it = m.insert(std::make_pair(flat_typelist, knownType ? knownType : new OneOfType(flat_typelist))).first; + OneOfType* res = new OneOfType(types); + OneOfType* concrete = (OneOfType*)res->forwardResolvesTo(); + + memo[types] = concrete; + return concrete; +} + +Type* OneOfType::cloneForForwardResolutionConcrete() { + // create a 'blank' oneof type + return new OneOfType(); +} + +void OneOfType::initializeFromConcrete( + Type* forwardDefinitionOfSelf +) { + OneOfType* selfT = (OneOfType*)forwardDefinitionOfSelf; + + m_types = selfT->m_types; +} + +void OneOfType::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + std::vector newTypes; + std::set seenTypes; + std::set everVisited; + + std::function visit = [&](Type* t) { + if (everVisited.find(t) != everVisited.end()) { + return; + } + everVisited.insert(t); + + if (t->getTypeCategory() == catOneOf) { + for (auto subt: ((OneOfType*)t)->getTypes()) { + visit(subt); + } + } else { + auto it = groupMap.find(t); + if (it != groupMap.end() && it->second != t) { + visit(it->second); + } else { + if (seenTypes.find(t) == seenTypes.end()) { + newTypes.push_back(t); + seenTypes.insert(t); + } + } + } + }; + + for (auto t: m_types) { + visit(t); + } + + m_types = newTypes; + + if (m_types.size() > 255) { + throw std::runtime_error("OneOf types are limited to 255 alternatives in this implementation"); + } +} + +void OneOfType::postInitializeConcrete() { + m_size = computeBytecount(); + + m_is_default_constructible = false; + + for (auto typePtr: m_types) { + if (typePtr->is_default_constructible()) { + m_is_default_constructible = true; + break; + } + } +} + +std::string OneOfType::computeRecursiveNameConcrete(TypeStack& typeStack) { + std::string res = "OneOf("; + + bool first = true; + + for (auto t: m_types) { + if (first) { + first = false; + } else { + res += ", "; + } + + res += t->computeRecursiveName(typeStack); } - return it->second; + res += ")"; + + return res; } diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 2dbc43f1d..a9fa712b4 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -26,17 +26,23 @@ PyDoc_STRVAR(OneOf_doc, class OneOfType : public Type { public: - OneOfType(const std::vector& types) noexcept : - Type(TypeCategory::catOneOf), - m_types(types) + // clone initialization + OneOfType() noexcept : + Type(TypeCategory::catOneOf) { - if (m_types.size() > 255) { - throw std::runtime_error("OneOf types are limited to 255 alternatives in this implementation"); - } + } - m_doc = OneOf_doc; + // forward initialization + OneOfType(const std::vector& types) noexcept : + Type(TypeCategory::catOneOf), + m_types(types) + { + m_is_forward_defined = true; + recomputeName(); + } - endOfConstructorInitialization(); // finish initializing the type object. + const char* docConcrete() { + return OneOf_doc; } template @@ -49,8 +55,6 @@ class OneOfType : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { for (auto& typePtr: m_types) { @@ -63,8 +67,6 @@ class OneOfType : public Type { _visitContainedTypes(visitor); } - bool _updateAfterForwardTypesChanged(); - bool isPODConcrete() { for (auto t: m_types) { if (!t->isPOD()) { @@ -75,8 +77,6 @@ class OneOfType : public Type { return true; } - std::string computeName() const; - void deepcopyConcrete( instance_ptr dest, instance_ptr src, @@ -147,11 +147,23 @@ class OneOfType : public Type { return m_types; } - void _updateTypeMemosAfterForwardResolution() { - OneOfType::Make(m_types, this); + static OneOfType* Make(const std::vector& types); + + void initializeDuringDeserialization(const std::vector& types) { + m_types = types; } - static OneOfType* Make(const std::vector& types, OneOfType* knownType = nullptr); + Type* cloneForForwardResolutionConcrete(); + + void initializeFromConcrete(Type* forwardDefinitionOfSelf); + + void postInitializeConcrete(); + + std::string computeRecursiveNameConcrete(TypeStack& stack); + + void updateInternalTypePointersConcrete( + const std::map& groupMap + ); private: std::vector m_types; diff --git a/typed_python/PointerToType.hpp b/typed_python/PointerToType.hpp index 127845f7e..8c2564df4 100644 --- a/typed_python/PointerToType.hpp +++ b/typed_python/PointerToType.hpp @@ -35,22 +35,31 @@ PyDoc_STRVAR(PointerTo_doc, "The statement p[3]=v is like the C++ statement 'p[3]=v;'\n" "The expression p+3 is of type PointerTo(T), like the C++ expression 'p+3'\n" "The expression p1-p2 is of type int, like the C++ expression 'p1-p2'\n" - ); +); class PointerTo : public Type { -protected: +public: typedef void* instance; -public: + // construct a non-forward defined pointer + PointerTo() : Type(TypeCategory::catPointerTo) + { + } + PointerTo(Type* t) : Type(TypeCategory::catPointerTo), m_element_type(t) { - m_size = sizeof(instance); - m_is_default_constructible = true; - m_doc = PointerTo_doc; + m_is_forward_defined = true; + recomputeName(); + } - endOfConstructorInitialization(); // finish initializing the type object. + void initializeDuringDeserialization(Type* eltType) { + m_element_type = eltType; + } + + const char* docConcrete() { + return PointerTo_doc; } template @@ -59,40 +68,53 @@ class PointerTo : public Type { v.visitTopo(m_element_type); } - void _updateTypeMemosAfterForwardResolution() { - PointerTo::Make(m_element_type, this); - } + static PointerTo* Make(Type* elt) { + if (elt->isForwardDefined()) { + return new PointerTo(elt); + } - static PointerTo* Make(Type* elt, PointerTo* knownType = nullptr) { PyEnsureGilAcquired getTheGil; - static std::map m; + static std::map memo; - auto it = m.find(elt); - if (it == m.end()) { - it = m.insert(std::make_pair(elt, knownType ? knownType : new PointerTo(elt))).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + PointerTo* res = new PointerTo(elt); + PointerTo* concrete = (PointerTo*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } bool isPODConcrete() { return true; } - bool _updateAfterForwardTypesChanged() { - std::string name = "PointerTo(" + m_element_type->name(true) + ")"; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "PointerTo(" + m_element_type->computeRecursiveName(typeStack) + ")"; + } - if (m_is_recursive_forward) { - name = m_recursive_name; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_element_type = ((PointerTo*)forwardDefinitionOfSelf)->m_element_type; + } - bool anyChanged = name != m_name; + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it = groupMap.find(m_element_type); + if (it != groupMap.end()) { + m_element_type = it->second; + } + } - m_name = name; - m_stripped_name = ""; + Type* cloneForForwardResolutionConcrete() { + return new PointerTo(); + } - return anyChanged; + void postInitializeConcrete() { + m_size = sizeof(instance); + m_is_default_constructible = true; } template diff --git a/typed_python/PyAlternativeInstance.cpp b/typed_python/PyAlternativeInstance.cpp index 23e502042..a15ea3c19 100644 --- a/typed_python/PyAlternativeInstance.cpp +++ b/typed_python/PyAlternativeInstance.cpp @@ -245,7 +245,7 @@ void PyAlternativeInstance::mirrorTypeInformationIntoPyTypeConcrete(Alternative* PyObjectStealer alternatives(PyTuple_New(alt->subtypes().size())); for (long k = 0; k < alt->subtypes().size(); k++) { - ConcreteAlternative* concrete = ConcreteAlternative::Make(alt, k); + ConcreteAlternative* concrete = alt->concreteSubtype(k); PyDict_SetItemString( pyType->tp_dict, @@ -422,9 +422,11 @@ std::pair PyConcreteAlternativeInstance::callMethod(const char* } auto res = PyFunctionInstance::tryToCallAnyOverload(method, nullptr, nullptr, targetArgTuple, nullptr); + if (res.first) { return res; } + PyErr_Format( PyExc_TypeError, "'%s.%s' cannot find a valid overload with these arguments", @@ -459,6 +461,10 @@ Py_ssize_t PyConcreteAlternativeInstance::mp_and_sq_length_concrete() { if (!p.first) { return -1; } + if (!p.second) { + return -1; + } + if (!PyLong_Check(p.second)) { return -1; } @@ -475,11 +481,28 @@ PyObject* PyConcreteAlternativeInstance::mp_subscript_concrete(PyObject* item) { } int PyConcreteAlternativeInstance::mp_ass_subscript_concrete(PyObject* item, PyObject* v) { - auto p = callMethod("__setitem__", item, v); - if (!p.first) { - PyErr_Format(PyExc_TypeError, "__setitem__ not defined for type %s", type()->name().c_str()); + std::pair res; + + if (!v) { + res = callMethod("__delitem__", item); + } else { + res = callMethod("__setitem__", item, v); + } + + if (!res.first) { + if (v) { + PyErr_Format(PyExc_TypeError, "__setitem__ not defined for type %s", type()->name().c_str()); + } else { + PyErr_Format(PyExc_TypeError, "__delitem__ not defined for type %s", type()->name().c_str()); + } + + return -1; + } + + if (!res.second) { return -1; } + return 0; } @@ -772,33 +795,18 @@ PyObject* PyConcreteAlternativeInstance::altExit(PyObject* o, PyObject* args, Py // static PyMethodDef* PyConcreteAlternativeInstance::typeMethodsConcrete(Type* t) { - - // List of magic methods that are not attached to direct function pointers in PyTypeObject. - // These need to be defined by adding entries to PyTypeObject.tp_methods - // and we need to avoid adding them to PyTypeObject.tp_dict ourselves. - // Also, we only want to add the entry to tp_methods if they are explicitly defined. - const std::map special_magic_methods = { - {"__format__", (PyCFunction)altFormat}, - {"__bytes__", (PyCFunction)altBytes}, - {"__dir__", (PyCFunction)altDir}, - {"__reversed__", (PyCFunction)altReversed}, - {"__complex__", (PyCFunction)altComplex}, - {"__round__", (PyCFunction)altRound}, - {"__trunc__", (PyCFunction)altTrunc}, - {"__floor__", (PyCFunction)altFloor}, - {"__ceil__", (PyCFunction)altCeil}, - {"__enter__", (PyCFunction)altEnter}, - {"__exit__", (PyCFunction)altExit} - }; - - int cur = 0; - auto altMethods = ((ConcreteAlternative*)t)->getAlternative()->getMethods(); - PyMethodDef* ret = new PyMethodDef[special_magic_methods.size() + 1]; - for (auto m: special_magic_methods) { - if (altMethods.find(m.first) != altMethods.end()) { - ret[cur++] = {m.first, m.second, METH_VARARGS | METH_KEYWORDS, NULL}; - } - } - ret[cur] = {NULL, NULL}; - return ret; + return new PyMethodDef [12] { + {"__format__", (PyCFunction)altFormat, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__bytes__", (PyCFunction)altBytes, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__dir__", (PyCFunction)altDir, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__reversed__", (PyCFunction)altReversed, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__complex__", (PyCFunction)altComplex, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__round__", (PyCFunction)altRound, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__trunc__", (PyCFunction)altTrunc, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__floor__", (PyCFunction)altFloor, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__ceil__", (PyCFunction)altCeil, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__enter__", (PyCFunction)altEnter, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__exit__", (PyCFunction)altExit, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} + }; } diff --git a/typed_python/PyCellType.hpp b/typed_python/PyCellType.hpp index 5ae0c52fa..74fdf3ef2 100644 --- a/typed_python/PyCellType.hpp +++ b/typed_python/PyCellType.hpp @@ -19,6 +19,11 @@ #include "util.hpp" #include "Type.hpp" + +PyDoc_STRVAR(PyCell_doc, + "PyCell: represents a python-native cell instance.\n" +); + //wraps a python Cell class PyCellType : public PyObjectHandleTypeBase { public: @@ -26,14 +31,12 @@ class PyCellType : public PyObjectHandleTypeBase { PyObjectHandleTypeBase(TypeCategory::catPyCell) { m_name = std::string("PyCell"); - - m_is_simple = false; - - endOfConstructorInitialization(); // finish initializing the type object. + m_size = sizeof(layout_type*); + m_is_default_constructible = true; } - bool isBinaryCompatibleWithConcrete(Type* other) { - return other == this; + const char* docConcrete() { + return PyCell_doc; } template @@ -49,14 +52,6 @@ class PyCellType : public PyObjectHandleTypeBase { void _visitReferencedTypes(const visitor_type& visitor) { } - bool _updateAfterForwardTypesChanged() { - m_size = sizeof(layout_type*); - - m_is_default_constructible = true; - - return false; - } - typed_python_hash_type hash(instance_ptr left) { return PyObject_Hash(getPyObj(left)); } @@ -150,6 +145,8 @@ class PyCellType : public PyObjectHandleTypeBase { getPyObj(self) = PyCell_New(nullptr); } + void postInitializeConcrete() {} + static PyCellType* Make() { static PyCellType* res = new PyCellType(); diff --git a/typed_python/PyClassInstance.cpp b/typed_python/PyClassInstance.cpp index 9f08cb2b5..ef5ea9777 100644 --- a/typed_python/PyClassInstance.cpp +++ b/typed_python/PyClassInstance.cpp @@ -658,43 +658,64 @@ PyObject* PyClassInstance::tpGetattrGeneric( return ret; } -/* initialize the type object's dict. -if 'asHeldClass', then this is the held class's dict we're initializing. -*/ -void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyTypeObject* pyType, bool asHeldClass) { - PyObjectStealer bases(PyTuple_New(classT->getHeldClass()->getBases().size())); +PyObject* convertPyTupleEltToPyObj(std::string s) { + return PyUnicode_FromString(s.c_str()); +} - for (long k = 0; k < classT->getHeldClass()->getBases().size(); k++) { - PyTuple_SetItem( - bases, - k, - incref(typePtrToPyTypeRepresentation(classT->getHeldClass()->getBases()[k]->getClassType())) - ); - } +PyObject* convertPyTupleEltToPyObj(PyObject* o) { + return o; +} - PyObjectStealer mro(PyTuple_New(classT->getHeldClass()->getMro().size())); +PyObject* convertPyTupleEltToPyObj(Type* t) { + return incref(PyInstance::typePtrToPyTypeRepresentation(t)); +} - for (long k = 0; k < classT->getHeldClass()->getMro().size(); k++) { - PyTuple_SetItem( - mro, - k, - incref(typePtrToPyTypeRepresentation(classT->getHeldClass()->getMro()[k]->getClassType())) - ); +template +PyObject* newPyTuple(const container_type& container, const func_type& objCreator) { + PyObject* tup = PyTuple_New(container.size()); + long ix = 0; + for (auto containerElt: container) { + PyTuple_SetItem(tup, ix++, convertPyTupleEltToPyObj(objCreator(containerElt))); } + return tup; +} - PyObjectStealer types(PyTuple_New(classT->getMembers().size())); +/* initialize the type object's dict. - for (long k = 0; k < classT->getMembers().size(); k++) { - PyTuple_SetItem(types, k, incref(typePtrToPyTypeRepresentation(classT->getMembers()[k].getType()))); - } +if 'asHeldClass', then this is the held class's dict we're initializing. +*/ +void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyTypeObject* pyType, bool asHeldClass) { + PyObjectStealer bases( + newPyTuple( + classT->getHeldClass()->getBases(), + [&](HeldClass* t) { return t->getClassType(); } + ) + ); - PyObjectStealer names(PyTuple_New(classT->getMembers().size())); - for (long k = 0; k < classT->getMembers().size(); k++) { - PyObject* namePtr = PyUnicode_FromString(classT->getMembers()[k].getName().c_str()); - PyTuple_SetItem(names, k, namePtr); - } + PyObjectStealer mro( + newPyTuple( + classT->getHeldClass()->getMro(), + [&](HeldClass* t) { + return t->getClassType(); + } + ) + ); + PyObjectStealer memberTypes( + newPyTuple( + classT->isForwardDefined() ? classT->getOwnMembers() : classT->getMembers(), + [&](const MemberDefinition& member) { return member.getType(); } + ) + ); + + PyObjectStealer names( + newPyTuple( + classT->isForwardDefined() ? classT->getOwnMembers() : classT->getMembers(), + [&](const MemberDefinition& member) { return member.getName(); } + ) + ); + PyObjectStealer defaults(PyDict_New()); for (long k = 0; k < classT->getMembers().size(); k++) { @@ -710,29 +731,47 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT defaults, classT->getHeldClass()->getMemberName(k).c_str(), defaultVal - ); + ); } } PyObjectStealer memberFunctions(PyDict_New()); - for (auto p: classT->getMemberFunctions()) { - PyDict_SetItemString(memberFunctions, p.first.c_str(), typePtrToPyTypeRepresentation(p.second)); + for (auto p: classT->isForwardDefined() ? classT->getOwnMemberFunctions() : classT->getMemberFunctions()) { + typePtrToPyTypeRepresentation(p.second); + if (!p.second->isTypeObjReady()) { + throw std::runtime_error("TypeObj for " + p.second->name() + " is not ready"); + } - // TODO: find a predefined function that does this method search - PyMethodDef* defined = pyType->tp_methods; - while (defined && defined->ml_name && !!strcmp(defined->ml_name, p.first.c_str())) - defined++; + PyDict_SetItemString( + memberFunctions, + p.first.c_str(), + typePtrToPyTypeRepresentation(p.second) + ); - if (!defined || !defined->ml_name) { - if (p.second->getClosureType()->bytecount()) { - std::cout << "WARNING: invalid class member " << classT->name() << p.first << " had a nonempty closure.\n"; - } else { - if (p.second->bytecount()) { - throw std::runtime_error("Somehow, a Class got a function with a closure type."); + if (!classT->isForwardDefined()) { + // TODO: clean this up. I'm not sure what we are guarding against - is it + // the case that sometimes we overwrite something already in the dict? + PyMethodDef* defined = pyType->tp_methods; + while (defined && defined->ml_name && !!strcmp(defined->ml_name, p.first.c_str())) + defined++; + + if (!defined || !defined->ml_name) { + if (p.second->getClosureType()->bytecount()) { + std::cout << "WARNING: invalid class member " << classT->name() << p.first << " had a nonempty closure.\n"; + } else { + if (p.second->bytecount()) { + throw std::runtime_error("Somehow, a Class got a function with a closure type."); + } + PyDict_SetItemString(pyType->tp_dict, p.first.c_str(), PyInstance::initialize(p.second, [&](instance_ptr) {})); } - PyDict_SetItemString(pyType->tp_dict, p.first.c_str(), PyInstance::initialize(p.second, [&](instance_ptr) {})); } + } else { + PyDict_SetItemString( + pyType->tp_dict, + p.first.c_str(), + typePtrToPyTypeRepresentation(p.second) + ); } } @@ -749,7 +788,7 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT PyDict_SetItemString(pyType->tp_dict, "HeldClass", typePtrToPyTypeRepresentation(classT->getHeldClass())); } - PyDict_SetItemString(pyType->tp_dict, "MemberTypes", types); + PyDict_SetItemString(pyType->tp_dict, "MemberTypes", memberTypes); PyDict_SetItemString(pyType->tp_dict, "BaseClasses", bases); PyDict_SetItemString(pyType->tp_dict, "IsFinal", classT->isFinal() ? Py_True : Py_False); PyDict_SetItemString(pyType->tp_dict, "MRO", mro); @@ -777,25 +816,44 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT PyObjectStealer staticMemberFunctions(PyDict_New()); PyDict_SetItemString(pyType->tp_dict, "StaticMemberFunctions", staticMemberFunctions); - for (auto nameAndObj: classT->getStaticFunctions()) { - PyDict_SetItemString(staticMemberFunctions, nameAndObj.first.c_str(), typePtrToPyTypeRepresentation(nameAndObj.second)); - - if (nameAndObj.second->getClosureType()->bytecount()) { - throw std::runtime_error( - "Somehow, " + classT->name() + "." - + nameAndObj.first + " has a populated closure." - ); + for (auto nameAndObj: classT->isForwardDefined() ? + classT->getOwnStaticFunctions() : classT->getStaticFunctions()) { + + typePtrToPyTypeRepresentation(nameAndObj.second); + if (!nameAndObj.second->isTypeObjReady()) { + asm("int3"); + throw std::runtime_error("TypeObj for " + nameAndObj.second->name() + " is not ready"); } PyDict_SetItemString( - pyType->tp_dict, - nameAndObj.first.c_str(), - PyStaticMethod_New( - PyInstance::initialize(nameAndObj.second, [&](instance_ptr data){ - //nothing to do - functions like this are just types. - }) - ) + staticMemberFunctions, + nameAndObj.first.c_str(), typePtrToPyTypeRepresentation(nameAndObj.second) ); + + if (!classT->isForwardDefined()) { + if (nameAndObj.second->getClosureType()->bytecount()) { + throw std::runtime_error( + "Somehow, " + classT->name() + "." + + nameAndObj.first + " has a populated closure." + ); + } + + PyDict_SetItemString( + pyType->tp_dict, + nameAndObj.first.c_str(), + PyStaticMethod_New( + PyInstance::initialize(nameAndObj.second, [&](instance_ptr data){ + //nothing to do - functions like this are just types. + }) + ) + ); + } else { + PyDict_SetItemString( + pyType->tp_dict, + nameAndObj.first.c_str(), + typePtrToPyTypeRepresentation(nameAndObj.second) + ); + } } PyObjectStealer classMemberFunctions(PyDict_New()); @@ -1299,33 +1357,18 @@ PyObject* PyClassInstance::clsExitGeneric(PyObject* o, Type* t, instance_ptr dat // static PyMethodDef* PyClassInstance::typeMethodsConcrete(Type* t) { - - // List of magic methods that are not attached to direct function pointers in PyTypeObject. - // These need to be defined by adding entries to PyTypeObject.tp_methods - // and we need to avoid adding them to PyTypeObject.tp_dict ourselves. - // Also, we only want to add the entry to tp_methods if they are explicitly defined. - const std::map special_magic_methods = { - {"__format__", (PyCFunction)clsFormat}, - {"__bytes__", (PyCFunction)clsBytes}, - {"__dir__", (PyCFunction)clsDir}, - {"__reversed__", (PyCFunction)clsReversed}, - {"__complex__", (PyCFunction)clsComplex}, - {"__round__", (PyCFunction)clsRound}, - {"__trunc__", (PyCFunction)clsTrunc}, - {"__floor__", (PyCFunction)clsFloor}, - {"__ceil__", (PyCFunction)clsCeil}, - {"__enter__", (PyCFunction)clsEnter}, - {"__exit__", (PyCFunction)clsExit} - }; - - int cur = 0; - auto clsMethods = ((Class*)t)->getMemberFunctions(); - PyMethodDef* ret = new PyMethodDef[special_magic_methods.size() + 1]; - for (auto m: special_magic_methods) { - if (clsMethods.find(m.first) != clsMethods.end()) { - ret[cur++] = {m.first, m.second, METH_VARARGS | METH_KEYWORDS, NULL}; - } - } - ret[cur] = {NULL, NULL}; - return ret; + return new PyMethodDef [12] { + {"__format__", (PyCFunction)clsFormat, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__bytes__", (PyCFunction)clsBytes, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__dir__", (PyCFunction)clsDir, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__reversed__", (PyCFunction)clsReversed, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__complex__", (PyCFunction)clsComplex, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__round__", (PyCFunction)clsRound, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__trunc__", (PyCFunction)clsTrunc, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__floor__", (PyCFunction)clsFloor, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__ceil__", (PyCFunction)clsCeil, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__enter__", (PyCFunction)clsEnter, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__exit__", (PyCFunction)clsExit, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} + }; } diff --git a/typed_python/PyForwardInstance.hpp b/typed_python/PyForwardInstance.hpp index 657d217fa..7ac26045d 100644 --- a/typed_python/PyForwardInstance.hpp +++ b/typed_python/PyForwardInstance.hpp @@ -81,33 +81,34 @@ class PyForwardInstance : public PyInstance { return incref(PyInstance::typePtrToPyTypeRepresentation(result)); } - - static PyObject* forwardGetReferencing(PyObject* o, PyObject* args) { + static PyObject* forwardResolve(PyObject* o, PyObject* args) { if (PyTuple_Size(args) != 0) { - PyErr_SetString(PyExc_TypeError, "Forward.getReferencing takes zero arguments"); + PyErr_SetString(PyExc_TypeError, "Forward.get takes zero arguments"); return NULL; } Forward* self_type = (Forward*)PyInstance::unwrapTypeArgToTypePtr(o); if (!self_type) { - PyErr_SetString(PyExc_TypeError, "Forward.getReferencing unexpected error"); + PyErr_SetString(PyExc_TypeError, "Forward.get unexpected error"); return NULL; } - PyObject* res = PyList_New(0); + return translateExceptionToPyObject([&]() { + Type* result = self_type->getTarget()->forwardResolvesTo(); - for (auto t: self_type->getReferencing()) { - PyList_Append(res, PyInstance::typePtrToPyTypeRepresentation(t)); - } + if (!result) { + return incref(Py_None); + } - return res; + return incref(PyInstance::typePtrToPyTypeRepresentation(result)); + }); } static PyMethodDef* typeMethodsConcrete(Type* t) { return new PyMethodDef [4] { {"define", (PyCFunction)PyForwardInstance::forwardDefine, METH_VARARGS | METH_CLASS, NULL}, {"get", (PyCFunction)PyForwardInstance::forwardGet, METH_VARARGS | METH_CLASS, NULL}, - {"getReferencing", (PyCFunction)PyForwardInstance::forwardGetReferencing, METH_VARARGS | METH_CLASS, NULL}, + {"resolve", (PyCFunction)PyForwardInstance::forwardResolve, METH_VARARGS | METH_CLASS, NULL}, {NULL, NULL} }; } diff --git a/typed_python/PyFunctionGlobal.cpp b/typed_python/PyFunctionGlobal.cpp new file mode 100644 index 000000000..641b1a406 --- /dev/null +++ b/typed_python/PyFunctionGlobal.cpp @@ -0,0 +1,246 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyFunctionGlobal.hpp" + + +PyDoc_STRVAR(PyFunctionGlobal_doc, + "A single overload of a Function type object.\n\n" +); + +PyMethodDef PyFunctionGlobal_methods[] = { + {"isUnresolved", (PyCFunction)PyFunctionGlobal::isUnresolved, METH_VARARGS | METH_KEYWORDS, NULL}, + {"getValue", (PyCFunction)PyFunctionGlobal::getValue, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL} /* Sentinel */ +}; + + +PyObject* PyFunctionGlobal::getValue(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs) { + static const char* kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + return NULL; + } + + return translateExceptionToPyObject([&]() { + PyObject* result = self->getGlobal().getValueAsPyobj(); + if (!result) { + // TODO: should this throw a NameError? + throw std::runtime_error("Global is not resolved"); + } + return incref(result); + }); +} + +PyObject* PyFunctionGlobal::isUnresolved(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs) { + static const char* kwlist[] = {"insistForwardsResolved", NULL}; + + int insistForwardsResolved = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|p", (char**)kwlist, &insistForwardsResolved)) { + return NULL; + } + + return translateExceptionToPyObject([&]() { + return incref(self->getGlobal().isUnresolved(insistForwardsResolved) ? Py_True : Py_False); + }); +} + +FunctionGlobal& PyFunctionGlobal::getGlobal() { + if (!mFuncType) { + throw std::runtime_error("FunctionGlobal is missing a function type"); + } + + if (!mName) { + throw std::runtime_error("FunctionGlobal is missing a name"); + } + + if (mOverloadIx < 0 || mOverloadIx >= mFuncType->getOverloads().size()){ + throw std::runtime_error("FunctionGlobal overload is out of bounds"); + } + + FunctionOverload& o = mFuncType->getOverloads()[mOverloadIx]; + + auto it = o.getGlobals().find(*mName); + if (it == o.getGlobals().end()) { + throw std::runtime_error("FunctionGlobal has no global named " + *mName); + } + + return it->second; +} + +/* static */ +void PyFunctionGlobal::dealloc(PyFunctionGlobal *self) +{ + if (self->mName) { + delete self->mName; + } + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyFunctionGlobal::newPyFunctionGlobal(Function* func, long overloadIx, std::string globalName) { + PyFunctionGlobal* self = (PyFunctionGlobal*)PyType_FunctionGlobal.tp_alloc(&PyType_FunctionGlobal, 0); + self->mFuncType = func; + self->mOverloadIx = overloadIx; + self->mName = new std::string(globalName); + return (PyObject*)self; +} + +/* static */ +PyObject* PyFunctionGlobal::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyFunctionGlobal* self; + + self = (PyFunctionGlobal*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mFuncType = nullptr; + self->mOverloadIx = 0; + self->mName = nullptr; + } + + return (PyObject*)self; +} + +PyObject* PyFunctionGlobal::tp_repr(PyObject *selfObj) { + PyFunctionGlobal* self = (PyFunctionGlobal*)selfObj; + + return translateExceptionToPyObject([&]() { + return PyUnicode_FromString(self->getGlobal().toString().c_str()); + }); +} + +PyObject* PyFunctionGlobal::tp_getattro(PyObject* selfObj, PyObject* attrName) { + PyFunctionGlobal* pyFuncGlobal = (PyFunctionGlobal*)selfObj; + + return translateExceptionToPyObject([&] { + if (!PyUnicode_Check(attrName)) { + throw std::runtime_error("Expected a string for attribute name"); + } + + std::string attr(PyUnicode_AsUTF8(attrName)); + + auto& global = pyFuncGlobal->getGlobal(); + + if (attr == "kind") { + if (global.isUnbound()) { + return PyUnicode_FromString("Unbound"); + } + + if (global.isNamedModuleMember()) { + return PyUnicode_FromString("NamedModuleMember"); + } + + if (global.isGlobalInCell()) { + return PyUnicode_FromString("GlobalInCell"); + } + + if (global.isGlobalInDict()) { + return PyUnicode_FromString("GlobalInDict"); + } + + if (global.isConstant()) { + return PyUnicode_FromString("Constant"); + } + + throw std::runtime_error("Unknown Global Kind"); + } + + if (attr == "constant" && global.isConstant()) { + return incref((PyObject*)PyInstance::typeObj(global.getConstant())); + } + + if (attr == "name" && (global.isNamedModuleMember() || global.isGlobalInDict())) { + return PyUnicode_FromString(global.getName().c_str()); + } + + if (attr == "moduleName" && global.isNamedModuleMember()) { + return PyUnicode_FromString(global.getModuleName().c_str()); + } + + if (attr == "cell" && global.isGlobalInCell()) { + return incref(global.getModuleDictOrCell()); + } + + if (attr == "moduleDict" && (global.isNamedModuleMember() || global.isGlobalInDict())) { + return incref(global.getModuleDictOrCell()); + } + + return PyObject_GenericGetAttr(selfObj, attrName); + }); +} + +/* static */ +int PyFunctionGlobal::init(PyFunctionGlobal *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "FunctionGlobal cannot be initialized directly"); + return -1; +} + +PyTypeObject PyType_FunctionGlobal = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "FunctionGlobal", + .tp_basicsize = sizeof(PyFunctionGlobal), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyFunctionGlobal::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = PyFunctionGlobal::tp_repr, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = PyFunctionGlobal::tp_getattro, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyFunctionGlobal_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyFunctionGlobal_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyFunctionGlobal::init, + .tp_alloc = 0, + .tp_new = PyFunctionGlobal::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyFunctionGlobal.hpp b/typed_python/PyFunctionGlobal.hpp new file mode 100644 index 000000000..9267e969f --- /dev/null +++ b/typed_python/PyFunctionGlobal.hpp @@ -0,0 +1,50 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" + +class PyFunctionGlobal { +public: + PyObject_HEAD + + Function* mFuncType; + + int64_t mOverloadIx; + + std::string* mName; + + FunctionGlobal& getGlobal(); + + static PyObject* tp_getattro(PyObject* selfObj, PyObject* attr); + + static PyObject* tp_repr(PyObject *selfObj); + + static void dealloc(PyFunctionGlobal *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyFunctionGlobal(Function* func, long overloadIx, std::string globalName); + + static int init(PyFunctionGlobal *self, PyObject *args, PyObject *kwargs); + + static PyObject* isUnresolved(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs); + + static PyObject* getValue(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs); +}; + +extern PyTypeObject PyType_FunctionGlobal; diff --git a/typed_python/PyFunctionInstance.cpp b/typed_python/PyFunctionInstance.cpp index 860bdee37..6f51d6c73 100644 --- a/typed_python/PyFunctionInstance.cpp +++ b/typed_python/PyFunctionInstance.cpp @@ -35,7 +35,7 @@ PyObject* PyFunctionInstance::prepareArgumentToBePassedToCompiler(PyObject* o) { // static std::pair -PyFunctionInstance::tryToCallAnyOverload(const Function* f, instance_ptr funcClosure, PyObject* self, +PyFunctionInstance::tryToCallAnyOverload(Function* f, instance_ptr funcClosure, PyObject* self, PyObject* args, PyObject* kwargs) { //if we are an entrypoint, map any untyped function arguments to typed functions PyObjectHolder mappedArgs; @@ -97,7 +97,7 @@ PyFunctionInstance::tryToCallAnyOverload(const Function* f, instance_ptr funcClo // static std::pair PyFunctionInstance::tryToCallOverload( - const Function* f, + Function* f, instance_ptr functionClosure, long overloadIx, PyObject* self, @@ -105,8 +105,7 @@ std::pair PyFunctionInstance::tryToCallOverload( PyObject* kwargs, ConversionLevel conversionLevel ) { - const Function::Overload& overload(f->getOverloads()[overloadIx]); - + FunctionOverload& overload(f->getOverloads()[overloadIx]); FunctionCallArgMapping mapping(overload); mapping.pushArguments(self, args, kwargs); @@ -192,11 +191,11 @@ std::pair PyFunctionInstance::tryToCallOverload( } std::pair PyFunctionInstance::getOverloadReturnType( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs ) { - const Function::Overload& overload(f->getOverloads()[overloadIx]); + FunctionOverload& overload(f->getOverloads()[overloadIx]); Type* returnType = overload.getReturnType(); if (overload.getSignatureFunction()) { @@ -228,7 +227,7 @@ std::pair PyFunctionInstance::getOverloadReturnType( } std::pair PyFunctionInstance::determineReturnTypeForMatchedCall( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs, PyObject* self, @@ -356,7 +355,7 @@ std::pair PyFunctionInstance::determineReturnTypeForMatchedCall( return std::make_pair(actualReturnTypeAndCls.first, false); } -std::pair PyFunctionInstance::tryToCall(const Function* f, instance_ptr closure, PyObject* arg0, PyObject* arg1, PyObject* arg2) { +std::pair PyFunctionInstance::tryToCall(Function* f, instance_ptr closure, PyObject* arg0, PyObject* arg1, PyObject* arg2) { PyObjectStealer argTuple( (arg0 && arg1 && arg2) ? PyTuple_Pack(3, arg0, arg1, arg2) : @@ -371,12 +370,12 @@ std::pair PyFunctionInstance::tryToCall(const Function* f, inst // static std::pair PyFunctionInstance::dispatchFunctionCallToNative( - const Function* f, + Function* f, instance_ptr functionClosure, long overloadIx, const FunctionCallArgMapping& mapper ) { - const Function::Overload& overload(f->getOverloads()[overloadIx]); + FunctionOverload& overload(f->getOverloads()[overloadIx]); for (const auto& spec: overload.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -420,7 +419,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( throw std::runtime_error("Somehow, the number of overloads in the function changed."); } - const Function::Overload& overload2(convertedF->getOverloads()[overloadIx]); + FunctionOverload& overload2(convertedF->getOverloads()[overloadIx]); for (const auto& spec: overload2.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -457,7 +456,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( decref(res); - const Function::Overload& convertedOverload(convertedF->getOverloads()[overloadIx]); + FunctionOverload& convertedOverload(convertedF->getOverloads()[overloadIx]); for (const auto& spec: convertedOverload.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -484,10 +483,10 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( } std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpecialization( - const Function::Overload& overload, + const FunctionOverload& overload, Type* closureType, instance_ptr closureData, - const Function::CompiledSpecialization& specialization, + const CompiledSpecialization& specialization, const FunctionCallArgMapping& mapper ) { Type* returnType = specialization.getReturnType(); @@ -496,7 +495,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpe throw std::runtime_error("Malformed function specialization: missing a return type."); } - std::vector mappingArgs; + std::vector mappingArgs; // first, see if we can short-circuit for (long k = 0; k < overload.getArgs().size(); k++) { @@ -514,7 +513,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpe auto arg = overload.getArgs()[k]; Type* argType = specialization.getArgTypes()[k]; - FunctionCallArgMapping::FunctionArg res = mapper.extractArgWithType(k, argType); + FunctionCallArgMapping::LiveFunctionArg res = mapper.extractArgWithType(k, argType); if (res.isValid()) { mappingArgs.push_back(res); @@ -557,96 +556,10 @@ std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpe // static PyObject* PyFunctionInstance::createOverloadPyRepresentation(Function* f) { - PyObject* funcOverload = staticPythonInstance("typed_python.internals", "FunctionOverload"); - PyObject* closureVariableCellLookupSingleton = staticPythonInstance("typed_python.internals", "CellAccess"); - PyObject* funcOverloadArg = staticPythonInstance("typed_python.internals", "FunctionOverloadArg"); - PyObjectStealer overloadTuple(PyTuple_New(f->getOverloads().size())); for (long k = 0; k < f->getOverloads().size(); k++) { - auto& overload = f->getOverloads()[k]; - - PyObjectStealer pyIndex(PyLong_FromLong(k)); - - PyObjectStealer pyGlobalCellDict(PyDict_New()); - - for (auto nameAndCell: overload.getFunctionGlobalsInCells()) { - PyDict_SetItemString(pyGlobalCellDict, nameAndCell.first.c_str(), nameAndCell.second); - } - - PyObjectStealer pyClosureVarsDict(PyDict_New()); - - for (auto nameAndClosureVar: overload.getClosureVariableBindings()) { - PyObjectStealer bindingObj(PyTuple_New(nameAndClosureVar.second.size())); - - for (long k = 0; k < nameAndClosureVar.second.size(); k++) { - ClosureVariableBindingStep step = nameAndClosureVar.second[k]; - - if (step.isFunction()) { - // recall that 'PyTuple_SetItem' steals a reference, so we need to incref it here - PyTuple_SetItem(bindingObj, k, incref((PyObject*)typePtrToPyTypeRepresentation(step.getFunction()))); - } else - if (step.isNamedField()) { - PyTuple_SetItem(bindingObj, k, PyUnicode_FromString(step.getNamedField().c_str())); - } else - if (step.isIndexedField()) { - PyTuple_SetItem(bindingObj, k, PyLong_FromLong(step.getIndexedField())); - } else - if (step.isCellAccess()) { - PyTuple_SetItem(bindingObj, k, incref(closureVariableCellLookupSingleton)); - } else { - throw std::runtime_error("Corrupt ClosureVariableBindingStep encountered"); - } - } - - PyDict_SetItemString(pyClosureVarsDict, nameAndClosureVar.first.c_str(), bindingObj); - } - - PyObjectStealer emptyTup(PyTuple_New(0)); - PyObjectStealer emptyDict(PyDict_New()); - - // note that we can't actually call into the Python interpreter during this call, - // because that can release the GIL and allow other threads to access our type - // object before it's done. - PyObjectStealer argsTup(PyTuple_New(f->getOverloads()[k].getArgs().size())); - - for (long argIx = 0; argIx < f->getOverloads()[k].getArgs().size(); argIx++) { - auto arg = f->getOverloads()[k].getArgs()[argIx]; - - PyObjectStealer pyArgInst( - ((PyTypeObject*)funcOverloadArg)->tp_new((PyTypeObject*)funcOverloadArg, emptyTup, emptyDict) - ); - - PyObjectStealer pyArgInstDict(PyObject_GenericGetDict(pyArgInst, nullptr)); - - PyObjectStealer pyName(PyUnicode_FromString(arg.getName().c_str())); - PyDict_SetItemString(pyArgInstDict, "name", pyName); - PyDict_SetItemString(pyArgInstDict, "defaultValue", arg.getDefaultValue() ? PyTuple_Pack(1, arg.getDefaultValue()) : Py_None); - PyDict_SetItemString(pyArgInstDict, "_typeFilter", arg.getTypeFilter() ? (PyObject*)typePtrToPyTypeRepresentation(arg.getTypeFilter()) : Py_None); - PyDict_SetItemString(pyArgInstDict, "isStarArg", arg.getIsStarArg() ? Py_True : Py_False); - PyDict_SetItemString(pyArgInstDict, "isKwarg", arg.getIsKwarg() ? Py_True : Py_False); - - PyTuple_SetItem(argsTup, argIx, incref(pyArgInst)); - } - - PyObjectStealer pyOverloadInst( - ((PyTypeObject*)funcOverload)->tp_new((PyTypeObject*)funcOverload, emptyTup, emptyDict) - ); - - PyObjectStealer pyOverloadInstDict(PyObject_GenericGetDict(pyOverloadInst, nullptr)); - - PyDict_SetItemString(pyOverloadInstDict, "functionTypeObject", typePtrToPyTypeRepresentation(f)); - PyDict_SetItemString(pyOverloadInstDict, "index", (PyObject*)pyIndex); - PyDict_SetItemString(pyOverloadInstDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); - PyDict_SetItemString(pyOverloadInstDict, "functionCode", (PyObject*)overload.getFunctionCode()); - PyDict_SetItemString(pyOverloadInstDict, "funcGlobalsInCells", (PyObject*)pyGlobalCellDict); - PyDict_SetItemString(pyOverloadInstDict, "returnType", overload.getReturnType() ? (PyObject*)typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "methodOf", overload.getMethodOf() ? (PyObject*)typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "_realizedGlobals", Py_None); - PyDict_SetItemString(pyOverloadInstDict, "args", argsTup); - - PyTuple_SetItem(overloadTuple, k, incref(pyOverloadInst)); + PyTuple_SetItem(overloadTuple, k, PyFunctionOverload::newPyFunctionOverload(f, k)); } return incref(overloadTuple); @@ -779,35 +692,6 @@ PyObject* PyFunctionInstance::extractPyFun(PyObject* funcObj, PyObject* args, Py }); } -/* static */ -PyObject* PyFunctionInstance::extractOverloadGlobals(PyObject* cls, PyObject* args, PyObject* kwargs) { - Type* selfType = PyInstance::unwrapTypeArgToTypePtr(cls); - - if (!selfType || selfType->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_Format(PyExc_TypeError, "Expected class to be a Function"); - return nullptr; - } - - Function* fType = (Function*)selfType; - - static const char *kwlist[] = {"overloadIx", NULL}; - - long overloadIx; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "l", (char**)kwlist, &overloadIx)) { - return nullptr; - } - - if (overloadIx < 0 || overloadIx >= fType->getOverloads().size()) { - PyErr_SetString(PyExc_IndexError, "Overload index out of bounds"); - return nullptr; - } - - return translateExceptionToPyObject([&]() { - return incref(fType->getOverloads()[overloadIx].getFunctionGlobals()); - }); -} - /* static */ PyObject* PyFunctionInstance::getClosure(PyObject* funcObj, PyObject* args, PyObject* kwargs) { static const char *kwlist[] = {NULL}; @@ -934,7 +818,7 @@ PyObject* PyFunctionInstance::overload(PyObject* funcObj, PyObject* args, PyObje throw PythonExceptionSet(); } - argT = convertPythonObjectToFunctionType(name, arg, false, false); + argT = convertPythonObjectToFunctionType(name, nullptr, arg, false, false); } if (!argT) { @@ -1002,7 +886,6 @@ PyMethodDef* PyFunctionInstance::typeMethodsConcrete(Type* t) { {"typeWithNocompile", (PyCFunction)PyFunctionInstance::typeWithNocompile, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, {"resultTypeFor", (PyCFunction)PyFunctionInstance::resultTypeFor, METH_VARARGS | METH_KEYWORDS, NULL}, {"extractPyFun", (PyCFunction)PyFunctionInstance::extractPyFun, METH_VARARGS | METH_KEYWORDS, NULL}, - {"extractOverloadGlobals", (PyCFunction)PyFunctionInstance::extractOverloadGlobals, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, {"getClosure", (PyCFunction)PyFunctionInstance::getClosure, METH_VARARGS | METH_KEYWORDS, NULL}, {"withClosureType", (PyCFunction)PyFunctionInstance::withClosureType, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, {"withOverloadVariableBindings", (PyCFunction)PyFunctionInstance::withOverloadVariableBindings, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, @@ -1106,6 +989,7 @@ PyObject* PyFunctionInstance::withOverloadVariableBindings(PyObject* cls, PyObje Function* PyFunctionInstance::convertPythonObjectToFunctionType( PyObject* name, + PyObject* classname, PyObject *funcObj, bool assumeClosuresGlobal, bool ignoreAnnotations @@ -1124,7 +1008,13 @@ Function* PyFunctionInstance::convertPythonObjectToFunctionType( PyObject* makeFunctionType = staticPythonInstance("typed_python.internals", "makeFunctionType"); - PyObjectStealer args(PyTuple_Pack(2, name, funcObj)); + PyObjectHolder args; + if (classname) { + args.steal(PyTuple_Pack(3, name, funcObj, classname)); + } else { + args.steal(PyTuple_Pack(2, name, funcObj)); + } + PyObjectStealer kwargs(PyDict_New()); if (assumeClosuresGlobal) { @@ -1215,36 +1105,82 @@ void PyFunctionInstance::copyConstructFromPythonInstanceConcrete(Function* type, PyObject* pyClosure = PyFunction_GetClosure(pyRepresentation); - if (!pyClosure || !PyTuple_Check(pyClosure) || PyTuple_Size(pyClosure) != closureType->getTypes().size()) { - throw std::runtime_error("Expected the pyClosure to have " + format(closureType->getTypes().size()) + " cells."); + // our closure may or may not have a 'globals' dict at the end of it. + long expectedCellCount = closureType->getTypes().size(); + if (closureType->getTypes().size() && closureType->getTypes().back()->isPythonObjectOfType()) { + expectedCellCount -= 1; + } + + if (expectedCellCount == 0) { + if (pyClosure && !PyTuple_Check(pyClosure)) { + throw std::runtime_error("This PyFunction closure is not a tuple"); + } + + if (pyClosure && PyTuple_Size(pyClosure)) { + throw std::runtime_error( + "Our closure of type " + + closureType->name() + + " means we expect the python function's closure to have " + + format(expectedCellCount) + + " variables but it has " + format(PyTuple_Size(pyClosure)) + ); + } + } else { + if (!pyClosure || !PyTuple_Check(pyClosure) || PyTuple_Size(pyClosure) != expectedCellCount) { + throw std::runtime_error( + "Our closure of type " + + closureType->name() + + " means we expect the python function's closure to have " + + format(expectedCellCount) + + " variables but it has " + format(PyTuple_Size(pyClosure)) + ); + } } closureType->constructor(tgt, [&](instance_ptr tgtCell, int index) { Type* closureTypeInst = closureType->getTypes()[index]; - PyObject* cell = PyTuple_GetItem(pyClosure, index); - if (!cell) { - throw PythonExceptionSet(); - } + if (index < expectedCellCount) { + PyObject* cell = PyTuple_GetItem(pyClosure, index); + if (!cell) { + throw PythonExceptionSet(); + } - if (!PyCell_Check(cell)) { - throw std::runtime_error("Expected function closure to be made up of cells."); - } + if (!PyCell_Check(cell)) { + throw std::runtime_error("Expected function closure to be made up of cells."); + } + + if (closureTypeInst->getTypeCategory() == Type::TypeCategory::catPyCell) { + // our representation in the closure is itself a PyCell, so we just reference + // the actual cell object. + static PyCellType* pct = PyCellType::Make(); + pct->initializeFromPyObject(tgtCell, cell); + } else { + if (!PyCell_GET(cell)) { + throw std::runtime_error("Cell for " + closureType->getNames()[index] + " was empty."); + } - if (closureTypeInst->getTypeCategory() == Type::TypeCategory::catPyCell) { - // our representation in the closure is itself a PyCell, so we just reference - // the actual cell object. - static PyCellType* pct = PyCellType::Make(); - pct->initializeFromPyObject(tgtCell, cell); + PyInstance::copyConstructFromPythonInstance( + closureType->getTypes()[index], + tgtCell, + PyCell_GET(cell), + ConversionLevel::Implicit + ); + } } else { - if (!PyCell_GET(cell)) { - throw std::runtime_error("Cell for " + closureType->getNames()[index] + " was empty."); + PyObject* functionGlobals = PyFunction_GetGlobals(pyRepresentation); + if (!functionGlobals) { + throw std::runtime_error("Somehow this function didn't have globals?"); + } + + if (closureTypeInst != PythonObjectOfType::AnyPyDict()) { + throw std::runtime_error("Somehow the function closure type is improperly formed"); } PyInstance::copyConstructFromPythonInstance( closureType->getTypes()[index], tgtCell, - PyCell_GET(cell), + functionGlobals, ConversionLevel::Implicit ); } diff --git a/typed_python/PyFunctionInstance.hpp b/typed_python/PyFunctionInstance.hpp index a1aae8387..76290e536 100644 --- a/typed_python/PyFunctionInstance.hpp +++ b/typed_python/PyFunctionInstance.hpp @@ -30,13 +30,13 @@ class PyFunctionInstance : public PyInstance { static void copyConstructFromPythonInstanceConcrete(modeled_type* type, instance_ptr tgt, PyObject* pyRepresentation, ConversionLevel level); - static std::pair tryToCall(const Function* f, instance_ptr functionClosure, PyObject* arg0=nullptr, PyObject* arg1=nullptr, PyObject* arg2=nullptr); + static std::pair tryToCall(Function* f, instance_ptr functionClosure, PyObject* arg0=nullptr, PyObject* arg1=nullptr, PyObject* arg2=nullptr); - static std::pair tryToCallAnyOverload(const Function* f, instance_ptr functionClosure, PyObject* self, PyObject* args, PyObject* kwargs); + static std::pair tryToCallAnyOverload(Function* f, instance_ptr functionClosure, PyObject* self, PyObject* args, PyObject* kwargs); // determine the exact return type of a specific overload. Returns static std::pair getOverloadReturnType( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs ); @@ -45,7 +45,7 @@ class PyFunctionInstance : public PyInstance { // this calls signature functions, and checks for return type consistency. // Returns static std::pair determineReturnTypeForMatchedCall( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs, PyObject* self, @@ -60,7 +60,7 @@ class PyFunctionInstance : public PyInstance { static PyObject* prepareArgumentToBePassedToCompiler(PyObject* o); static std::pair tryToCallOverload( - const Function* f, + Function* f, instance_ptr funcClosure, long overloadIx, PyObject* self, @@ -74,7 +74,7 @@ class PyFunctionInstance : public PyInstance { //if 'isEntrypoint', then if we don't match a compiled specialization, ask the runtime to produce //one for us. static std::pair dispatchFunctionCallToNative( - const Function* f, + Function* f, instance_ptr functionClosure, long overloadIx, const FunctionCallArgMapping& mapping @@ -84,10 +84,10 @@ class PyFunctionInstance : public PyInstance { //we can't convert, then return . If we do dispatch, return and set //the python exception if native code returns an exception. static std::pair dispatchFunctionCallToCompiledSpecialization( - const Function::Overload& overload, + const FunctionOverload& overload, Type* closureType, instance_ptr closureData, - const Function::CompiledSpecialization& specialization, + const CompiledSpecialization& specialization, const FunctionCallArgMapping& mapping ); @@ -109,8 +109,6 @@ class PyFunctionInstance : public PyInstance { static PyObject* extractPyFun(PyObject* funcObj, PyObject* args, PyObject* kwargs); - static PyObject* extractOverloadGlobals(PyObject* funcObj, PyObject* args, PyObject* kwargs); - static PyObject* typeWithEntrypoint(PyObject* cls, PyObject* args, PyObject* kwargs); static PyObject* typeWithNocompile(PyObject* cls, PyObject* args, PyObject* kwargs); @@ -127,6 +125,7 @@ class PyFunctionInstance : public PyInstance { static Function* convertPythonObjectToFunctionType( PyObject* name, + PyObject* classname, PyObject *funcObj, bool assumeClosuresGlobal, // if true, then place closures in the function type itself // this is appropriate if this function is a method of a class diff --git a/typed_python/PyFunctionOverload.cpp b/typed_python/PyFunctionOverload.cpp new file mode 100644 index 000000000..0ea4d32ae --- /dev/null +++ b/typed_python/PyFunctionOverload.cpp @@ -0,0 +1,295 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyFunctionOverload.hpp" + + +PyDoc_STRVAR(PyFunctionOverload_doc, + "A single overload of a Function type object.\n\n" +); + +PyMethodDef PyFunctionOverload_methods[] = { + {NULL} /* Sentinel */ +}; + +FunctionOverload& PyFunctionOverload::getOverload() { + if (!mFunction) { + throw std::runtime_error("FunctionOverload has an empty FunctionType"); + } + + if (mOverloadIx < 0 || mOverloadIx >= mFunction->getOverloads().size()) { + throw std::runtime_error("FunctionOverload overloadIx out of bounds"); + } + + return mFunction->getOverloads()[mOverloadIx]; +} + +/* static */ +void PyFunctionOverload::dealloc(PyFunctionOverload *self) +{ + decref(self->mDict); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyFunctionOverload::newPyFunctionOverload(Function* f, int64_t overloadIndex) { + PyFunctionOverload* self = (PyFunctionOverload*)PyType_FunctionOverload.tp_alloc(&PyType_FunctionOverload, 0); + + self->mFunction = f; + self->mOverloadIx = overloadIndex; + self->mDict = PyDict_New(); + self->mIsInitialized = false; + + return (PyObject*)self; +} + +/* static */ +PyObject* PyFunctionOverload::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyFunctionOverload* self; + + self = (PyFunctionOverload*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mFunction = nullptr; + self->mOverloadIx = 0; + self->mDict = PyDict_New(); + self->mIsInitialized = false; + } + + return (PyObject*)self; +} + +void PyFunctionOverload::ensureInitialized() { + if (!mIsInitialized) { + initFields(); + } +} + +void PyFunctionOverload::initFields() { + if (mIsInitialized) { + throw std::runtime_error("Can't initialize a PyFunctionOverload twice."); + } + + FunctionOverload& overload = getOverload(); + + PyObjectStealer pyClosureVarsDict(PyDict_New()); + + // note that we can't actually call into the Python interpreter during this call, + // because that can release the GIL and allow other threads to access our type + // object before it's done. + PyObjectStealer argsTup(PyTuple_New(overload.getArgs().size())); + + PyObject* closureVariableCellLookupSingleton = staticPythonInstance("typed_python.internals", "CellAccess"); + PyObject* funcOverloadArg = staticPythonInstance("typed_python.internals", "FunctionOverloadArg"); + + PyObjectStealer pyFunctionGlobals(PyDict_New()); + + for (auto nameAndGlobal: overload.getGlobals()) { + PyDict_SetItemString( + pyFunctionGlobals, + nameAndGlobal.first.c_str(), + PyFunctionGlobal::newPyFunctionGlobal( + mFunction, + mOverloadIx, + nameAndGlobal.first + ) + ); + } + + for (auto nameAndClosureVar: overload.getClosureVariableBindings()) { + PyObjectStealer bindingObj(PyTuple_New(nameAndClosureVar.second.size())); + + for (long k = 0; k < nameAndClosureVar.second.size(); k++) { + ClosureVariableBindingStep step = nameAndClosureVar.second[k]; + + if (step.isFunction()) { + // recall that 'PyTuple_SetItem' steals a reference, so we need to incref it here + PyTuple_SetItem(bindingObj, k, incref((PyObject*)PyInstance::typePtrToPyTypeRepresentation(step.getFunction()))); + } else + if (step.isNamedField()) { + PyTuple_SetItem(bindingObj, k, PyUnicode_FromString(step.getNamedField().c_str())); + } else + if (step.isIndexedField()) { + PyTuple_SetItem(bindingObj, k, PyLong_FromLong(step.getIndexedField())); + } else + if (step.isCellAccess()) { + PyTuple_SetItem(bindingObj, k, incref(closureVariableCellLookupSingleton)); + } else { + throw std::runtime_error("Corrupt ClosureVariableBindingStep encountered"); + } + } + + PyDict_SetItemString(pyClosureVarsDict, nameAndClosureVar.first.c_str(), bindingObj); + } + + PyObjectStealer pyClosureVarnames( + PyTuple_New(overload.getFunctionClosureVarnames().size()) + ); + + for (long k = 0; k < overload.getFunctionClosureVarnames().size(); k++) { + PyTuple_SetItem( + pyClosureVarnames, + k, + PyUnicode_FromString(overload.getFunctionClosureVarnames()[k].c_str()) + ); + } + + PyObjectStealer emptyTup(PyTuple_New(0)); + PyObjectStealer emptyDict(PyDict_New()); + + for (long argIx = 0; argIx < overload.getArgs().size(); argIx++) { + auto arg = overload.getArgs()[argIx]; + + PyObjectStealer pyArgInst( + ((PyTypeObject*)funcOverloadArg)->tp_new((PyTypeObject*)funcOverloadArg, emptyTup, emptyDict) + ); + + PyObjectStealer pyArgInstDict(PyObject_GenericGetDict(pyArgInst, nullptr)); + + PyObjectStealer pyName(PyUnicode_FromString(arg.getName().c_str())); + PyDict_SetItemString(pyArgInstDict, "name", pyName); + PyDict_SetItemString(pyArgInstDict, "defaultValue", arg.getDefaultValue() ? PyTuple_Pack(1, arg.getDefaultValue()) : Py_None); + PyDict_SetItemString(pyArgInstDict, "_typeFilter", arg.getTypeFilter() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(arg.getTypeFilter()) : Py_None); + PyDict_SetItemString(pyArgInstDict, "isStarArg", arg.getIsStarArg() ? Py_True : Py_False); + PyDict_SetItemString(pyArgInstDict, "isKwarg", arg.getIsKwarg() ? Py_True : Py_False); + + PyTuple_SetItem(argsTup, argIx, incref(pyArgInst)); + } + + PyObject* funcTypeObj = PyInstance::typePtrToPyTypeRepresentation(mFunction); + PyDict_SetItemString(mDict, "functionTypeObject", funcTypeObj); + PyDict_SetItemString(mDict, "index", (PyObject*)PyLong_FromLong(mOverloadIx)); + + PyDict_SetItemString(mDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); + PyDict_SetItemString(mDict, "closureVars", (PyObject*)pyClosureVarnames); + PyDict_SetItemString(mDict, "functionCode", (PyObject*)overload.getFunctionCode()); + PyDict_SetItemString(mDict, "globals", (PyObject*)pyFunctionGlobals); + PyDict_SetItemString(mDict, "returnType", overload.getReturnType() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); + PyDict_SetItemString(mDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); + PyDict_SetItemString(mDict, "methodOf", overload.getMethodOf() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); + PyDict_SetItemString(mDict, "args", argsTup); + PyDict_SetItemString(mDict, "name", PyUnicode_FromString(mFunction->name().c_str())); + + mIsInitialized = true; +} + +/* static */ +int PyFunctionOverload::init(PyFunctionOverload *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "FunctionOverload cannot be initialized directly"); + return -1; +} + +// static +PyObject* PyFunctionOverload::tp_repr(PyObject *selfObj) { + PyFunctionOverload* self = (PyFunctionOverload*)selfObj; + + return PyUnicode_FromString( + ("FunctionOverload(" + self->mFunction->getOverloads()[self->mOverloadIx].toString() + ")").c_str() + ); +} + +PyObject* PyFunctionOverload::tp_getattro(PyObject *o, PyObject* attrName) { + PyFunctionOverload* pyFuncOverload = (PyFunctionOverload*)o; + + pyFuncOverload->ensureInitialized(); + + return translateExceptionToPyObject([&] { + if (!PyUnicode_Check(attrName)) { + throw std::runtime_error("Expected a string for attribute name"); + } + + std::string attr(PyUnicode_AsUTF8(attrName)); + + if (attr == "realizedGlobals") { + PyObject* pyFunctionGlobals = PyDict_New(); + + FunctionOverload& overload = pyFuncOverload->getOverload(); + + for (auto nameAndGlobal: overload.getGlobals()) { + PyObject* val = nameAndGlobal.second.getValueAsPyobj(); + + if (val) { + PyDict_SetItemString( + pyFunctionGlobals, + nameAndGlobal.first.c_str(), + val + ); + } + } + + return pyFunctionGlobals; + } + + return PyObject_GenericGetAttr(o, attrName); + }); +} + + +PyTypeObject PyType_FunctionOverload = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "FunctionOverload", + .tp_basicsize = sizeof(PyFunctionOverload), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyFunctionOverload::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = PyFunctionOverload::tp_repr, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = PyFunctionOverload::tp_getattro, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyFunctionOverload_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyFunctionOverload_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = offsetof(PyFunctionOverload, mDict), + .tp_init = (initproc) PyFunctionOverload::init, + .tp_alloc = 0, + .tp_new = PyFunctionOverload::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyFunctionOverload.hpp b/typed_python/PyFunctionOverload.hpp new file mode 100644 index 000000000..f6ba23830 --- /dev/null +++ b/typed_python/PyFunctionOverload.hpp @@ -0,0 +1,52 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" + +class PyFunctionOverload { +public: + PyObject_HEAD + + Function* mFunction; + + int64_t mOverloadIx; + + PyObject* mDict; + + bool mIsInitialized; + + FunctionOverload& getOverload(); + + void initFields(); + + void ensureInitialized(); + + static PyObject* tp_getattro(PyObject *o, PyObject* attrName); + + static PyObject* tp_repr(PyObject *self); + + static void dealloc(PyFunctionOverload *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyFunctionOverload(Function* f, int64_t overloadIndex); + + static int init(PyFunctionOverload *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_FunctionOverload; diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 080d89648..daaf304dd 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -40,7 +40,6 @@ #include "PyNoneInstance.hpp" #include "PyRegisterTypeInstance.hpp" #include "PyValueInstance.hpp" -#include "PyPythonSubclassInstance.hpp" #include "PyPythonObjectOfTypeInstance.hpp" #include "PyOneOfInstance.hpp" #include "PySubclassOfInstance.hpp" @@ -321,53 +320,56 @@ PyObject* PyInstance::extractPythonObjectConcrete(Type* eltType, instance_ptr da // static PyObject* PyInstance::tp_new_type(PyTypeObject *subtype, PyObject *args, PyObject *kwds) { - Type::TypeCategory catToProduce = ((NativeTypeCategoryWrapper*)subtype)->mCategory; - - if (catToProduce != Type::TypeCategory::catNamedTuple && - catToProduce != Type::TypeCategory::catAlternative) { - if (kwds && PyDict_Size(kwds)) { - PyErr_Format(PyExc_TypeError, "Type %S does not accept keyword arguments", (PyObject*)subtype); - return NULL; + return translateExceptionToPyObject([&]() -> PyObject* { + Type::TypeCategory catToProduce = ((NativeTypeCategoryWrapper*)subtype)->mCategory; + + if (catToProduce != Type::TypeCategory::catNamedTuple && + catToProduce != Type::TypeCategory::catAlternative) { + if (kwds && PyDict_Size(kwds)) { + PyErr_Format(PyExc_TypeError, "Type %S does not accept keyword arguments", (PyObject*)subtype); + return nullptr; + } } - } - if (catToProduce == Type::TypeCategory::catListOf) { return MakeListOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catTupleOf) { return MakeTupleOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catPointerTo ) { return MakePointerToType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catRefTo ) { return MakeRefToType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catTuple ) { return MakeTupleType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catConstDict ) { return MakeConstDictType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catSet ) { return MakeSetType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catDict ) { return MakeDictType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catOneOf ) { return MakeOneOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catNamedTuple ) { return MakeNamedTupleType(nullptr, args, kwds); } - if (catToProduce == Type::TypeCategory::catBool ) { return MakeBoolType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt8 ) { return MakeInt8Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt16 ) { return MakeInt16Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt32 ) { return MakeInt32Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt64 ) { return MakeInt64Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catFloat32 ) { return MakeFloat32Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catFloat64 ) { return MakeFloat64Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt8 ) { return MakeUInt8Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt16 ) { return MakeUInt16Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt32 ) { return MakeUInt32Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt64 ) { return MakeUInt64Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catString ) { return MakeStringType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catBytes ) { return MakeBytesType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catEmbeddedMessage ) { return MakeEmbeddedMessageType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catPyCell ) { return MakePyCellType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catTypedCell ) { return MakeTypedCellType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catNone ) { return MakeNoneType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catValue ) { return MakeValueType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catBoundMethod ) { return MakeBoundMethodType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catAlternativeMatcher ) { return MakeAlternativeMatcherType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catFunction ) { return MakeFunctionType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catClass ) { return MakeClassType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catAlternative ) { return MakeAlternativeType(nullptr, args, kwds); } - if (catToProduce == Type::TypeCategory::catSubclassOf ) { return MakeSubclassOfType(nullptr, args); } - - PyErr_Format(PyExc_TypeError, "unknown TypeCategory %S", (PyObject*)subtype); - return NULL; + if (catToProduce == Type::TypeCategory::catListOf) { return (PyObject*)MakeListOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catTupleOf) { return (PyObject*)MakeTupleOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catPointerTo ) { return (PyObject*)MakePointerToType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catRefTo ) { return (PyObject*)MakeRefToType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catTuple ) { return (PyObject*)MakeTupleType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catConstDict ) { return (PyObject*)MakeConstDictType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catSet ) { return (PyObject*)MakeSetType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catDict ) { return (PyObject*)MakeDictType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catOneOf ) { return (PyObject*)MakeOneOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catNamedTuple ) { return (PyObject*)MakeNamedTupleType(nullptr, args, kwds); } + if (catToProduce == Type::TypeCategory::catBool ) { return (PyObject*)MakeBoolType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt8 ) { return (PyObject*)MakeInt8Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt16 ) { return (PyObject*)MakeInt16Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt32 ) { return (PyObject*)MakeInt32Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt64 ) { return (PyObject*)MakeInt64Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catFloat32 ) { return (PyObject*)MakeFloat32Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catFloat64 ) { return (PyObject*)MakeFloat64Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt8 ) { return (PyObject*)MakeUInt8Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt16 ) { return (PyObject*)MakeUInt16Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt32 ) { return (PyObject*)MakeUInt32Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt64 ) { return (PyObject*)MakeUInt64Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catString ) { return (PyObject*)MakeStringType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catBytes ) { return (PyObject*)MakeBytesType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catEmbeddedMessage ) { return (PyObject*)MakeEmbeddedMessageType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catPyCell ) { return (PyObject*)MakePyCellType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catTypedCell ) { return (PyObject*)MakeTypedCellType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catNone ) { return (PyObject*)MakeNoneType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catValue ) { return (PyObject*)MakeValueType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catBoundMethod ) { return (PyObject*)MakeBoundMethodType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catAlternativeMatcher ) { return (PyObject*)MakeAlternativeMatcherType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catFunction ) { return (PyObject*)MakeFunctionType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catClass ) { return (PyObject*)MakeClassType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catAlternative ) { return (PyObject*)MakeAlternativeType(nullptr, args, kwds); } + if (catToProduce == Type::TypeCategory::catSubclassOf ) { return (PyObject*)MakeSubclassOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catForward ) { return (PyObject*)MakeForwardType(nullptr, args, kwds); } + + PyErr_Format(PyExc_TypeError, "unknown TypeCategory %S", (PyObject*)subtype); + return nullptr; + }); } // static @@ -381,40 +383,28 @@ PyObject* PyInstance::tp_new(PyTypeObject *subtype, PyObject *args, PyObject *kw eltType->assertForwardsResolvedSufficientlyToInstantiate(); - if (isSubclassOfNativeType(subtype)) { - PyInstance* self = (PyInstance*)subtype->tp_alloc(subtype, 0); - - try { - self->initializeEmpty(); - - self->initialize([&](instance_ptr data) { - constructFromPythonArguments(data, eltType, args, kwds); - }, eltType); + if (eltType->isClass() && ((Class*)eltType)->getOwnStaticFunctions().find("__typed_python_template__") + != ((Class*)eltType)->getOwnStaticFunctions().end()) { + auto matchedAndResult = PyFunctionInstance::tryToCallAnyOverload( + ((Class*)eltType)->getOwnStaticFunctions().find("__typed_python_template__")->second, + nullptr, + nullptr, + args, + kwds + ); - return (PyObject*)self; - } catch(...) { - subtype->tp_dealloc((PyObject*)self); - throw; + if (!matchedAndResult.first) { + throw std::runtime_error("Somehow, __typed_python_template__ didn't dispatch"); } - // not reachable - assert(false); - } else { - if (eltType->isClass() && ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__") - != ((Class*)eltType)->getOwnClassMembers().end()) { - return PyObject_Call( - ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__")->second, - args, - kwds - ); - } + return matchedAndResult.second; + } - Instance inst(eltType, [&](instance_ptr tgt) { - constructFromPythonArguments(tgt, eltType, args, kwds); - }); + Instance inst(eltType, [&](instance_ptr tgt) { + constructFromPythonArguments(tgt, eltType, args, kwds); + }); - return extractPythonObject(inst.data(), eltType); - } + return extractPythonObject(inst.data(), eltType); }); } @@ -714,6 +704,9 @@ int PyInstance::sq_ass_item_concrete(Py_ssize_t ix, PyObject* v) { PyTypeObject* PyInstance::typeObj(Type* inType) { if (!inType->getTypeRep()) { inType->setTypeRep(typeObjInternal(inType)); + if (!typeObjInternal(inType)) { + throw std::runtime_error("Somehow, typeObjInternal didn't produce a type."); + } } return inType->getTypeRep(); @@ -913,25 +906,6 @@ bool PyInstance::isNativeType(PyTypeObject* typeObj) { return typeObj->tp_as_buffer == bufferProcs(); } -/** - * Return true if the given PyTypeObject* is a subclass of a NativeType. - * This will return false when called with a native type -*/ -// static -bool PyInstance::isSubclassOfNativeType(PyTypeObject* typeObj) { - if (isNativeType(typeObj)) { - return false; - } - - while (typeObj) { - if (isNativeType(typeObj)) { - return true; - } - typeObj = typeObj->tp_base; - } - return false; -} - Type* PyInstance::rootNativeType(PyTypeObject* typeObj) { PyTypeObject* baseType = typeObj; @@ -948,10 +922,6 @@ Type* PyInstance::rootNativeType(PyTypeObject* typeObj) { // static Type* PyInstance::extractTypeFrom(PyTypeObject* typeObj, bool includePrimitives) { - if (isSubclassOfNativeType(typeObj)) { - return PythonSubclass::Make(rootNativeType(typeObj), typeObj); - } - if (isNativeType(typeObj)) { return ((NativeTypeWrapper*)typeObj)->mType; } @@ -1162,9 +1132,6 @@ PyTypeObject* PyInstance::typeCategoryBaseType(Type::TypeCategory category) { } PyTypeObject* PyInstance::typeObjInternal(Type* inType) { - if (inType->getTypeCategory() == ::Type::TypeCategory::catPythonSubclass) { - return inType->getTypeRep(); - } if (inType->getTypeCategory() == ::Type::TypeCategory::catInt64) { return &PyLong_Type; } @@ -1297,14 +1264,12 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { } PyType_Ready((PyTypeObject*)types[inType]); + inType->markTypeObjReady(); - PyDict_SetItemString( - types[inType]->typeObj.tp_dict, - "__typed_python_category__", - categoryToPyString(inType->getTypeCategory()) - ); - - mirrorTypeInformationIntoPyType(inType, &types[inType]->typeObj); + if (!inType->isActivelyBeingDeserialized()) { + finalizePyTypeObjectPhase1(inType, (PyTypeObject*)types[inType]); + finalizePyTypeObjectPhase2(inType, (PyTypeObject*)types[inType]); + } return (PyTypeObject*)types[inType]; } @@ -1574,13 +1539,22 @@ PyObject* PyInstance::tp_str_concrete() { // static bool PyInstance::typeCanBeSubclassed(Type* t) { return ( - t->getTypeCategory() == Type::TypeCategory::catNamedTuple || t->getTypeCategory() == Type::TypeCategory::catClass ); } // static -void PyInstance::mirrorTypeInformationIntoPyType(Type* inType, PyTypeObject* pyType) { +void PyInstance::finalizePyTypeObjectPhase1(Type* inType, PyTypeObject* pyType) { + //pyType->tp_name = (new std::string(inType->nameWithModule()))->c_str(); +} + +void PyInstance::finalizePyTypeObjectPhase2(Type* inType, PyTypeObject* pyType) { + PyDict_SetItemString( + pyType->tp_dict, + "__typed_python_category__", + categoryToPyString(inType->getTypeCategory()) + ); + specializeStatic(inType->getTypeCategory(), [&](auto* concrete_null_ptr) { typedef typename std::remove_reference::type py_instance_type; @@ -1590,6 +1564,7 @@ void PyInstance::mirrorTypeInformationIntoPyType(Type* inType, PyTypeObject* pyT ); }); } + void PyInstance::mirrorTypeInformationIntoPyTypeConcrete(Type* inType, PyTypeObject* pyType) { //noop } @@ -1628,7 +1603,6 @@ PyObject* PyInstance::categoryToPyString(Type::TypeCategory cat) { if (cat == Type::TypeCategory::catConstDict) { static PyObject* res = PyUnicode_FromString("ConstDict"); return res; } if (cat == Type::TypeCategory::catAlternative) { static PyObject* res = PyUnicode_FromString("Alternative"); return res; } if (cat == Type::TypeCategory::catConcreteAlternative) { static PyObject* res = PyUnicode_FromString("ConcreteAlternative"); return res; } - if (cat == Type::TypeCategory::catPythonSubclass) { static PyObject* res = PyUnicode_FromString("PythonSubclass"); return res; } if (cat == Type::TypeCategory::catBoundMethod) { static PyObject* res = PyUnicode_FromString("BoundMethod"); return res; } if (cat == Type::TypeCategory::catAlternativeMatcher) { static PyObject* res = PyUnicode_FromString("AlternativeMatcher"); return res; } if (cat == Type::TypeCategory::catClass) { static PyObject* res = PyUnicode_FromString("Class"); return res; } @@ -1768,6 +1742,17 @@ Type* PyInstance::tryUnwrapPyInstanceToType(PyObject* arg) { return possibleType; } + if (PyFunction_Check(arg)) { + try { + return Forward::MakeFromFunction(arg); + } catch(std::exception& e) { + return nullptr; + } catch(PythonExceptionSet& e) { + PyErr_Clear(); + return nullptr; + } + } + if (PyDict_Contains(nonTypesAcceptedAsTypes(), arg)) { PyObject* nonType = PyDict_GetItem(nonTypesAcceptedAsTypes(), arg); if (!nonType) { @@ -1775,8 +1760,7 @@ Type* PyInstance::tryUnwrapPyInstanceToType(PyObject* arg) { } return PythonObjectOfType::Make( - (PyTypeObject*)nonType, - arg + (PyTypeObject*)nonType ); } @@ -1811,30 +1795,14 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { return StringType::Make(); } - if (PyInstance::isSubclassOfNativeType(pyType)) { - Type* nativeT = PyInstance::rootNativeType(pyType); + Type* res = PyInstance::extractTypeFrom(pyType); - if (!nativeT) { - PyErr_SetString(PyExc_TypeError, - ("Type " + std::string(pyType->tp_name) + " looked like a native type subclass, but has no base").c_str() - ); - return NULL; - } - - //this is now a permanent object - incref(typearg); - - return PythonSubclass::Make(nativeT, pyType); + if (res) { + // we have a native type -> return it + return res; } else { - Type* res = PyInstance::extractTypeFrom(pyType); - - if (res) { - // we have a native type -> return it - return res; - } else { - // we have a python type -> wrap it - return PythonObjectOfType::Make(pyType); - } + // we have a python type -> wrap it + return PythonObjectOfType::Make(pyType); } } @@ -1845,11 +1813,19 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { } return PythonObjectOfType::Make( - (PyTypeObject*)nonType, - typearg + (PyTypeObject*)nonType ); } + if (PyFunction_Check(typearg)) { + try { + return Forward::MakeFromFunction(typearg); + } catch(std::exception& e) { + PyErr_Format(PyExc_TypeError, e.what()); + return NULL; + } + } + // else: typearg is not a type -> it is a value Type* valueType = PyInstance::tryUnwrapPyInstanceToValueType(typearg, false); @@ -1857,6 +1833,7 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { return valueType; } + PyErr_Format(PyExc_TypeError, "Cannot convert %R to a typed_python Type", typearg); return NULL; } diff --git a/typed_python/PyInstance.hpp b/typed_python/PyInstance.hpp index c3f40e5a0..76190c617 100644 --- a/typed_python/PyInstance.hpp +++ b/typed_python/PyInstance.hpp @@ -52,7 +52,6 @@ class PyFunctionInstance; class PyBoundMethodInstance; class PyNoneInstance; class PyValueInstance; -class PyPythonSubclassInstance; class PyPythonObjectOfTypeInstance; class PyOneOfInstance; class PySubclassOfInstance; @@ -166,8 +165,6 @@ class PyInstance { return f(*(PyConcreteAlternativeInstance*)obj); case Type::TypeCategory::catAlternativeMatcher: return f(*(PyAlternativeMatcherInstance*)obj); - case Type::TypeCategory::catPythonSubclass: - return f(*(PyPythonSubclassInstance*)obj); case Type::TypeCategory::catPythonObjectOfType: return f(*(PyPythonObjectOfTypeInstance*)obj); case Type::TypeCategory::catClass: @@ -247,8 +244,6 @@ class PyInstance { return f((PyAlternativeInstance*)nullptr); case Type::TypeCategory::catConcreteAlternative: return f((PyConcreteAlternativeInstance*)nullptr); - case Type::TypeCategory::catPythonSubclass: - return f((PyPythonSubclassInstance*)nullptr); case Type::TypeCategory::catPythonObjectOfType: return f((PyPythonObjectOfTypeInstance*)nullptr); case Type::TypeCategory::catClass: @@ -568,8 +563,6 @@ class PyInstance { static PyMappingMethods* mappingMethods(Type* t); - static bool isSubclassOfNativeType(PyTypeObject* typeObj); - static Type* rootNativeType(PyTypeObject* typeObj); static bool isNativeType(PyTypeObject* typeObj); @@ -579,6 +572,14 @@ class PyInstance { // float and int, which happen to have both PyTypeObject* and Type* representations static Type* extractTypeFrom(PyTypeObject* typeObj, bool includePrimitives=false); + static Type* extractTypeFrom(PyObject* typeObj, bool includePrimitives=false) { + if (!PyType_Check(typeObj)) { + return nullptr; + } + + return extractTypeFrom((PyTypeObject*)typeObj, includePrimitives); + } + // if 'obj' is a PyInstance, return its type and data ptrs, after unwrapping // any 'RefTo' classes, which is the standard behavior for most classes, // which don't specifically know anything about 'RefTo' @@ -640,7 +641,9 @@ class PyInstance { */ static PyTypeObject* typeObjInternal(Type* inType); - static void mirrorTypeInformationIntoPyType(Type* inType, PyTypeObject* pyType); + static void finalizePyTypeObjectPhase1(Type* inType, PyTypeObject* pyType); + + static void finalizePyTypeObjectPhase2(Type* inType, PyTypeObject* pyType); static void mirrorTypeInformationIntoPyTypeConcrete(Type* inType, PyTypeObject* pyType); diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp new file mode 100644 index 000000000..f15b5ed28 --- /dev/null +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -0,0 +1,445 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyObjGraphSnapshot.hpp" +#include "PyObjSnapshotGroupSearch.hpp" + +PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBack, bool linkToInternal) : + mGroupsConsumed(0), + mGroupSearch(this) +{ + std::unordered_map objMapCache; + std::unordered_map typeMapCache; + std::unordered_map instanceCache; + + // this finds every reachable python objects from 'root', with the search terminating at + // named module objects and named module dicts. This is enough information to completely + // characterize a type's "identity hash", since there can be at most one version of a + // given named module per program. + PyObjSnapshotMaker snapMaker( + objMapCache, + typeMapCache, + instanceCache, + this, + linkBack, + linkToInternal + ); + + snapMaker.internalize(root); +} + +/* walk over the graph and point any forwards to the type they actually point to. + +This will modify nodes in the following way: + Any Kind::ForwardType will have a target set if its resolvable + Any snapshot pointing at a Forward will now point at what the forward points to + Any snapshot with a valid cache that can reach a type that was changed will have its + cache cleared + +*/ +void PyObjGraphSnapshot::resolveForwards() { + // any snapshot that was modified + std::set modified; + + // point forwards where they go + for (auto o: mObjects) { + if (o->getKind() == PyObjSnapshot::Kind::ForwardType) { + o->pointForwardToFinalType(); + modified.insert(o); + } else if (o->willBeATpType()) { + if (o->markTypeNotFwdDefined()) { + modified.insert(o); + } + } + } + + for (auto o: mObjects) { + if (o->replaceOutboundForwardsWithTargets()) { + modified.insert(o); + } + } + + // build a reverse graph map + std::map > inbound; + + for (auto o: mObjects) { + o->visitOutbound([&](PyObjSnapshot* outbound) { + if (outbound->getGraph() == this) { + inbound[outbound].insert(o); + } + }); + } + + std::set modifiedTransitive; + std::function check = [&](PyObjSnapshot* s) { + if (modifiedTransitive.find(s) == modifiedTransitive.end()) { + modifiedTransitive.insert(s); + for (auto upstream: inbound[s]) { + check(upstream); + } + } + }; + + for (auto m: modified) { + check(m); + } + + for (auto m: modifiedTransitive) { + m->clearCache(); + } +} + +void PyObjGraphSnapshot::recomputeNames() { + for (auto o: mObjects) { + mGroupSearch.add(o, false); + } + + for (auto& group: mGroupSearch.getGroups()) { + for (auto elt: *group) { + if (elt->willBeATpType()) { + elt->recomputeRecursiveName(*group); + } + } + } +} + +void PyObjGraphSnapshot::internalize() { + internal().internalize(*this, false); +} + +void PyObjGraphSnapshot::internalize(PyObjGraphSnapshot& otherGraph, bool markInternalized) { + std::map > skeletons; + + for (auto s: otherGraph.mObjects) { + ShaHash h = otherGraph.hashFor(s); + + if (!snapshotForHash(h)) { + skeletons[h] = std::make_pair(s, createSkeleton(h)); + } + } + + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->cloneFromSnapByHash(hashAndSkeleton.second.first); + } + + if (markInternalized) { + try { + // rehydrate this portion of the graph, so that we have concrete objects for everything + // we just interned + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->rehydrate(); + } + + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->markInternalizedOnType(); + } + } catch(...) { + // undo this since the state is somehow corrupt + for (auto hashAndSkeleton: skeletons) { + mObjectsByIndex.pop_back(); + mObjects.erase(hashAndSkeleton.second.second); + mHashToSnap.erase(hashAndSkeleton.first); + } + throw; + } + } +} + +PyObjSnapshot* PyObjGraphSnapshot::createSkeleton(ShaHash h) { + if (mHashToSnap.find(h) != mHashToSnap.end()) { + throw std::runtime_error("Can't create a skeleton for an object that already exists"); + } + + PyObjSnapshot* snap = new PyObjSnapshot(this); + + mHashToSnap[h] = snap; + mObjects.insert(snap); + mObjectsByIndex.push_back(snap); + return snap; +} + + +template +ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { + if (snap->getKind() == PyObjSnapshot::Kind::PrimitiveInstance) { + if (snap->getInstance().type()->isPOD()) { + return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + + ShaHash::SHA1(snap->getInstance().data(), snap->getInstance().type()->bytecount()); + } + if (snap->getInstance().type()->isBytes()) { + static BytesType* b = BytesType::Make(); + + return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + + ShaHash::SHA1( + b->eltPtr(snap->getInstance().data(), 0), + b->count(snap->getInstance().data()) + ); + } + if (snap->getInstance().type()->isString()) { + static StringType* b = StringType::Make(); + + std::string s = b->toUtf8String(snap->getInstance().data()); + + return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + + ShaHash::SHA1( + &s[0], + s.size() + ); + } + throw std::runtime_error( + "Can't hash a PyObjSnapshot Instance of type " + snap->getInstance().type()->name() + ); + } + if (snap->getKind() == PyObjSnapshot::Kind::PrimitiveType) { + return ShaHash(int(snap->getKind())) + ShaHash(int(snap->getType()->getTypeCategory())); + } + if (snap->getKind() == PyObjSnapshot::Kind::ArbitraryPyObject) { + // probably we should try to serialize it? + if (PyType_Check(snap->getPyObj())) { + throw std::runtime_error( + "Can't hash a PyObjSnapshot.ArbitraryPyObject for type " + + std::string(((PyTypeObject*)snap->getPyObj())->tp_name) + ); + } + + throw std::runtime_error( + "Can't hash a PyObjSnapshot.ArbitraryPyObject of type " + + std::string(snap->getPyObj()->ob_type->tp_name) + ); + } + + ShaHash res(int(snap->getKind())); + + if (snap->getStringValue().size()) { + res = res + ShaHash(1) + ShaHash(snap->getStringValue()); + } + if (snap->getName().size()) { + res = res + ShaHash(2) + ShaHash(snap->getName()); + } + if (snap->getModuleName().size()) { + res = res + ShaHash(3) + ShaHash(snap->getModuleName()); + } + if (snap->getQualname().size()) { + res = res + ShaHash(4) + ShaHash(snap->getQualname()); + } + if (snap->names().size()) { + res = res + ShaHash(6) + ShaHash(snap->names().size()); + + for (auto n: snap->names()) { + res = res + ShaHash(n); + } + } + if (snap->elements().size()) { + res = res + ShaHash(7) + ShaHash(snap->elements().size()); + + for (auto n: snap->elements()) { + res = res + compute(n); + } + } + if (snap->keys().size()) { + res = res + ShaHash(8) + ShaHash(snap->keys().size()); + + for (auto n: snap->keys()) { + res = res + compute(n); + } + } + if (snap->namedElements().size()) { + res = res + ShaHash(10) + ShaHash(snap->namedElements().size()); + + for (auto& nameAndElt: snap->namedElements()) { + res = res + ShaHash(nameAndElt.first) + compute(nameAndElt.second); + } + } + if (snap->namedInts().size()) { + res = res + ShaHash(11) + ShaHash(snap->namedInts().size()); + + for (auto& nameAndElt: snap->namedInts()) { + res = res + ShaHash(nameAndElt.first) + ShaHash(nameAndElt.second); + } + } + + return res; +} + + +PyObjSnapshot* PyObjGraphSnapshot::snapshotForHash(ShaHash hash) { + auto it = mHashToSnap.find(hash); + if (it != mHashToSnap.end()) { + return it->second; + } + return nullptr; +} + + +ShaHash PyObjGraphSnapshot::hashFor(PyObjSnapshot* snap) { + if (snap->getGraph() != this) { + throw std::runtime_error("Can't graph a hash of a PyObjSnapshot we don't have"); + } + + auto it = mSnapToHash.find(snap); + if (it != mSnapToHash.end()) { + return it->second; + } + + mGroupSearch.add(snap); + + while (mGroupsConsumed < mGroupSearch.getGroups().size()) { + computeHashesFor(*mGroupSearch.getGroups()[mGroupsConsumed++]); + } + + it = mSnapToHash.find(snap); + + if (it != mSnapToHash.end()) { + return it->second; + } + + throw std::runtime_error("PyObjGraphSnapshot failed to produce a hash for one of its objects."); +} + +void PyObjGraphSnapshot::computeHashesFor(const std::unordered_set& group) { + PyObjSnapshot* root = pickRootFor(group); + + // order our snapshots by the order in which we would + // visit them + std::unordered_map snapsSeen; + std::vector snapsInOrder; + + std::function visit = [&](PyObjSnapshot* s) { + // if its not in the group, ignore it + if (group.find(s) == group.end()) { + return; + } + + // if we saw it already, skip it + if (snapsSeen.find(s) != snapsSeen.end()) { + return; + } + + // record it + int snapIx = snapsSeen.size(); + snapsSeen[s] = snapIx; + snapsInOrder.push_back(s); + + // visit our outbound edges + s->visitOutbound(visit); + }; + + visit(root); + + if (snapsSeen.size() != group.size()) { + throw std::runtime_error("PyObjGraphSnapshot::computeHashFor failed to find all snaps"); + } + + ShaHash groupHash = ShaHash(snapsInOrder.size()); + + for (long i = 0; i < snapsInOrder.size(); i++) { + ShaHash thisSnapHash = computeHashFor( + snapsInOrder[i], + [&](PyObjSnapshot* snap) { + auto it = snapsSeen.find(snap); + if (it != snapsSeen.end()) { + return ShaHash(1) + ShaHash(it->second); + } + auto it2 = mSnapToHash.find(snap); + if (it2 != mSnapToHash.end()) { + return ShaHash(2) + it2->second; + } + + throw std::runtime_error("PyObjGraphSnapshot::computeHashFor failed to find a snap."); + } + ); + + groupHash = groupHash + ShaHash(i) + thisSnapHash; + } + + for (long i = 0; i < snapsInOrder.size(); i++) { + installSnapHash(snapsInOrder[i], groupHash + ShaHash(i)); + } +} + +void PyObjGraphSnapshot::installSnapHash(PyObjSnapshot* snap, ShaHash h) { + mSnapToHash[snap] = h; + + if (!mHashToSnap[h]) { + mHashToSnap[h] = snap; + } else { + // this can happen any time in a single graph we have multiple copies of objects that + // look the same (as far as what can be seen from their references). As an example, + // the empty tuple or empty list will always be the same hash in any graph + mRedundantSnaps.insert(snap); + } +} + +PyObjSnapshot* PyObjGraphSnapshot::pickRootFor(const std::unordered_set& group) { + if (!group.size()) { + throw std::runtime_error("Can't pick a root for an empty group"); + } + + if (group.size() == 1) { + return *group.begin(); + } + + // pick a 'root' node by computing a shallow hash and taking the first value for which + // there is only one hash + std::unordered_map curHashes; + std::unordered_map newHashes; + long passes = 0; + + while (passes < group.size() + 1) { + passes += 1; + + for (auto s: group) { + ShaHash h = computeHashFor(s, [&](PyObjSnapshot* edge) { + if (group.find(edge) == group.end()) { + auto it = mSnapToHash.find(edge); + if (it != mSnapToHash.end()) { + return it->second; + } + + throw std::runtime_error("Found a PyObjSnapshot that's not in our graph"); + } + + return curHashes[s]; + }); + + newHashes[s] = h; + } + + // find the first hash that's unique + std::map > toHash; + for (auto snapAndHash: newHashes) { + toHash[snapAndHash.second].insert(snapAndHash.first); + } + + for (auto& hashAndObj: toHash) { + if (hashAndObj.second.size() == 1) { + return *hashAndObj.second.begin(); + } + } + } + + // we didn't find one even after N passes, so our objects look identical. + // just pick one. + return *group.begin(); +} + +void PyObjGraphSnapshot::getTypes(std::unordered_set& outTypes) { + for (auto snap: mObjects) { + if (snap->willBeATpType()) { + outTypes.insert(snap); + } + } +} diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp new file mode 100644 index 000000000..e8713c196 --- /dev/null +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -0,0 +1,131 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include +#include "PyObjSnapshotGroupSearch.hpp" +#include "ShaHash.hpp" + +class PyObjSnapshot; + + +/********************************* +PyObjGraphSnapshot + +Holds a collection of PyObjSnapshot objects. This graph object owns its snapshots +and frees them when it iself is release. + +There's a global PyObjGraphSnapshot that holds all the interned objects accessible from + + PyObjGraphSnapshot::internal() + +A graph knows how to assign a unique hash to every object it contains. + +**********************************/ + +class PyObjGraphSnapshot { +public: + PyObjGraphSnapshot() : + mGroupsConsumed(0), + mGroupSearch(this) + { + } + + PyObjGraphSnapshot(Type* root, bool linkBackToOriginal=true, bool linkToInternal=true); + + ~PyObjGraphSnapshot() { + for (auto oPtr: mObjects) { + delete oPtr; + } + } + + // walk over the graph and point any forwards to the type they actually point to. + // after this, any Kind::ForwardType will have a target set and empty name. + // Any outbound link pointing to a Forward will now point to what the forward was + // pointing to. Any Snapshots that had valid caches that were modified or that can + // reach modified values will have their snapshots cleared. + void resolveForwards(); + + // copy this into the 'internal' graph. + void internalize(); + + // internalize a graph into ourselves + void internalize(PyObjGraphSnapshot& graph, bool markInternalized); + + void recomputeNames(); + + // get the "internal" graph snapshot, which is responsible for holding all the objects + // that are actually interned inside the system. + static PyObjGraphSnapshot& internal() { + static PyObjGraphSnapshot* graph = new PyObjGraphSnapshot(); + return *graph; + } + + size_t registerSnapshot(PyObjSnapshot* obj) { + if (obj->getGraph() != this) { + throw std::runtime_error("Can't register a snapshot for a different graph"); + } + size_t ix = mObjects.size(); + mObjects.insert(obj); + mObjectsByIndex.push_back(obj); + + return ix; + } + + const std::unordered_set& getObjects() const { + return mObjects; + } + + const std::vector& getObjectsByIndex() const { + return mObjectsByIndex; + } + + ShaHash hashFor(PyObjSnapshot* snap); + + PyObjSnapshot* snapshotForHash(ShaHash h); + + // all contained snapshots that represent TP types + void getTypes(std::unordered_set& outTypeSnaps); + +private: + PyObjSnapshot* createSkeleton(ShaHash h); + + void installSnapHash(PyObjSnapshot* snap, ShaHash h); + + void computeHashesFor(const std::unordered_set& group); + + PyObjSnapshot* pickRootFor(const std::unordered_set& group); + + // all of our objects + std::unordered_set mObjects; + + std::vector mObjectsByIndex; + + // for each snapshot, a hash + std::unordered_map mSnapToHash; + + // for each hash, the object that matches it. It's possible to produce objects in a graph + // that are indistinguishable from each other, in which case we simply pick one to go + // 'first' and include that extra information in the hash. + std::unordered_map mHashToSnap; + + // snaps with the same sha hash as each other + std::unordered_set mRedundantSnaps; + + PyObjSnapshotGroupSearch mGroupSearch; + long mGroupsConsumed; +}; diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp new file mode 100644 index 000000000..7937c4b33 --- /dev/null +++ b/typed_python/PyObjRehydrator.cpp @@ -0,0 +1,1132 @@ +#include "PyObjRehydrator.hpp" +#include "PyObjSnapshot.hpp" + +void PyObjRehydrator::start(PyObjSnapshot* snapshot) { + std::function visit = [&](PyObjSnapshot* snap) { + if (mSnapshots.find(snap) != mSnapshots.end()) { + return; + } + + if (!snap->isUncacheable()) { + if (!snap->needsHydration()) { + return; + } + + mSnapshots.insert(snap); + } + + for (auto elt: snap->mElements) { + visit(elt); + } + + for (auto elt: snap->mKeys) { + visit(elt); + } + + for (auto& nameAndElt: snap->mNamedElements) { + visit(nameAndElt.second); + } + }; + + visit(snapshot); +} + +void PyObjRehydrator::buildPyobjForTpType(PyObjSnapshot* s) { + if (!s->mType) { + throw std::runtime_error("Somehow this Snapshot doesn't have a Type."); + } + if (s->mPyObject) { + return; + } + + // these objects don't actually build a python representation for the type - we always + // use the actual type + if (s->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + return; + } + + s->mPyObject = incref((PyObject*)PyInstance::typeObj(s->mType)); + + if (!s->mPyObject) { + throw std::runtime_error("Somehow we don't have a PyObject for our type."); + } +} + + +void PyObjRehydrator::rehydrateAll() { + // first, do a pass building the skeletons for all of our objects. + // after this, there will be a Type* allocated for every new Type object with the + // correct name, which is enough to construct the PyTypeObject for it. + for (auto s: mSnapshots) { + if (s->willBeATpType()) { + buildTypeSkeleton(s); + } + } + + // now that every TP type knows its name, we can instantiate it + for (auto s: mSnapshots) { + if (s->willBeATpType()) { + buildPyobjForTpType(s); + } + } + + // then actually initializethe type objects. these will build PyObjects as needed + for (auto s: mSnapshots) { + if (s->willBeATpType()) { + rehydrateTpType(s); + } + } + + for (auto s: mSnapshots) { + rehydrate(s); + } + + for (auto n: mSnapshots) { + finalizeRehydration(n); + } + + std::vector types; + for (auto o: mSnapshots) { + if (o->mType) { + types.push_back(o->mType); + } + } + + std::sort(types.begin(), types.end(), [&](Type* l, Type* r) { + if (l->typeLevel() < r->typeLevel()) { + return true; + } + if (l->typeLevel() > r->typeLevel()) { + return false; + } + return l < r; + }); + + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto t: types) { + if (t->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > types.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } + } + + // let each type update any internal caches it might need before it gets instantiated + for (auto t: types) { + t->finalizeType(); + } + + for (auto t: types) { + t->typeFinishedBeingDeserializedPhase1(); + } + + // now that we've set function globals, we need to go over all the classes and + // held classes and make sure the versions of the function types they're actually + // using have the new definitions. Unfortunately, Overload objects are not pointers + // (maybe they should be?) and so when we merge Overloads from base/child classes + // they don't get their globals replaced with the appropriate values + for (auto& t: types) { + if (t && t->isHeldClass()) { + ((HeldClass*)t)->mergeOwnFunctionsIntoInheritanceTree(); + } + } + + for (auto t: types) { + t->typeFinishedBeingDeserializedPhase2(); + } + + for (auto obj: mSnapshots) { + if (obj->mType) { + obj->mPyObject = incref((PyObject*)PyInstance::typeObj(obj->mType)); + } + } +} + +Type* PyObjRehydrator::typeFor(PyObjSnapshot* snapshot) { + if (!snapshot->needsHydration()) { + return snapshot->getType(); + } + + if (snapshot->willBeATpType()) { + rehydrate(snapshot); + return snapshot->mType; + } + + return nullptr; +} + + +PyObject* PyObjRehydrator::pyobjFor(PyObjSnapshot* snapshot) { + if (!snapshot->needsHydration()) { + return snapshot->getPyObj(); + } + + rehydrate(snapshot); + + return snapshot->mPyObject; +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + Type*& out +) { + out = typeFor(snapshot); + if (!out) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a type"); + } +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + PyObject*& out +) { + out = pyobjFor(snapshot); + if (!out) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a pyobj"); + } +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + HeldClass*& out +) { + Type* t = typeFor(snapshot); + if (!t->isHeldClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a HeldClass"); + } + out = (HeldClass*)t; +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + Function*& out +) { + Type* t = typeFor(snapshot); + if (!t->isFunction()) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a Function"); + } + out = (Function*)t; +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* e, + MemberDefinition& out +) { + out = MemberDefinition( + e->mName, + getNamedElementType(e, "type"), + getNamedElementInstance(e, "defaultValue"), + e->mNamedInts["isNonempty"] + ); +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + FunctionOverload& out +) { + std::map globals; + std::vector pyFuncClosureVarnames; + std::vector globalsInClosureVarnames; + std::map closureBindings; + std::vector args; + + getNamedBundle(snapshot, "func_globals", globals); + getNamedBundle(snapshot, "func_closure_varnames", pyFuncClosureVarnames); + getNamedBundle(snapshot, "func_globals_in_closure_varnames", globalsInClosureVarnames); + getNamedBundle(snapshot, "func_closure_variable_bindings", closureBindings); + getNamedBundle(snapshot, "func_args", args); + + out = FunctionOverload( + getNamedElementPyobj(snapshot, "func_code"), + getNamedElementPyobj(snapshot, "func_defaults", true), + getNamedElementPyobj(snapshot, "func_annotations", true), + globals, + pyFuncClosureVarnames, + globalsInClosureVarnames, + closureBindings, + getNamedElementType(snapshot, "func_ret_type", true), + getNamedElementPyobj(snapshot, "func_signature_func", true), + args, + getNamedElementType(snapshot, "func_method_of", true) + ); +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + std::string& out +) { + if (!snapshot || snapshot->mKind != PyObjSnapshot::Kind::String) { + throw std::runtime_error("Corrupt PyObjSnapshot.String"); + } + + out = snapshot->getStringValue(); +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + FunctionGlobal& out +) { + if (snapshot->mNamedElements.find("moduleDict") != snapshot->mNamedElements.end() && snapshot->mModuleName.size()) { + out = FunctionGlobal::NamedModuleMember( + getNamedElementPyobj(snapshot, "moduleDict"), + snapshot->mModuleName, + snapshot->mName + ); + return; + } + + if (snapshot->mNamedElements.find("moduleDict") != snapshot->mNamedElements.end()) { + out = FunctionGlobal::GlobalInDict( + getNamedElementPyobj(snapshot, "moduleDict"), + snapshot->mName + ); + return; + } + + if (snapshot->mNamedElements.find("cell") != snapshot->mNamedElements.end()) { + out = FunctionGlobal::GlobalInCell( + getNamedElementPyobj(snapshot, "cell") + ); + return; + } +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + FunctionArg& out +) { + Type* filter = getNamedElementType(snapshot, "arg_type_filter", true); + PyObject* defaultVal = getNamedElementPyobj(snapshot, "arg_default_value", true); + bool isStarArg = snapshot->mNamedInts["arg_is_star_arg"]; + bool isKwarg = snapshot->mNamedInts["arg_is_kwarg"]; + + out = FunctionArg( + snapshot->mName, + filter, + defaultVal, + isStarArg, + isKwarg + ); +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + ClosureVariableBinding& out +) { + std::vector steps; + + for (auto e: snapshot->mElements) { + ClosureVariableBindingStep step; + getFrom(e, step); + steps.push_back(step); + } + + out = ClosureVariableBinding(steps); +} + +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + ClosureVariableBindingStep& out +) { + if (snapshot->mNamedElements.find("step_func") != snapshot->mNamedElements.end()) { + Function* f; + getFrom(snapshot->mNamedElements.find("step_func")->second, f); + out = ClosureVariableBindingStep(f); + return; + } + + if (snapshot->mNames.size()) { + out = ClosureVariableBindingStep(snapshot->mNames[0]); + return; + } + + if (snapshot->mNamedInts.find("step_index") != snapshot->mNamedInts.end()) { + out = ClosureVariableBindingStep(snapshot->mNamedInts["step_index"]); + return; + } +} + +Instance PyObjRehydrator::getNamedElementInstance( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + // TODO: this is just wrong + return snapshot->mInstance; +} + +PyObject* PyObjRehydrator::getNamedElementPyobj( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + if (allowEmpty) { + return nullptr; + } + throw std::runtime_error( + "Corrupt PyObjSnapshot." + snapshot->kindAsString() + ". missing " + name + ); + } + + if (!it->second) { + throw std::runtime_error("Corrupt PyObjSnapshot: nullptr in mNamedElements"); + } + + PyObject* o = pyobjFor(it->second); + + if (!o && !allowEmpty) { + throw std::runtime_error("Corrupt " + snapshot->toString() + + ": no pyobj for elt " + name + " which is " + it->second->toString() + ); + } + + return pyobjFor(it->second); +} + +Type* PyObjRehydrator::getNamedElementType( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + if (allowEmpty) { + return nullptr; + } + throw std::runtime_error( + "Corrupt PyObjSnapshot." + snapshot->kindAsString() + ". missing " + name + ); + } + + return typeFor(it->second); +} + +void PyObjRehydrator::buildTypeSkeleton(PyObjSnapshot* snap) { + if (snap->mKind == PyObjSnapshot::Kind::PrimitiveType) { + // nothing to do + return; + } + + if (snap->mType) { + throw std::runtime_error("Somehow this snapshot already has a Type?"); + } + + if (snap->mKind == PyObjSnapshot::Kind::ListOfType) { + snap->mType = new ListOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::TupleOfType) { + snap->mType = new TupleOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::DictType) { + snap->mType = new DictType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ConstDictType) { + snap->mType = new ConstDictType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::SetType) { + snap->mType = new SetType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::PointerToType) { + snap->mType = new PointerTo(); + } else + if (snap->mKind == PyObjSnapshot::Kind::RefToType) { + snap->mType = new RefTo(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ValueType) { + snap->mType = new Value(); + } else + if (snap->mKind == PyObjSnapshot::Kind::OneOfType) { + snap->mType = new OneOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + snap->mType = new PythonObjectOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::TupleType) { + snap->mType = new Tuple(); + } else + if (snap->mKind == PyObjSnapshot::Kind::NamedTupleType) { + snap->mType = new NamedTuple(); + } else + if (snap->mKind == PyObjSnapshot::Kind::BoundMethodType) { + snap->mType = new BoundMethod(); + } else + if (snap->mKind == PyObjSnapshot::Kind::TypedCellType) { + snap->mType = new TypedCellType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ForwardType) { + snap->mType = new Forward(snap->mName); + } else + if (snap->mKind == PyObjSnapshot::Kind::ConcreteAlternativeType) { + snap->mType = new ConcreteAlternative(); + } else + if (snap->mKind == PyObjSnapshot::Kind::AlternativeMatcherType) { + snap->mType = new AlternativeMatcher(); + } else + if (snap->mKind == PyObjSnapshot::Kind::AlternativeType) { + snap->mType = new Alternative(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ClassType) { + snap->mType = new Class(); + } else + if (snap->mKind == PyObjSnapshot::Kind::HeldClassType) { + snap->mType = new HeldClass(); + } else + if (snap->mKind == PyObjSnapshot::Kind::FunctionType) { + snap->mType = new Function(); + } else { + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); + } + + snap->mType->markActivelyBeingDeserialized( + snap->mNamedInts["type_is_forward"], + snap->mModuleName, + snap->mName + ); + + return; +} + +void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { + if (!snap->mType) { + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ListOfType) { + ((ListOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::TupleOfType) { + ((TupleOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::DictType) { + ((DictType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "key_type"), + getNamedElementType(snap, "value_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ConstDictType) { + ((ConstDictType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "key_type"), + getNamedElementType(snap, "value_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::SetType) { + ((SetType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "key_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::PointerToType) { + ((PointerTo*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "element_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::RefToType) { + ((RefTo*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "element_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ValueType) { + if (snap->mNamedElements.find("value_instance") == snap->mNamedElements.end()) { + throw std::runtime_error("Corrupt PyObjSnapshot::Value"); + } + + // TODO: this needs to be a proper rehydration when we are + // able to fully model 'instance' objects. + // rehydrate(snap->mNamedElements["value_instance"]); + + ((Value*)snap->mType)->initializeDuringDeserialization( + snap->mNamedElements["value_instance"]->mInstance + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::OneOfType) { + std::vector types; + for (auto s: snap->mElements) { + types.push_back(typeFor(s)); + } + + ((OneOfType*)snap->mType)->initializeDuringDeserialization(types); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + PyObject* obType = getNamedElementPyobj(snap, "element_type", false); + + if (!PyType_Check(obType)) { + throw std::runtime_error("Can't rehydrate a PythonObjectOfType without a subclass of type"); + }; + + ((PythonObjectOfType*)snap->mType)->initializeDuringDeserialization((PyTypeObject*)obType); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::TupleType) { + std::vector types; + for (auto s: snap->mElements) { + types.push_back(typeFor(s)); + } + + ((Tuple*)snap->mType)->initializeDuringDeserialization(types); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::NamedTupleType) { + std::vector types; + for (auto s: snap->mElements) { + types.push_back(typeFor(s)); + } + + ((NamedTuple*)snap->mType)->initializeDuringDeserialization(types, snap->mNames); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::BoundMethodType) { + if (snap->mNames.size() != 1) { + throw std::runtime_error("Corrupt PyObjSnapshot::BoundMethod"); + } + + ((BoundMethod*)snap->mType)->initializeDuringDeserialization( + snap->mNames[0], + getNamedElementType(snap, "self_type") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::TypedCellType) { + ((TypedCellType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "element_type") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ForwardType) { + if (snap->mNamedElements.find("fwd_cell_or_dict") != snap->mNamedElements.end()) { + ((Forward*)snap->mType)->setCellOrDict( + getNamedElementPyobj(snap, "fwd_cell_or_dict") + ); + } + + if (snap->mNamedElements.find("fwd_target") != snap->mNamedElements.end()) { + ((Forward*)snap->mType)->define( + getNamedElementType(snap, "fwd_target") + ); + } + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ConcreteAlternativeType) { + if (!getNamedElementType(snap, "alternative")->isAlternative()){ + throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); + } + + ((ConcreteAlternative*)snap->mType)->initializeDuringDeserialization( + snap->mNamedInts["which"], + (Alternative*)getNamedElementType(snap, "alternative") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::AlternativeMatcherType) { + ((AlternativeMatcher*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "alternative") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::AlternativeType) { + std::vector > types; + std::map methods; + std::vector subtypesConcrete; + + for (long k = 0; k < snap->mElements.size() && k < snap->mNames.size(); k++) { + Type* nt = typeFor(snap->mElements[k]); + + if (!nt->isNamedTuple()) { + throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); + } + + types.push_back( + std::make_pair( + snap->mNames[k], + (NamedTuple*)nt + ) + ); + } + + getNamedBundle(snap, "alt_methods", methods); + getNamedBundle(snap, "alt_subtypes", subtypesConcrete); + + ((Alternative*)snap->mType)->initializeDuringDeserialization( + snap->mName, + snap->mModuleName, + types, + methods, + subtypesConcrete + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ClassType) { + Type* heldClass = getNamedElementType(snap, "held_class_type"); + + if (!heldClass->isHeldClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot.Class"); + } + + ((Class*)snap->mType)->initializeDuringDeserialization( + snap->mName, + (HeldClass*)heldClass + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::HeldClassType) { + std::vector bases; + std::vector members; + std::map memberFunctions; + std::map staticFunctions; + std::map propertyFunctions; + std::map classMembers; + std::map classMethods; + + getNamedBundle(snap, "cls_bases", bases); + getNamedBundle(snap, "cls_members", members); + getNamedBundle(snap, "cls_methods", memberFunctions); + getNamedBundle(snap, "cls_staticmethods", staticFunctions); + getNamedBundle(snap, "cls_properties", propertyFunctions); + getNamedBundle(snap, "cls_classmembers", classMembers); + getNamedBundle(snap, "cls_classmethods", classMethods); + + std::vector own_members; + std::map own_memberFunctions; + std::map own_staticFunctions; + std::map own_propertyFunctions; + std::map own_classMembers; + std::map own_classMethods; + + getNamedBundle(snap, "cls_own_members", own_members); + getNamedBundle(snap, "cls_own_methods", own_memberFunctions); + getNamedBundle(snap, "cls_own_staticmethods", own_staticFunctions); + getNamedBundle(snap, "cls_own_properties", own_propertyFunctions); + getNamedBundle(snap, "cls_own_classmembers", own_classMembers); + getNamedBundle(snap, "cls_own_classmethods", own_classMethods); + + Type* clsType = getNamedElementType(snap, "cls_type"); + if (!clsType->isClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot.HeldClass"); + } + + ((HeldClass*)snap->mType)->initializeDuringDeserialization( + snap->mName, + bases, + snap->mNamedInts["cls_is_final"], + members, + memberFunctions, + staticFunctions, + propertyFunctions, + classMembers, + classMethods, + + own_members, + own_memberFunctions, + own_staticFunctions, + own_propertyFunctions, + own_classMembers, + own_classMethods, + + (Class*)clsType + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::FunctionType) { + std::string name = snap->mName; + std::string qualname = snap->mQualname; + std::string moduleName = snap->mModuleName; + std::vector overloads; + Type* closureType = getNamedElementType(snap, "closure_type"); + bool isEntrypoint = snap->mNamedInts["is_nocompile"]; + bool isNocompile = snap->mNamedInts["is_nocompile"]; + + getNamedBundle(snap, "func_overloads", overloads); + + ((Function*)snap->mType)->initializeDuringDeserialization( + name, + qualname, + moduleName, + overloads, + closureType, + isEntrypoint, + isNocompile + ); + return; + } + + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); +} + +void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { + if (mSnapshots.find(obj) == mSnapshots.end()) { + throw std::runtime_error( + "PyObjRehydrator walk didn't pick up snap of type " + + obj->kindAsString() + ); + } + + PyObjSnapshot::Kind kind = obj->mKind; + PyObject*& pyObject(obj->mPyObject); + + // already rehydrated + if (pyObject) { + return; + } + + static PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + if (kind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + // PythonObjectOfType is special because its Type* doesn't have a builtin + // representation of the Type as a PythonTypeObject since we just use the real thing. + // this is similar to how we have a type OneOf(T1, T2,...) but never see an instance + // of it. + pyObject = getNamedElementPyobj(obj, "element_type", false); + } else + if (kind == PyObjSnapshot::Kind::ArbitraryPyObject) { + throw std::runtime_error("Corrupt PyObjSnapshot.ArbitraryPyObject: missing pyObject"); + } else +if (kind == PyObjSnapshot::Kind::PrimitiveType) { + pyObject = (PyObject*)PyInstance::typeObj(obj->mType); + } else +if (kind == PyObjSnapshot::Kind::String) { + pyObject = PyUnicode_FromString(obj->mStringValue.c_str()); + } else +if (kind == PyObjSnapshot::Kind::PrimitiveInstance) { + pyObject = PyInstance::extractPythonObject(obj->mInstance); + } else +if (kind == PyObjSnapshot::Kind::NamedPyObject) { + PyObjectStealer nameAsStr(PyUnicode_FromString(obj->mModuleName.c_str())); + PyObjectStealer moduleObject( + PyObject_GetItem(sysModuleModules, nameAsStr) + ); + + if (!moduleObject) { + PyErr_Clear(); + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + obj->mModuleName + " is not loaded!"); + } + PyObjectStealer inst(PyObject_GetAttrString(moduleObject, obj->mName.c_str())); + if (!inst) { + PyErr_Clear(); + + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + obj->mModuleName + " is missing member " + obj->mName); + } + + pyObject = incref(inst); + } else +if (kind == PyObjSnapshot::Kind::PyDict || kind == PyObjSnapshot::Kind::PyClassDict) { + pyObject = PyDict_New(); + } else +if (kind == PyObjSnapshot::Kind::PyList) { + pyObject = PyList_New(0); + + for (long k = 0; k < obj->mElements.size(); k++) { + PyList_Append(pyObject, pyobjFor(obj->mElements[k])); + } + } else +if (kind == PyObjSnapshot::Kind::PyTuple) { + pyObject = PyTuple_New(obj->mElements.size()); + + // first initialize it in case we throw somehow + for (long k = 0; k < obj->mElements.size(); k++) { + PyTuple_SetItem(pyObject, k, incref(Py_None)); + } + + for (long k = 0; k < obj->mElements.size(); k++) { + PyTuple_SetItem(pyObject, k, incref(pyobjFor(obj->mElements[k]))); + } + } else +if (kind == PyObjSnapshot::Kind::PySet) { + pyObject = PySet_New(nullptr); + + for (long k = 0; k < obj->mElements.size(); k++) { + PySet_Add(pyObject, incref(pyobjFor(obj->mElements[k]))); + } + } else +if (kind == PyObjSnapshot::Kind::PyClass) { + PyObjectStealer argTup(PyTuple_New(3)); + PyTuple_SetItem(argTup, 0, PyUnicode_FromString(obj->mName.c_str())); + PyTuple_SetItem(argTup, 1, incref(getNamedElementPyobj(obj, "cls_bases"))); + PyTuple_SetItem(argTup, 2, incref(getNamedElementPyobj(obj, "cls_dict"))); + + pyObject = PyType_Type.tp_new(&PyType_Type, argTup, nullptr); + + if (!pyObject) { + throw PythonExceptionSet(); + } + } else +if (kind == PyObjSnapshot::Kind::PyModule) { + PyObjectStealer nameAsStr(PyUnicode_FromString(obj->mName.c_str())); + PyObjectStealer moduleObject( + PyObject_GetItem(sysModuleModules, nameAsStr) + ); + + if (!moduleObject) { + PyErr_Clear(); + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + obj->mName + " is not loaded!"); + } + + pyObject = incref(moduleObject); + } else +if (kind == PyObjSnapshot::Kind::PyModuleDict) { + pyObject = PyObject_GenericGetDict(getNamedElementPyobj(obj, "module_dict_of"), nullptr); + if (!pyObject) { + throw PythonExceptionSet(); + } + } else +if (kind == PyObjSnapshot::Kind::PyCell) { + pyObject = PyCell_New(nullptr); + } else +if (kind == PyObjSnapshot::Kind::PyObject) { + PyObject* t = getNamedElementPyobj(obj, "inst_type"); + PyObject* d = getNamedElementPyobj(obj, "inst_dict"); + + static PyObject* emptyTuple = PyTuple_Pack(0); + + pyObject = ((PyTypeObject*)t)->tp_new( + ((PyTypeObject*)t), + emptyTuple, + nullptr + ); + + if (PyObject_GenericSetDict(pyObject, d, nullptr)) { + throw PythonExceptionSet(); + } + } else +if (kind == PyObjSnapshot::Kind::PyFunction) { + pyObject = PyFunction_New( + getNamedElementPyobj(obj, "func_code"), + getNamedElementPyobj(obj, "func_globals") + ); + + if (obj->mNamedElements.find("func_closure") != obj->mNamedElements.end()) { + PyFunction_SetClosure( + pyObject, + getNamedElementPyobj(obj, "func_closure") + ); + } + + if (obj->mNamedElements.find("func_annotations") != obj->mNamedElements.end()) { + PyFunction_SetAnnotations( + pyObject, + getNamedElementPyobj(obj, "func_annotations") + ); + } + + if (obj->mNamedElements.find("func_defaults") != obj->mNamedElements.end()) { + PyFunction_SetAnnotations( + pyObject, + getNamedElementPyobj(obj, "func_defaults") + ); + } + + if (obj->mNamedElements.find("func_qualname") != obj->mNamedElements.end()) { + PyObject_SetAttrString( + pyObject, + "__qualname__", + getNamedElementPyobj(obj, "func_qualname") + ); + } + + if (obj->mNamedElements.find("func_kwdefaults") != obj->mNamedElements.end()) { + PyObject_SetAttrString( + pyObject, + "__kwdefaults__", + getNamedElementPyobj(obj, "func_kwdefaults") + ); + } + + if (obj->mNamedElements.find("func_name") != obj->mNamedElements.end()) { + PyObject_SetAttrString( + pyObject, + "__name__", + getNamedElementPyobj(obj, "func_name") + ); + } + } else +if (kind == PyObjSnapshot::Kind::PyCodeObject) { +#if PY_MINOR_VERSION < 8 + pyObject = (PyObject*)PyCode_New( +#else + pyObject = (PyObject*)PyCode_NewWithPosOnlyArgs( +#endif + obj->mNamedInts["co_argcount"], +#if PY_MINOR_VERSION >= 8 + obj->mNamedInts["co_posonlyargcount"], +#endif + obj->mNamedInts["co_kwonlyargcount"], + obj->mNamedInts["co_nlocals"], + obj->mNamedInts["co_stacksize"], + obj->mNamedInts["co_flags"], + getNamedElementPyobj(obj, "co_code"), + getNamedElementPyobj(obj, "co_consts"), + getNamedElementPyobj(obj, "co_names"), + getNamedElementPyobj(obj, "co_varnames"), + getNamedElementPyobj(obj, "co_freevars"), + getNamedElementPyobj(obj, "co_cellvars"), + getNamedElementPyobj(obj, "co_filename"), + getNamedElementPyobj(obj, "co_name"), + obj->mNamedInts["co_firstlineno"], + getNamedElementPyobj(obj, +#if PY_MINOR_VERSION >= 10 + "co_linetable" +#else + "co_lnotab" +#endif + ) + ); + } else +if (kind == PyObjSnapshot::Kind::PyStaticMethod) { + pyObject = PyStaticMethod_New(Py_None); + + JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)pyObject; + decref(method->cm_callable); + method->cm_callable = incref(getNamedElementPyobj(obj, "meth_func")); + } else +if (kind == PyObjSnapshot::Kind::PyClassMethod) { + pyObject = PyClassMethod_New(Py_None); + + JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)pyObject; + decref(method->cm_callable); + method->cm_callable = incref(getNamedElementPyobj(obj, "meth_func")); + } else +if (kind == PyObjSnapshot::Kind::PyProperty) { + static PyObject* nones = PyTuple_Pack(3, Py_None, Py_None, Py_None); + + pyObject = PyObject_CallObject((PyObject*)&PyProperty_Type, nones); + + JustLikeAPropertyObject* dest = (JustLikeAPropertyObject*)pyObject; + + decref(dest->prop_get); + decref(dest->prop_set); + decref(dest->prop_del); + decref(dest->prop_doc); + + dest->prop_get = nullptr; + dest->prop_set = nullptr; + dest->prop_del = nullptr; + dest->prop_doc = nullptr; + + #if PY_MINOR_VERSION >= 10 + decref(dest->prop_name); + dest->prop_name = nullptr; + #endif + + dest->prop_get = incref(getNamedElementPyobj(obj, "prop_get", true)); + dest->prop_set = incref(getNamedElementPyobj(obj, "prop_set", true)); + dest->prop_del = incref(getNamedElementPyobj(obj, "prop_del", true)); + dest->prop_doc = incref(getNamedElementPyobj(obj, "prop_doc", true)); + + #if PY_MINOR_VERSION >= 10 + decref(dest->prop_name); + dest->prop_name = incref(getNamedElementPyobj(obj, "prop_name", true)); + #endif + } else +if (kind == PyObjSnapshot::Kind::PyBoundMethod) { + pyObject = PyMethod_New(Py_None, Py_None); + PyMethodObject* method = (PyMethodObject*)pyObject; + decref(method->im_func); + decref(method->im_self); + method->im_func = nullptr; + method->im_self = nullptr; + + method->im_func = incref(getNamedElementPyobj(obj, "meth_func")); + method->im_self = incref(getNamedElementPyobj(obj, "meth_self")); + } else { + if (obj->willBeATpType()) { + asm("int3"); + throw std::runtime_error("Expected PyObjSnapshot of kind " + obj->kindAsString() + " to already have a pyobj."); + } + + throw std::runtime_error( + "Can't make a python object representation for a PyObjSnapshot of kind " + + obj->kindAsString() + ); + } +} + +void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { + if (obj->mKind == PyObjSnapshot::Kind::PyClass) { + if (obj->mNamedElements.find("cls_dict") == obj->mNamedElements.end()) { + throw std::runtime_error("Corrupt PyClass - no cls_dict"); + } + + PyObjSnapshot* clsDictPyObj = obj->mNamedElements["cls_dict"]; + + for (long k = 0; k < clsDictPyObj->elements().size() + && k < clsDictPyObj->keys().size(); k++) { + if (clsDictPyObj->keys()[k]->isString()) { + PyObject_SetAttrString( + obj->mPyObject, + clsDictPyObj->keys()[k]->getStringValue().c_str(), + clsDictPyObj->elements()[k]->getPyObj() + ); + } + } + } else + if (obj->mKind == PyObjSnapshot::Kind::PyDict || obj->mKind == PyObjSnapshot::Kind::PyClassDict) { + for (long k = 0; k < obj->mElements.size() + && k < obj->mKeys.size(); k++) { + PyDict_SetItem( + obj->mPyObject, + obj->mKeys[k]->getPyObj(), + obj->mElements[k]->getPyObj() + ); + } + } else + if (obj->mKind == PyObjSnapshot::Kind::PyCell) { + auto it = obj->mNamedElements.find("cell_contents"); + if (it != obj->mNamedElements.end()) { + PyCell_Set( + obj->mPyObject, + it->second->getPyObj() + ); + } + } +} diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp new file mode 100644 index 000000000..69ce0d701 --- /dev/null +++ b/typed_python/PyObjRehydrator.hpp @@ -0,0 +1,186 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include +#include "PyObjSnapshot.hpp" + +class FunctionOverload; +class FunctionGlobal; + +class PyObjRehydrator { +public: + void start(PyObjSnapshot* snapshot); + + Type* typeFor(PyObjSnapshot* snapshot); + + PyObject* pyobjFor(PyObjSnapshot* snapshot); + + void rehydrateAll(); + + PyObject* getNamedElementPyobj( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty=false + ); + + Type* getNamedElementType( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty=false + ); + + Instance getNamedElementInstance( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty=false + ); + + void getFrom( + PyObjSnapshot* snapshot, + Type*& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + PyObject*& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + Function*& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + HeldClass*& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + MemberDefinition& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + FunctionOverload& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + std::string& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + FunctionGlobal& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + FunctionArg& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + ClosureVariableBinding& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + ClosureVariableBindingStep& out + ); + + template + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::map& out + ) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto nameAndType: it->second->mNamedElements) { + T item; + + getFrom(nameAndType.second, item); + + out[nameAndType.first] = item; + } + } + + template + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& out + ) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto snap: it->second->mElements) { + T item; + + getFrom(snap, item); + + out.push_back(item); + } + } + + template + void getNamed( + PyObjSnapshot* snapshot, + std::string name, + T& out + ) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + getFrom(it->second, out); + } + +private: + void rehydrate(PyObjSnapshot* snapshot); + + void buildTypeSkeleton(PyObjSnapshot* snap); + + void rehydrateTpType(PyObjSnapshot* snap); + + void buildPyobjForTpType(PyObjSnapshot* s); + + void finalizeRehydration(PyObjSnapshot* snap); + + std::unordered_set mSnapshots; +}; diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp new file mode 100644 index 000000000..2c60d9637 --- /dev/null +++ b/typed_python/PyObjSnapshot.cpp @@ -0,0 +1,1406 @@ +#include "PyObjSnapshot.hpp" +#include "PyObjSnapshotMaker.hpp" +#include "PyObjGraphSnapshot.hpp" +#include "PyObjRehydrator.hpp" + + + +PyObjSnapshot::PyObjSnapshot(PyObjGraphSnapshot* inGraph) : + mKind(Kind::Uninitialized), + mType(nullptr), + mPyObject(nullptr), + mGraph(inGraph), + mIndexInGraph(0) +{ + if (mGraph) { + mIndexInGraph = mGraph->registerSnapshot(this); + } +} + +void PyObjSnapshot::rehydrate() { + if (!needsHydration()) { + return; + } + + PyObjRehydrator rehydrator; + + rehydrator.start(this); + rehydrator.rehydrateAll(); +} + +std::string PyObjSnapshot::kindAsString() const { + if (mKind == Kind::Uninitialized) { + return "Uninitialized"; + } + if (mKind == Kind::String) { + return "String"; + } + if (mKind == Kind::NamedPyObject) { + return "NamedPyObject"; + } + if (mKind == Kind::PrimitiveInstance) { + return "PrimitiveInstance"; + } + if (mKind == Kind::ValueInstance) { + return "ValueInstance"; + } + if (mKind == Kind::FunctionInstance) { + return "FunctionInstance"; + } + if (mKind == Kind::ClassInstance) { + return "ClassInstance"; + } + if (mKind == Kind::TupleInstance) { + return "TupleInstance"; + } + if (mKind == Kind::NamedTupleInstance) { + return "NamedTupleInstance"; + } + if (mKind == Kind::TupleOfInstance) { + return "TupleOfInstance"; + } + if (mKind == Kind::ListOfInstance) { + return "ListOfInstance"; + } + if (mKind == Kind::SetInstance) { + return "SetInstance"; + } + if (mKind == Kind::DictInstance) { + return "DictInstance"; + } + if (mKind == Kind::ConstDictInstance) { + return "ConstDictInstance"; + } + if (mKind == Kind::PointerToInstance) { + return "PointerToInstance"; + } + if (mKind == Kind::RefToInstance) { + return "RefToInstance"; + } + if (mKind == Kind::AlternativeInstance) { + return "AlternativeInstance"; + } + if (mKind == Kind::ConcreteAlternativeInstance) { + return "ConcreteAlternativeInstance"; + } + if (mKind == Kind::AlternativeMatcherInstance) { + return "AlternativeMatcherInstance"; + } + if (mKind == Kind::BoundMethodInstance) { + return "BoundMethodInstance"; + } + if (mKind == Kind::TypedCellInstance) { + return "TypedCellInstance"; + } + if (mKind == Kind::PrimitiveType) { + return "PrimitiveType"; + } + if (mKind == Kind::ListOfType) { + return "ListOfType"; + } + if (mKind == Kind::TupleOfType) { + return "TupleOfType"; + } + if (mKind == Kind::TupleType) { + return "TupleType"; + } + if (mKind == Kind::NamedTupleType) { + return "NamedTupleType"; + } + if (mKind == Kind::OneOfType) { + return "OneOfType"; + } + if (mKind == Kind::ValueType) { + return "ValueType"; + } + if (mKind == Kind::DictType) { + return "DictType"; + } + if (mKind == Kind::ConstDictType) { + return "ConstDictType"; + } + if (mKind == Kind::SetType) { + return "SetType"; + } + if (mKind == Kind::PointerToType) { + return "PointerToType"; + } + if (mKind == Kind::RefToType) { + return "RefToType"; + } + if (mKind == Kind::AlternativeType) { + return "AlternativeType"; + } + if (mKind == Kind::ConcreteAlternativeType) { + return "ConcreteAlternativeType"; + } + if (mKind == Kind::AlternativeMatcherType) { + return "AlternativeMatcherType"; + } + if (mKind == Kind::PythonObjectOfTypeType) { + return "PythonObjectOfTypeType"; + } + if (mKind == Kind::SubclassOfTypeType) { + return "SubclassOfTypeType"; + } + if (mKind == Kind::ClassType) { + return "ClassType"; + } + if (mKind == Kind::HeldClassType) { + return "HeldClassType"; + } + if (mKind == Kind::FunctionType) { + return "FunctionType"; + } + if (mKind == Kind::FunctionOverload) { + return "FunctionOverload"; + } + if (mKind == Kind::FunctionGlobal) { + return "FunctionGlobal"; + } + if (mKind == Kind::FunctionArg) { + return "FunctionArg"; + } + if (mKind == Kind::FunctionClosureVariableBinding) { + return "FunctionClosureVariableBinding"; + } + if (mKind == Kind::FunctionClosureVariableBindingStep) { + return "FunctionClosureVariableBindingStep"; + } + if (mKind == Kind::ClassMemberDefinition) { + return "ClassMemberDefinition"; + } + if (mKind == Kind::BoundMethodType) { + return "BoundMethodType"; + } + if (mKind == Kind::ForwardType) { + return "ForwardType"; + } + if (mKind == Kind::TypedCellType) { + return "TypedCellType"; + } + if (mKind == Kind::PyList) { + return "PyList"; + } + if (mKind == Kind::PyDict) { + return "PyDict"; + } + if (mKind == Kind::PySet) { + return "PySet"; + } + if (mKind == Kind::PyTuple) { + return "PyTuple"; + } + if (mKind == Kind::PyClass) { + return "PyClass"; + } + if (mKind == Kind::PyObject) { + return "PyObject"; + } + if (mKind == Kind::PyFunction) { + return "PyFunction"; + } + if (mKind == Kind::PyCodeObject) { + return "PyCodeObject"; + } + if (mKind == Kind::PyCell) { + return "PyCell"; + } + if (mKind == Kind::PyModule) { + return "PyModule"; + } + if (mKind == Kind::PyModuleDict) { + return "PyModuleDict"; + } + if (mKind == Kind::PyClassDict) { + return "PyClassDict"; + } + if (mKind == Kind::PyStaticMethod) { + return "PyStaticMethod"; + } + if (mKind == Kind::PyClassMethod) { + return "PyClassMethod"; + } + if (mKind == Kind::PyBoundMethod) { + return "PyBoundMethod"; + } + if (mKind == Kind::PyProperty) { + return "PyProperty"; + } + if (mKind == Kind::ArbitraryPyObject) { + return "ArbitraryPyObject"; + } + if (mKind == Kind::InternalBundle) { + return "InternalBundle"; + } + + throw std::runtime_error("Unknown PyObjSnapshot Kind: " + format((int)mKind)); +} + + +void PyObjSnapshot::cloneFromSnapByHash(PyObjSnapshot* snap) { + mKind = snap->mKind; + mType = snap->mType; + mInstance = snap->mInstance; + mPyObject = incref(snap->mPyObject); + mNamedInts = snap->mNamedInts; + mNames = snap->mNames; + mStringValue = snap->mStringValue; + mModuleName = snap->mModuleName; + mName = snap->mName; + mQualname = snap->mQualname; + + auto intern = [&](PyObjSnapshot* s) { + PyObjSnapshot* res = mGraph->snapshotForHash( + s->mGraph->hashFor(s) + ); + + if (!res) { + throw std::runtime_error("Somehow, when interning, a hash is missing"); + } + + return res; + }; + + for (auto e: snap->mElements) { + mElements.push_back(intern(e)); + } + + for (auto e: snap->mKeys) { + mKeys.push_back(intern(e)); + } + + for (auto nameAndE: snap->mNamedElements) { + mNamedElements[nameAndE.first] = intern(nameAndE.second); + } +} + +void PyObjSnapshot::becomeInternalizedOf( + Type* t, + PyObjSnapshotMaker& maker +) { + // we're always the internalized version of this object + if (maker.linkBackToOriginalObject()) { + mType = t; + } + + mNamedInts["type_is_forward"] = t->isForwardDefined() ? 1 : 0; + mName = t->name(); + mModuleName = t->moduleName(); + + if (t->isListOf()) { + mKind = Kind::ListOfType; + mNamedElements["element_type"] = maker.internalize(((ListOfType*)t)->getEltType()); + return; + } + + if (t->isTupleOf()) { + mKind = Kind::TupleOfType; + mNamedElements["element_type"] = maker.internalize(((TupleOfType*)t)->getEltType()); + return; + } + + if (t->isTuple()) { + mKind = Kind::TupleType; + for (auto eltType: ((Tuple*)t)->getTypes()) { + mElements.push_back(maker.internalize(eltType)); + } + return; + } + + if (t->isNamedTuple()) { + mKind = Kind::NamedTupleType; + for (auto eltType: ((NamedTuple*)t)->getTypes()) { + mElements.push_back(maker.internalize(eltType)); + } + for (auto name: ((NamedTuple*)t)->getNames()) { + mNames.push_back(name); + } + return; + } + + if (t->isOneOf()) { + mKind = Kind::OneOfType; + for (auto eltType: ((OneOfType*)t)->getTypes()) { + mElements.push_back(maker.internalize(eltType)); + } + return; + } + + if (t->isValue()) { + mKind = Kind::ValueType; + mNamedElements["value_instance"] = maker.internalize( + ((Value*)t)->value() + ); + return; + } + + if (t->isDict()) { + mKind = Kind::DictType; + mNamedElements["key_type"] = maker.internalize( + ((DictType*)t)->keyType() + ); + mNamedElements["value_type"] = maker.internalize( + ((DictType*)t)->valueType() + ); + return; + } + + if (t->isConstDict()) { + mKind = Kind::ConstDictType; + mNamedElements["key_type"] = maker.internalize( + ((ConstDictType*)t)->keyType() + ); + mNamedElements["value_type"] = maker.internalize( + ((ConstDictType*)t)->valueType() + ); + return; + } + + if (t->isSet()) { + mKind = Kind::SetType; + mNamedElements["key_type"] = maker.internalize( + ((SetType*)t)->keyType() + ); + return; + } + + if (t->isPointerTo()) { + mKind = Kind::PointerToType; + mNamedElements["element_type"] = maker.internalize( + ((PointerTo*)t)->getEltType() + ); + return; + } + + if (t->isRefTo()) { + mKind = Kind::RefToType; + mNamedElements["element_type"] = maker.internalize( + ((RefTo*)t)->getEltType() + ); + return; + } + + if (t->isPythonObjectOfType()) { + mKind = Kind::PythonObjectOfTypeType; + mNamedElements["element_type"] = maker.internalize( + (PyObject*)((PythonObjectOfType*)t)->pyType() + ); + return; + } + + if (t->isSubclassOf()) { + mKind = Kind::SubclassOfTypeType; + mNamedElements["element_type"] = maker.internalize( + ((SubclassOfType*)t)->getSubclassOf() + ); + return; + } + + if (t->isTypedCell()) { + mKind = Kind::TypedCellType; + mNamedElements["element_type"] = maker.internalize( + ((TypedCellType*)t)->getHeldType() + ); + return; + } + + if (t->isBoundMethod()) { + mKind = Kind::BoundMethodType; + mNamedElements["self_type"] = maker.internalize( + ((BoundMethod*)t)->getFirstArgType() + ); + mNames.push_back(((BoundMethod*)t)->getFuncName()); + return; + } + + if (t->isAlternativeMatcher()) { + mKind = Kind::AlternativeMatcherType; + mNamedElements["alternative"] = maker.internalize( + ((AlternativeMatcher*)t)->getAlternative() + ); + return; + } + + if (t->isConcreteAlternative()) { + mKind = Kind::ConcreteAlternativeType; + mNamedElements["alternative"] = maker.internalize( + ((ConcreteAlternative*)t)->getAlternative() + ); + mNamedInts["which"] = ((ConcreteAlternative*)t)->which(); + return; + } + + if (t->isAlternative()) { + mKind = Kind::AlternativeType; + Alternative* alt = (Alternative*)t; + + mName = alt->name(); + mModuleName = alt->moduleName(); + + for (long k = 0; k < alt->subtypes().size(); k++) { + mElements.push_back(maker.internalize(alt->subtypes()[k].second)); + mNames.push_back(alt->subtypes()[k].first); + } + + mNamedElements["alt_methods"] = maker.internalize(alt->getMethods()); + mNamedElements["alt_subtypes"] = maker.internalize(alt->getSubtypesConcrete()); + return; + } + + if (t->isClass()) { + mKind = Kind::ClassType; + mName = ((Class*)t)->name(); + mModuleName = ((Class*)t)->moduleName(); + mNamedElements["held_class_type"] = maker.internalize(((Class*)t)->getHeldClass()); + return; + } + + if (t->isHeldClass()) { + mKind = Kind::HeldClassType; + HeldClass* cls = (HeldClass*)t; + + mName = cls->name(); + mModuleName = cls->moduleName(); + + mNamedElements["cls_type"] = maker.internalize(cls->getClassType()); + mNamedElements["cls_bases"] = maker.internalize(cls->getBases()); + + mNamedElements["cls_own_methods"] = maker.internalize(cls->getOwnMemberFunctions()); + mNamedElements["cls_own_staticmethods"] = maker.internalize(cls->getOwnStaticFunctions()); + mNamedElements["cls_own_classmethods"] = maker.internalize(cls->getOwnClassMethods()); + mNamedElements["cls_own_properties"] = maker.internalize(cls->getOwnPropertyFunctions()); + mNamedElements["cls_own_classmembers"] = maker.internalize(cls->getOwnClassMembers()); + mNamedElements["cls_own_members"] = maker.internalize(cls->getOwnMembers()); + + mNamedElements["cls_methods"] = maker.internalize(cls->getMemberFunctions()); + mNamedElements["cls_staticmethods"] = maker.internalize(cls->getStaticFunctions()); + mNamedElements["cls_classmethods"] = maker.internalize(cls->getClassMethods()); + mNamedElements["cls_properties"] = maker.internalize(cls->getPropertyFunctions()); + mNamedElements["cls_classmembers"] = maker.internalize(cls->getClassMembers()); + mNamedElements["cls_members"] = maker.internalize(cls->getMembers()); + + mNamedInts["cls_is_final"] = cls->isFinal(); + return; + } + + if (t->isFunction()) { + mKind = Kind::FunctionType; + Function* f = (Function*)t; + + mNamedElements["closure_type"] = maker.internalize(f->getClosureType()); + mNamedInts["is_entrypoint"] = f->isEntrypoint() ? 1 : 0; + mNamedInts["is_nocompile"] = f->isNocompile() ? 1 : 0; + + mName = f->name(); + mQualname = f->qualname(); + mModuleName = f->moduleName(); + + mNamedElements["func_overloads"] = maker.internalize(f->getOverloads()); + return; + } + + if (t->isForward()) { + mKind = Kind::ForwardType; + Forward* f = (Forward*)t; + + if (f->getCellOrDict()) { + mNamedElements["fwd_cell_or_dict"] = maker.internalize(f->getCellOrDict()); + + // if we are a 'lambda' forward, we need to know what we resolve to, assuming + // we do resolve. + if (PyCell_Check(f->getCellOrDict()) && PyCell_Get(f->getCellOrDict())) { + mNamedElements["fwd_cell_resolves_to"] = maker.internalize(PyCell_Get(f->getCellOrDict())); + } else + if (PyDict_Check(f->getCellOrDict())) { + PyObject* o = PyDict_GetItemString(f->getCellOrDict(), f->name().c_str()); + if (o) { + mNamedElements["fwd_dict_resolves_to"] = maker.internalize(o); + } + } + } + + if (f->getTarget()) { + mNamedElements["fwd_target"] = maker.internalize(f->getTarget()); + } + + mName = f->name(); + return; + } + + if (t->isRegister() + || t->isString() + || t->isBytes() + || t->isPyCell() + || t->isEmbeddedMessage() + || t->isNone() + ) { + mKind = Kind::PrimitiveType; + mType = t; + return; + } + + throw std::runtime_error("Type of category " + t->getTypeCategoryString() + " can't be snapshotted"); +} + +void PyObjSnapshot::becomeInternalizedOf( + PyObject* val, + PyObjSnapshotMaker& maker +) { + if (PyInstance::extractTypeFrom(val, true)) { + throw std::runtime_error("types should have been passed to the other pathway"); + } + + // we're always the internalized version of this object + if (maker.linkBackToOriginalObject()) { + mPyObject = incref(val); + } + + static PyObject* lockType = staticPythonInstance("typed_python.internals", "lockType"); + + if (val == lockType) { + mKind = Kind::NamedPyObject; + mName = "lockType"; + mModuleName = "typed_python.internals"; + return; + } + + static PyObject* environType = staticPythonInstance("os", "_Environ"); + + if (val->ob_type == (PyTypeObject*)environType) { + mKind = Kind::NamedPyObject; + mName = "_Environ"; + mModuleName = "os"; + return; + } + + static PyObject* pyObj_object = staticPythonInstance("builtins", "object"); + + if (val == pyObj_object) { + mKind = Kind::NamedPyObject; + mName = "object"; + mModuleName = "builtins"; + return; + } + + static PyObject* pyObj_type = staticPythonInstance("builtins", "type"); + + if (val == pyObj_type) { + mKind = Kind::NamedPyObject; + mName = "type"; + mModuleName = "builtins"; + return; + } + + if (PyUnicode_Check(val)) { + mKind = Kind::String; + mStringValue = PyUnicode_AsUTF8(val); + return; + } + + if (PyTuple_Check(val)) { + mKind = Kind::PyTuple; + for (long i = 0; i < PyTuple_Size(val); i++) { + mElements.push_back(maker.internalize(PyTuple_GetItem(val, i))); + } + return; + } + + if (PyList_Check(val)) { + mKind = Kind::PyList; + for (long i = 0; i < PyList_Size(val); i++) { + mElements.push_back(maker.internalize(PyList_GetItem(val, i))); + } + return; + } + + if (PySet_Check(val)) { + mKind = Kind::PySet; + iterate(val, [&](PyObject* o) { + mElements.push_back(maker.internalize(o)); + }); + return; + } + + if (PyDict_Check(val)) { + // see if this is a moduledict + PyObject* mname = PyDict_GetItemString(val, "__name__"); + if (mname && PyUnicode_Check(mname)) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + PyObjectStealer moduleObj(PyObject_GetItem(sysModuleModules, mname)); + + if (moduleObj) { + PyObjectStealer moduleObjDict(PyObject_GenericGetDict(moduleObj, nullptr)); + if (!moduleObjDict) { + PyErr_Clear(); + } else + if (moduleObjDict == val) { + mKind = Kind::PyModuleDict; + mName = PyUnicode_AsUTF8(mname); + mNamedElements["module_dict_of"] = maker.internalize(moduleObj); + return; + } + } else { + PyErr_Clear(); + } + } + + // see if this is a vanilla class dict + PyObject* dictAccessor = PyDict_GetItemString(val, "__dict__"); + if (dictAccessor && dictAccessor->ob_type == &PyGetSetDescr_Type) { + PyTypeObject* clsType = PyDescr_TYPE(dictAccessor); + if (clsType) { + if (clsType->tp_dict == val) { + if (isVanillaClassType(clsType)) { + mKind = Kind::PyClassDict; + + mNamedElements["class_dict_of"] = maker.internalize( + (PyObject*)clsType + ); + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (val && PyDict_Next(val, &pos, &key, &value)) { + if (PyUnicode_Check(key) + && PyUnicode_AsUTF8(key) != std::string("__dict__") + && PyUnicode_AsUTF8(key) != std::string("__weakref__") + ) { + mElements.push_back(maker.internalize(value)); + mKeys.push_back(maker.internalize(key)); + } + } + + return; + } + } + } + } + + // this is a vanilla dict + mKind = Kind::PyDict; + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (val && PyDict_Next(val, &pos, &key, &value)) { + mElements.push_back(maker.internalize(value)); + mKeys.push_back(maker.internalize(key)); + } + return; + } + + if (PyCell_Check(val)) { + mKind = Kind::PyCell; + + if (PyCell_Get(val)) { + mNamedElements["cell_contents"] = maker.internalize(PyCell_Get(val)); + } + + return; + } + + if (PyType_Check(val)) { + PyTypeObject* tp = (PyTypeObject*)val; + + if (isVanillaClassType(tp)) { + mKind = Kind::PyClass; + + mName = tp->tp_name; + mModuleName = dictGetStringOrEmpty(tp->tp_dict, "__module__"); + + if (tp->tp_dict) { + mNamedElements["cls_dict"] = maker.internalize(tp->tp_dict); + } + if (tp->tp_bases) { + mNamedElements["cls_bases"] = maker.internalize(tp->tp_bases); + } + + return; + } + } + + if (PyFunction_Check(val)) { + mKind = Kind::PyFunction; + + PyFunctionObject* f = (PyFunctionObject*)val; + + mName = stringOrEmpty(f->func_name); + mModuleName = stringOrEmpty(f->func_module); + + if (f->func_name) { + mNamedElements["func_name"] = maker.internalize(f->func_name); + } + if (f->func_module) { + mNamedElements["func_module"] = maker.internalize(f->func_module); + } + if (f->func_qualname) { + mNamedElements["func_qualname"] = maker.internalize(f->func_qualname); + } + if (PyFunction_GetClosure(val)) { + mNamedElements["func_closure"] = maker.internalize(PyFunction_GetClosure(val)); + } + if (PyFunction_GetCode(val)) { + mNamedElements["func_code"] = maker.internalize(PyFunction_GetCode(val)); + } + if (PyFunction_GetModule(val)) { + mNamedElements["func_module"] = maker.internalize(PyFunction_GetModule(val)); + } + if (PyFunction_GetAnnotations(val)) { + mNamedElements["func_annotations"] = maker.internalize(PyFunction_GetAnnotations(val)); + } + if (PyFunction_GetDefaults(val)) { + mNamedElements["func_defaults"] = maker.internalize(PyFunction_GetDefaults(val)); + } + if (PyFunction_GetKwDefaults(val)) { + mNamedElements["func_kwdefaults"] = maker.internalize(PyFunction_GetKwDefaults(val)); + } + if (PyFunction_GetGlobals(val)) { + mNamedElements["func_globals"] = maker.internalize(PyFunction_GetGlobals(val)); + } + return; + } + + if (PyCode_Check(val)) { + mKind = Kind::PyCodeObject; + + PyCodeObject* co = (PyCodeObject*)val; + + mNamedInts["co_argcount"] = co->co_argcount; + mNamedInts["co_kwonlyargcount"] = co->co_kwonlyargcount; + mNamedInts["co_nlocals"] = co->co_nlocals; + mNamedInts["co_stacksize"] = co->co_stacksize; + mNamedInts["co_firstlineno"] = co->co_firstlineno; + mNamedInts["co_posonlyargcount"] = co->co_posonlyargcount; + mNamedInts["co_flags"] = co->co_flags; + + mNamedElements["co_code"] = maker.internalize(co->co_code); + mNamedElements["co_consts"] = maker.internalize(co->co_consts); + mNamedElements["co_names"] = maker.internalize(co->co_names); + mNamedElements["co_varnames"] = maker.internalize(co->co_varnames); + mNamedElements["co_freevars"] = maker.internalize(co->co_freevars); + mNamedElements["co_cellvars"] = maker.internalize(co->co_cellvars); + mNamedElements["co_name"] = maker.internalize(co->co_name); + mNamedElements["co_filename"] = maker.internalize(co->co_filename); + +# if PY_MINOR_VERSION >= 10 + mNamedElements["co_linetable"] = maker.internalize(co->co_linetable); +# else + mNamedElements["co_lnotab"] = maker.internalize(co->co_lnotab); +# endif + + return; + } + + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); + if (instanceType) { + throw std::runtime_error( + "PyObjSnapshotMaker::internalize should already have cast this to an instance call" + ); + return; + } + + if (val == Py_None) { + mKind = Kind::PrimitiveInstance; + return; + } + + if (PyBool_Check(val)) { + mKind = Kind::PrimitiveInstance; + mInstance = Instance::create(val == Py_True); + return; + } + + if (PyLong_Check(val)) { + mKind = Kind::PrimitiveInstance; + + try { + mInstance = Instance::create((int64_t)PyLong_AsLongLong(val)); + } + catch(...) { + mInstance = Instance::create((uint64_t)PyLong_AsUnsignedLongLong(val)); + } + + return; + } + + if (PyFloat_Check(val)) { + mKind = Kind::PrimitiveInstance; + mInstance = Instance::create(PyFloat_AsDouble(val)); + return; + } + + if (PyModule_Check(val)) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + PyObjectStealer name(PyObject_GetAttrString(val, "__name__")); + if (name) { + if (PyUnicode_Check(name)) { + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, name)); + if (moduleObject) { + if (moduleObject == val) { + mKind = Kind::PyModule; + mName = PyUnicode_AsUTF8(name); + return; + } + } else { + PyErr_Clear(); + } + } + } else { + PyErr_Clear(); + } + } + + if (PyBytes_Check(val)) { + mKind = Kind::PrimitiveInstance; + mInstance = Instance::createAndInitialize( + BytesType::Make(), + [&](instance_ptr i) { + BytesType::Make()->constructor( + i, + PyBytes_GET_SIZE(val), + PyBytes_AsString(val) + ); + } + ); + return; + } + + if (isVanillaClassType(val->ob_type)) { + mKind = Kind::PyObject; + + mNamedElements["inst_type"] = maker.internalize((PyObject*)val->ob_type); + + PyObjectStealer dict(PyObject_GenericGetDict(val, nullptr)); + if (dict) { + mNamedElements["inst_dict"] = maker.internalize(dict); + } + return; + } + + if (val->ob_type == &PyStaticMethod_Type || val->ob_type == &PyClassMethod_Type) { + if (val->ob_type == &PyStaticMethod_Type) { + mKind = Kind::PyStaticMethod; + } else { + mKind = Kind::PyClassMethod; + } + + PyObjectStealer funcObj(PyObject_GetAttrString(val, "__func__")); + + mNamedElements["meth_func"] = maker.internalize(funcObj); + + return; + } + + if (val->ob_type == &PyProperty_Type) { + mKind = Kind::PyProperty; + + JustLikeAPropertyObject* prop = (JustLikeAPropertyObject*)val; + + if (prop->prop_get) { + mNamedElements["prop_get"] = maker.internalize(prop->prop_get); + } + if (prop->prop_set) { + mNamedElements["prop_set"] = maker.internalize(prop->prop_set); + } + if (prop->prop_del) { + mNamedElements["prop_del"] = maker.internalize(prop->prop_del); + } + if (prop->prop_doc) { + mNamedElements["prop_doc"] = maker.internalize(prop->prop_doc); + } + + #if PY_MINOR_VERSION >= 10 + if (prop->prop_name) { + mNamedElements["prop_name"] = maker.internalize(prop->prop_name); + } + #endif + + return; + } + + if (val->ob_type == &PyMethod_Type) { + mKind = Kind::PyBoundMethod; + + PyObjectStealer fself(PyObject_GetAttrString(val, "__self__")); + PyObjectStealer ffunc(PyObject_GetAttrString(val, "__func__")); + + mNamedElements["meth_self"] = maker.internalize(fself); + mNamedElements["meth_func"] = maker.internalize(ffunc); + return; + } + + if (isPyObjectGloballyIdentifiable(val)) { + mKind = Kind::NamedPyObject; + + // no checks are necessary because isPyObjectGloballyIdentifiable + // confirms that this is OK. + PyObjectStealer moduleName(PyObject_GetAttrString(val, "__module__")); + PyObjectStealer clsName(PyObject_GetAttrString(val, "__name__")); + + mModuleName = PyUnicode_AsUTF8(moduleName); + mName = PyUnicode_AsUTF8(clsName); + return; + } + + mKind = Kind::ArbitraryPyObject; + mPyObject = incref(val); +} + +void PyObjSnapshot::becomeInternalizedOf( + InstanceRef val, + PyObjSnapshotMaker& maker +) { + if (val.type()->isFunction()) { + mKind = Kind::FunctionInstance; + mNamedElements["type"] = maker.internalize(val.type()); + mNamedElements["closure"] = maker.internalize( + InstanceRef(val.data(), ((Function*)val.type())->getClosureType()) + ); + return; + } + + if (val.type()->isClass()) { + mKind = Kind::ClassInstance; + mNamedElements["type"] = maker.internalize(val.type()); + + // instances are supposed to hold actual classes. If we have a ListOf(Base) + // we expect that the "instance objects" representing a child of 'base' actually + // know that they are 'child' + Class* actual = Class::actualTypeForLayout(val.data()); + if (actual != val.type()) { + throw std::runtime_error( + "PyObjSnapshot expected instance of " + val.type()->name() + " to not be " + + "a subclass" + ); + } + + for (long i = 0; i < actual->getMembers().size(); i++) { + if (actual->checkInitializationFlag(val.data(), i)) { + mNamedElements["member" + format(i)] = maker.internalize( + InstanceRef( + actual->eltPtr(val.data(), i), + actual->getMembers()[i].getType() + ) + ); + } + } + + return; + } + + if (val.type()->isTupleOf() || val.type()->isListOf()) { + if (val.type()->isTupleOf()) { + mKind = Kind::TupleOfInstance; + } else { + mKind = Kind::ListOfInstance; + } + + mNamedElements["type"] = maker.internalize(val.type()); + TupleOrListOfType* nt = (TupleOrListOfType*)val.type(); + + for (long i = 0; i < nt->count(val.data()); i++) { + mElements.push_back( + maker.internalize( + InstanceRef( + nt->eltPtr(val.data(), i), + nt->getEltType() + ) + ) + ); + } + + return; + } + + if (val.type()->isNamedTuple() || val.type()->isTuple()) { + if (val.type()->isNamedTuple()) { + mKind = Kind::NamedTupleInstance; + } else { + mKind = Kind::TupleInstance; + } + + mNamedElements["type"] = maker.internalize(val.type()); + CompositeType* nt = (CompositeType*)val.type(); + + for (long i = 0; i < nt->getTypes().size(); i++) { + mElements.push_back( + maker.internalize( + InstanceRef( + nt->eltPtr(val.data(), i), + nt->getTypes()[i] + ) + ) + ); + } + + return; + } + + if (val.type()->isString()) { + mKind = Kind::String; + mStringValue = ((StringType*)val.type())->toUtf8String(val.data()); + return; + } + + if (val.type()->isNone() || val.type()->isBytes() || val.type()->isRegister()) { + mKind = Kind::PrimitiveInstance; + mInstance = val; + return; + } + + throw std::runtime_error( + "Can't make a PyObjSnapshot out of an instance of type " + + val.type()->name() + ); +} + +void PyObjSnapshot::becomeInternalizedOf( + const MemberDefinition& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::ClassMemberDefinition; + mName = val.getName(); + mNamedElements["type"] = maker.internalize(val.getType()); + mNamedElements["defaultValue"] = maker.internalize(val.getDefaultValue()); + mNamedInts["isNonempty"] = val.getIsNonempty() ? 1 : 0; +} + +void PyObjSnapshot::becomeInternalizedOf( + const FunctionGlobal& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionGlobal; + + if (val.isUnbound()) { + return; + } + + if (val.isNamedModuleMember()) { + mModuleName = val.getModuleName(); + mName = val.getName(); + mNamedElements["moduleDict"] = maker.internalize(val.getModuleDictOrCell()); + + // TODO: ultimately we should get rid of this. It shouldn't be necessary to + // establish a valid 'identity' and its a little arbitrary that we're only recursing + // into types instead of all python objects. For the moment, we need it to keep everything + // working. + PyObject* o = val.extractGlobalRefFromDictOrCell(); + if (o && PyType_Check(o) && PyInstance::extractTypeFrom(o, true)) { + mNamedElements["resolve_to"] = maker.internalize( + PyInstance::extractTypeFrom(o, true) + ); + } + + return; + } + + if (val.isGlobalInCell()) { + mNamedElements["cell"] = maker.internalize(val.getModuleDictOrCell()); + return; + } + + if (val.isGlobalInDict()) { + mNamedElements["moduleDict"] = maker.internalize(val.getModuleDictOrCell()); + mName = val.getName(); + return; + } + + if (val.isConstant()) { + mNamedElements["constant"] = maker.internalize(val.getConstant()); + return; + } +} + +void PyObjSnapshot::becomeInternalizedOf( + const FunctionOverload& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionOverload; + + mNamedElements["func_globals"] = maker.internalize(val.getGlobals()); + mNamedElements["func_code"] = maker.internalize(val.getFunctionCode()); + if (val.getFunctionDefaults()) { + mNamedElements["func_defaults"] = maker.internalize(val.getFunctionDefaults()); + } + if (val.getFunctionAnnotations()) { + mNamedElements["func_annotations"] = maker.internalize(val.getFunctionAnnotations()); + } + mNamedElements["func_args"] = maker.internalize(val.getArgs()); + mNamedElements["func_closure_varnames"] = maker.internalize( + val.getFunctionClosureVarnames() + ); + mNamedElements["func_globals_in_closure_varnames"] = maker.internalize( + val.getFunctionGlobalsInClosureVarnames() + ); + + if (val.getMethodOf()) { + mNamedElements["func_method_of"] = maker.internalize(val.getMethodOf()); + } + + if (val.getSignatureFunction()) { + mNamedElements["func_signature_func"] = maker.internalize(val.getSignatureFunction()); + } + + mNamedElements["func_closure_variable_bindings"] = maker.internalize( + val.getClosureVariableBindings() + ); + + if (val.getReturnType()) { + mNamedElements["func_ret_type"] = maker.internalize(val.getReturnType()); + } +} + + +void PyObjSnapshot::becomeInternalizedOf( + const FunctionArg& arg, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionArg; + + mName = arg.getName(); + if (arg.getTypeFilter()) { + mNamedElements["arg_type_filter"] = maker.internalize(arg.getTypeFilter()); + } + + if (arg.getDefaultValue()) { + mNamedElements["arg_default_value"] = maker.internalize(arg.getDefaultValue()); + } + + mNamedInts["arg_is_star_arg"] = arg.getIsStarArg() ? 1 : 0; + mNamedInts["arg_is_kwarg"] = arg.getIsKwarg() ? 1 : 0; +} + +void PyObjSnapshot::becomeInternalizedOf( + const ClosureVariableBinding& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionClosureVariableBinding; + + for (long k = 0; k < val.size(); k++) { + mElements.push_back(maker.internalize(val[k])); + } +} + +void PyObjSnapshot::becomeInternalizedOf( + const ClosureVariableBindingStep& step, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionClosureVariableBindingStep; + + if (step.isFunction()) { + mNamedElements["step_func"] = maker.internalize(step.getFunction()); + } + if (step.isNamedField()) { + mNames.push_back(step.getNamedField()); + } + if (step.isIndexedField()) { + mNamedInts["step_index"] = step.getIndexedField(); + } +} + +void PyObjSnapshot::becomeInternalizedOf( + const std::string& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::String; + mStringValue = val; +} + +PyObjSnapshot* PyObjSnapshot::computeForwardTargetTransitive() { + PyObjSnapshot* source = this; + int steps = 0; + + while (true) { + steps += 1; + if (steps > mGraph->getObjects().size()) { + throw std::runtime_error("Forward cycle detected"); + } + + if (source->mKind != Kind::ForwardType) { + // TODO - what if we're not a type? + if (!source->willBeATpType()) { + throw std::runtime_error("Should we be converting this to a Value type?"); + } + + return source; + } + + source = source->computeForwardTarget(); + if (!source) { + return nullptr; + } + } +} + + +void PyObjSnapshot::recomputeRecursiveName(const std::unordered_set& group) { + SnapshotStack stack; + + mName = computeRecursiveName(stack, group); +} + +std::string PyObjSnapshot::computeRecursiveName(SnapshotStack& stack, const std::unordered_set& group) { + if (mKind == Kind::PrimitiveType) { + return mType->name(); + } + + // if we're not in the group, then we will have a name computed already + if (group.find(this) == group.end()) { + return mName; + } + + // the names of these classes can't change - they are given to us + if (mKind == Kind::ClassType || mKind == Kind::HeldClassType || mKind == Kind::AlternativeType) { + return mName; + } + + long index = stack.indexOf(this); + if (index != -1) { + return "^" + format(index); + } + + PushSnapshotStack push(stack, this); + + if (mKind == Kind::PythonObjectOfTypeType) { + return "PythonObjectOfType(" + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + ")"; + } + + if (mKind == Kind::AlternativeMatcherType) { + return "AlternativeMatcher(" + getNamedElement("alternative", false)->computeRecursiveName(stack, group) + ")"; + } + + if (mKind == Kind::ConcreteAlternativeType) { + return getNamedElement("alternative", false)->computeRecursiveName(stack, group) + "." + + getNamedElement("alternative", false)->getNameByIx(mNamedInts["which"]); + } + + if (mKind == Kind::BoundMethodType) { + if (mNames.size() != 1) { + throw std::runtime_error("Corrupt PyObjSnapshot.BoundMethodType: no name"); + } + + return "BoundMethod(" + getNamedElement("self_type", false)->computeRecursiveName(stack, group) + ", " + mNames[0] + ")"; + } + + if (mKind == Kind::NamedTupleType) { + std::string name = "NamedTuple("; + + for (long k = 0; k < mElements.size() && k < mNames.size(); k++) { + if (k) { + name += ", "; + } + name += mNames[k] + "=" + mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::TupleType) { + std::string name = "Tuple("; + + for (long k = 0; k < mElements.size() && k < mNames.size(); k++) { + if (k) { + name += ", "; + } + name += mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::DictType) { + return "Dict(" + + getNamedElement("key_type", false)->computeRecursiveName(stack, group) + + ", " + + getNamedElement("value_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::ConstDictType) { + return "ConstDict(" + + getNamedElement("key_type", false)->computeRecursiveName(stack, group) + + ", " + + getNamedElement("value_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::ListOfType) { + return "ListOf(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::SetType) { + return "Set(" + + getNamedElement("key_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::TypedCellType) { + return "TypedCell(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::TupleOfType) { + return "TupleOf(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::PointerToType) { + return "PointerTo(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::SubclassOfTypeType) { + return "SubclassOf(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::RefToType) { + return "RefTo(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::OneOfType) { + std::string name = "mElements("; + + for (long k = 0; k < mElements.size(); k++) { + if (k) { + name += ", "; + } + name += mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::OneOfType) { + std::string name = "mElements("; + + for (long k = 0; k < mElements.size(); k++) { + if (k) { + name += ", "; + } + name += mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::ForwardType) { + return mName; + } + + return ""; +} diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp new file mode 100644 index 000000000..67d7d719c --- /dev/null +++ b/typed_python/PyObjSnapshot.hpp @@ -0,0 +1,839 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include +#include "Type.hpp" +#include "Instance.hpp" +#include "PythonTypeInternals.hpp" +#include "PyObjSnapshotMaker.hpp" +#include "TypeStack.hpp" + +class PyObjGraphSnapshot; +class PyObjSnapshot; +class FunctionGlobal; +class FunctionOverload; +class PyObjRehydrator; +class PyObjSnapshotMaker; + +typedef PtrStack SnapshotStack; +typedef PushPtrStack PushSnapshotStack; + +/********************************* +PyObjSnapshot + +A representation of a python object that's owned by TypedPython. They are intended to be a +'snapshot' of the state of a collection of python objects and contain enough information +for the compiler to use them to build compiled code, and for us to rebuild the object +entirely from the snapshot. This gives us a scaffolding to describe objects and object +graphs independent of the state of the python interpreter. + +To the extent that they visit 'mutable' objects like a list (which might be contained within +a default argument), they will encode the state of the object when it was first seen. This +lets us determine if the object was modified (which would break compiler invariants) and also +gives us a self-consistent view of the world to compile against so we don't have state +changing underneath us. +**********************************/ + +class PyObjSnapshot { +private: + friend class PyObjSnapshotMaker; + friend class PyObjGraphSnapshot; + friend class PyObjRehydrator; + + PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr); + +public: + enum class Kind { + // this should never be visible in a running program + Uninitialized = 0, + // a string held in mStringObject + String, + // an instance of a FunctionType + FunctionInstance, + ValueInstance, + ClassInstance, + TupleInstance, + NamedTupleInstance, + TupleOfInstance, + ListOfInstance, + SetInstance, + DictInstance, + ConstDictInstance, + PointerToInstance, + RefToInstance, + AlternativeInstance, + ConcreteAlternativeInstance, + AlternativeMatcherInstance, + BoundMethodInstance, + TypedCellInstance, + // we're pointing into a TP leaf instance (a register type, none, or bytes) + PrimitiveInstance, + // we're pointing to a "canonical" python object that's visible from a module + // and that we don't want to look inside of. mName will contain the + // name, and mModuleName will contain the module. We will assume that this object + // is the same across program invocations (its a C function and we can't + // look inside of it) + NamedPyObject, + // we're a primitive type (like int) + PrimitiveType, + // a TP ListOf type. The element type will be in element_type + ListOfType, + // a TP TupleOf type. The element type will be in element_type + TupleOfType, + // a TP Tuple type. The subtypes will be in mElements + TupleType, + // a TP NamedTuple type. The subtypes will be in mElements and the names in mNames + NamedTupleType, + // a TP OneOf type. The subtypes will be in mElements + OneOfType, + // a TP Value type. The instance will be in value_instance + ValueType, + // a TP DictType type. Will have key_type and value_type + DictType, + // a TP ConstDictType type. Will have key_type and value_type + ConstDictType, + // a TP ConstDictType type. Will have key_type and value_type + SetType, + // a TP PointerTo type. The element type will be in element_type + PointerToType, + // a TP RefTo type. The element type will be in element_type + RefToType, + // a TP Alternative type. + AlternativeType, + // a TP ConcreteAlternative type. + ConcreteAlternativeType, + // a TP AlternativeMatcher type. + AlternativeMatcherType, + // a TP PythonObjectOfType type. + PythonObjectOfTypeType, + // a TP SubclassOfType type. + SubclassOfTypeType, + // a TP Class type. + ClassType, + // a TP HeldClass type. + HeldClassType, + // a TP FunctionType type. + FunctionType, + // a TP FunctionOverload. + FunctionOverload, + // a TP FunctionGlobal. + FunctionGlobal, + // a TP FunctionArg. + FunctionArg, + // a TP ClosureVariableBinding. + FunctionClosureVariableBinding, + // a TP ClosureVariableBinding.Step + FunctionClosureVariableBindingStep, + // a TP MemberDefinition in a Class. + ClassMemberDefinition, + // a TP BoundMethod type + BoundMethodType, + // a TP Forward type. Contains fwd_target for the target, if set, + // fwd_cell_or_dict for a forward defined using a lambda, and + // mName if the forward dict was a global. + ForwardType, + // a TP TypedCellType + TypedCellType, + // a python list, with elements in mElements + PyList, + // a python Dict, with values in mElements and keys in mKeys + PyDict, + // a python Dict, with values in mElements and keys in mKeys + PySet, + // a python tuple with elements in mElements + PyTuple, + // a vanilla python 'class' which must be constructible by calling 'type' + PyClass, + // a vanilla python object whose type is a PyClass + PyObject, + // a python function. mNamedElements will contain code, annotations, ec. + // mStringObject will contain the name. + PyFunction, + // a code object. mName + PyCodeObject, + PyCell, + // a module object that can be looked up by name. We don't walk into this + // from here to avoid making a snapshot of module objects while they're being imported + PyModule, + // a module object's dict. + PyModuleDict, + // the dict of a vanilla PyClass + PyClassDict, + PyStaticMethod, + PyClassMethod, + PyBoundMethod, + PyProperty, + // the bailout pathway for cases we don't handle well. We should + // assume that the compiler will treat this object as a plain + // PyObject without looking inside of it, and the details of this + // object are insufficient to differentiate two different Function + // types that both refer to different ArbitraryPyObject instances. + ArbitraryPyObject, + // a bundle of types that doesn't represent a specific object or type + // but holds collections of types. + InternalBundle + }; + + ~PyObjSnapshot() { + decref(mPyObject); + } + + Kind getKind() const { + return mKind; + } + + PyObjGraphSnapshot* getGraph() const { + return mGraph; + } + + bool isType() const { + return mKind == Kind::PrimitiveType; + } + + bool isString() const { + return mKind == Kind::String; + } + + std::string kindAsString() const; + + static std::string dictGetStringOrEmpty(PyObject* dict, const char* name) { + if (!dict || !PyDict_Check(dict)) { + return ""; + } + + PyObject* o = PyDict_GetItemString(dict, name); + if (!o || !PyUnicode_Check(o)) { + return ""; + } + + return PyUnicode_AsUTF8(o); + } + + static std::string stringOrEmpty(PyObject* o) { + if (!o || !PyUnicode_Check(o)) { + return ""; + } + + return PyUnicode_AsUTF8(o); + } + + static PyTypeObject* createAVanillaType() { + PyObjectStealer emptyBases(PyTuple_New(0)); + PyObjectStealer emptyDict(PyDict_New()); + + return (PyTypeObject*)PyObject_CallFunction( + (PyObject*)&PyType_Type, + "sOO", + "AVanillaType", + (PyObject*)emptyBases, + (PyObject*)emptyDict + ); + } + + bool isVanillaClassType(PyTypeObject* o) { + static PyTypeObject* aVanillaType = createAVanillaType(); + return o->tp_new == aVanillaType->tp_new; + } + + static bool isPyObjectGloballyIdentifiable(PyObject* h) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + if (PyObject_HasAttrString(h, "__module__") && PyObject_HasAttrString(h, "__name__")) { + PyObjectStealer moduleName(PyObject_GetAttrString(h, "__module__")); + if (!moduleName) { + PyErr_Clear(); + return false; + } + + PyObjectStealer clsName(PyObject_GetAttrString(h, "__name__")); + if (!clsName) { + PyErr_Clear(); + return false; + } + + if (!PyUnicode_Check(moduleName) || !PyUnicode_Check(clsName)) { + return false; + } + + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, moduleName)); + + if (!moduleObject) { + PyErr_Clear(); + return false; + } + + PyObjectStealer obj(PyObject_GetAttr(moduleObject, clsName)); + + if (!obj) { + PyErr_Clear(); + return false; + } + if ((PyObject*)obj == h) { + return true; + } + } + + return false; + } + + void becomeInternalizedOf( + const std::string& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + const FunctionArg& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + const ClosureVariableBinding& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + const ClosureVariableBindingStep& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + const MemberDefinition& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + const FunctionGlobal& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + const FunctionOverload& val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + InstanceRef val, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + Type* t, + PyObjSnapshotMaker& maker + ); + + void becomeInternalizedOf( + PyObject* val, + PyObjSnapshotMaker& maker + ); + + void append(PyObjSnapshot* elt) { + if (mKind != Kind::PyTuple) { + throw std::runtime_error("Expected a PyTuple"); + } + + mElements.push_back(elt); + } + + const std::vector& elements() const { + return mElements; + } + + const std::vector& keys() const { + return mKeys; + } + + const std::map& namedElements() const { + return mNamedElements; + } + + bool hasNamedElement(std::string s) const { + return mNamedElements.find(s) != mNamedElements.end(); + } + + PyObjSnapshot* getNamedElement(std::string s, bool allowEmpty = true) const { + auto it = mNamedElements.find(s); + + if (it != mNamedElements.end()) { + return it->second; + } + + if (!allowEmpty) { + throw std::runtime_error("Can't find element named '" + s + "'."); + } + + return nullptr; + } + + const std::vector& names() const { + return mNames; + } + + const std::map& namedInts() const { + return mNamedInts; + } + + const std::string& getStringValue() const { + return mStringValue; + } + + const std::string& getName() const { + return mName; + } + + const std::string& getModuleName() const { + return mModuleName; + } + + const std::string& getQualname() const { + return mQualname; + } + + bool willBeATpType() const { + return mKind == Kind::PrimitiveType + || mKind == Kind::ListOfType + || mKind == Kind::TupleOfType + || mKind == Kind::TupleType + || mKind == Kind::NamedTupleType + || mKind == Kind::OneOfType + || mKind == Kind::ValueType + || mKind == Kind::DictType + || mKind == Kind::ConstDictType + || mKind == Kind::SetType + || mKind == Kind::PointerToType + || mKind == Kind::RefToType + || mKind == Kind::AlternativeType + || mKind == Kind::ConcreteAlternativeType + || mKind == Kind::AlternativeMatcherType + || mKind == Kind::PythonObjectOfTypeType + || mKind == Kind::SubclassOfTypeType + || mKind == Kind::ClassType + || mKind == Kind::HeldClassType + || mKind == Kind::FunctionType + || mKind == Kind::BoundMethodType + || mKind == Kind::ForwardType + || mKind == Kind::TypedCellType + ; + } + + ::Type* getType() { + if (mType) { + return mType; + } + + if (mKind == Kind::PrimitiveType) { + return mType; + } + + if (willBeATpType() && !mType) { + rehydrate(); + return mType; + } + + return nullptr; + } + + const ::Instance& getInstance() { + return mInstance; + } + + bool needsHydration() const { + if (mKind == Kind::PrimitiveInstance) { + return false; + } + + if (mKind == Kind::String) { + return false; + } + + if (mKind == Kind::Uninitialized + || mKind == Kind::InternalBundle + || mKind == Kind::FunctionOverload + || mKind == Kind::FunctionGlobal + || mKind == Kind::FunctionArg + || mKind == Kind::FunctionClosureVariableBinding + || mKind == Kind::FunctionClosureVariableBindingStep + || mKind == Kind::ClassMemberDefinition + ) { + return false; + } + + if (mPyObject) { + return false; + } + + return true; + } + + // are we an object that doesn't actually produce a cacheable value + bool isUncacheable() const { + return mKind == Kind::InternalBundle + || mKind == Kind::FunctionOverload + || mKind == Kind::FunctionGlobal + || mKind == Kind::FunctionArg + || mKind == Kind::FunctionClosureVariableBinding + || mKind == Kind::FunctionClosureVariableBindingStep + || mKind == Kind::ClassMemberDefinition + ; + } + + // get the python object representation of this object, which isn't guaranteed + // to exist and may need to be constructed on demand. this will do a pass over + // all reachable objects, building skeletons, and then performs a second pass where + // we fill items out once we have the skeletons in place. + PyObject* getPyObj() { + if (mPyObject) { + return mPyObject; + } + + if (mType) { + mPyObject = incref((PyObject*)PyInstance::typeObj(mType)); + return mPyObject; + } + + if (mKind == Kind::String) { + mPyObject = PyUnicode_FromString(mStringValue.c_str()); + return mPyObject; + } + + if (mKind == Kind::PrimitiveInstance) { + mPyObject = PyInstance::extractPythonObject(mInstance); + return mPyObject; + } + + rehydrate(); + return mPyObject; + } + + void rehydrate(); + + std::string toString() const { + std::string inner; + if (mType) { + inner = mType->name(); + } else + if (mName.size()) { + inner = mName; + if (mModuleName.size()) { + inner = mModuleName + "." + mName; + } + } else + if (mPyObject) { + inner = std::string("of type ") + mPyObject->ob_type->tp_name; + } + if (inner.size()) { + inner = ", " + inner; + } + + return "PyObjSnapshot." + kindAsString() + "(ix=" + format(mIndexInGraph) + inner + ")"; + } + + size_t getIndexInGraph() const { + return mIndexInGraph; + } + + template + void becomeBundleOf(const std::map& namedElements, PyObjSnapshotMaker& maker) { + mKind = Kind::InternalBundle; + + for (auto& nameAndElt: namedElements) { + mNamedElements[nameAndElt.first] = maker.internalize(nameAndElt.second); + } + } + + template + void becomeBundleOf(const std::vector& elements, PyObjSnapshotMaker& maker) { + mKind = Kind::InternalBundle; + + for (auto& elt: elements) { + mElements.push_back(maker.internalize(elt)); + } + } + + // TODO: fix serialization. This is just not right. + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + uint32_t id; + bool isNew; + std::tie(id, isNew) = buffer.cachePointer((void*)this, nullptr); + + if (!isNew) { + buffer.writeBeginCompound(fieldNumber); + buffer.writeUnsignedVarintObject(0, id); + buffer.writeEndCompound(); + return; + } else { + buffer.writeBeginCompound(fieldNumber); + buffer.writeUnsignedVarintObject(0, id); + buffer.writeUnsignedVarintObject(1, (int)mKind); + + // we only serialize Type, Instance, and PyObj if they are not caches. + if (mKind == Kind::PrimitiveType) { + context.serializeNativeType(mType, buffer, 2); + } else + if (mKind == Kind::PrimitiveInstance) { + buffer.writeBeginCompound(3); + context.serializeNativeType(mInstance.type(), buffer, 0); + mInstance.type()->serialize(mInstance.data(), buffer, 1); + buffer.writeEndCompound(); + } else + if (mKind == Kind::ArbitraryPyObject) { + context.serializePythonObject(mPyObject, buffer, 5); + } else + if (mElements.size()) { + buffer.writeBeginCompound(4); + for (long i = 0; i < mElements.size(); i++) { + mElements[i]->serialize(context, buffer, i); + } + buffer.writeEndCompound(); + } + + buffer.writeEndCompound(); + } + } + + template + static PyObjSnapshot* deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + int64_t kind = 0; + + ::Type* type = nullptr; + + std::vector vec; + ::Instance i; + uint32_t id = -1; + PyObjectHolder pyobj; + PyObjSnapshot* result = nullptr; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + id = buffer.readUnsignedVarint(); + + void* ptr = buffer.lookupCachedPointer(id); + if (ptr) { + result = (PyObjSnapshot*)result; + } else { + result = new PyObjSnapshot(); + buffer.addCachedPointer(id, result, nullptr); + } + } else + if (fieldNumber == 1) { + kind = buffer.readUnsignedVarint(); + } else + if (fieldNumber == 2) { + type = context.deserializeNativeType(buffer, wireType); + } else + if (fieldNumber == 3) { + i = context.deserializeNativeInstance(buffer, wireType); + } else + if (fieldNumber == 4) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + vec.push_back(PyObjSnapshot::deserialize(context, buffer, wireType)); + }); + } else + if (fieldNumber == 5) { + pyobj.steal(context.deserializePythonObject(buffer, wireType)); + } + }); + + if (!result) { + throw std::runtime_error("corrupt PyObjSnapshot - no memo found"); + } + + if (kind == -1) { + throw std::runtime_error("corrupt PyObjSnapshot - invalid kind"); + } + + result->mKind = Kind(kind); + result->mType = type; + result->mInstance = i; + result->mPyObject = incref(pyobj); + result->mElements = vec; + + result->validateAfterDeserialization(); + + return result; + } + + void clearCache() { + if (mKind == Kind::ArbitraryPyObject + || mKind == Kind::PrimitiveInstance + || mKind == Kind::PrimitiveType + ) { + throw std::runtime_error("Can't clear the cache of a leaf node."); + } + + mType = nullptr; + mInstance = Instance(); + if (mPyObject) { + decref(mPyObject); + mPyObject = nullptr; + } + } + + template + void visitOutbound(const visitor_type& v) { + for (auto& e: mElements) { + v(e); + } + + for (auto& e: mKeys) { + v(e); + } + + for (auto& nameAndE: mNamedElements) { + v(nameAndE.second); + } + } + + void pointForwardToFinalType() { + if (mKind != Kind::ForwardType) { + throw std::runtime_error("Makes no sense to call this on a non-forward"); + } + + PyObjSnapshot* target = computeForwardTargetTransitive(); + + if (!target) { + throw std::runtime_error("Forward doesn't resolve to a valid target"); + } + + mNamedElements["fwd_target"] = target; + } + + // replace any outbound links that are pointing to forwards with the forward target + // returns true if we modified the object + bool replaceOutboundForwardsWithTargets() { + bool updated = false; + + visitOutbound([&](PyObjSnapshot*& snap) { + if (snap->mKind == Kind::ForwardType) { + PyObjSnapshot* tgt = snap->getNamedElement("fwd_target"); + if (!tgt) { + throw std::runtime_error("Somehow a forward doesn't have a target"); + } + snap = tgt; + updated = true; + } + }); + + return updated; + } + + std::string getNameByIx(long ix) { + if (ix < 0 || ix >= mNames.size()) { + throw std::runtime_error("NameIndex out of bounds."); + } + + return mNames[ix]; + } + + PyObjSnapshot* computeForwardTarget() { + if (mKind != Kind::ForwardType) { + return nullptr; + } + PyObjSnapshot* snap = getNamedElement("fwd_target"); + if (snap) { + return snap; + } + + snap = getNamedElement("fwd_cell_resolves_to"); + if (snap) { + return snap; + } + + snap = getNamedElement("fwd_dict_resolves_to"); + if (snap) { + return snap; + } + return nullptr; + } + + PyObjSnapshot* computeForwardTargetTransitive(); + + std::string computeRecursiveName(SnapshotStack& stack, const std::unordered_set& group); + + void recomputeRecursiveName(const std::unordered_set& group); + +private: + void markInternalizedOnType() { + if (!mType) { + return; + } + + mType->setSnapshot(this); + } + + bool markTypeNotFwdDefined() { + if (mNamedInts["type_is_forward"]) { + mNamedInts["type_is_forward"] = 0; + return true; + } + + return false; + } + + void cloneFromSnapByHash(PyObjSnapshot* snap); + + // ensure we won't crash if we interact with this object. + void validateAfterDeserialization() { + if (mKind == Kind::PrimitiveType) { + if (!mType) { + throw std::runtime_error("Corrupt PyObjSnapshot: no Type"); + } + } + + if (mKind == Kind::ArbitraryPyObject) { + if (!mPyObject) { + throw std::runtime_error("Corrupt PyObjSnapshot: no PyObject"); + } + } + } + + Kind mKind; + PyObjGraphSnapshot* mGraph; + size_t mIndexInGraph; + + // if we are a PrimitiveType, then our type. If we resolve to a Type, a representation + // of us as that type. Otherwise nullptr. + ::Type* mType; + + // if we are a PrimitiveInstance, the instance value. Otherwise a cache of our value + // as an instance. + ::Instance mInstance; + + // if we are an ArbitraryPythonObject, then our value. Otherwise, a cache of our value + // as a PyObject. + PyObject* mPyObject; + + // these are the actual data that represent us + std::map mNamedElements; + + std::map mNamedInts; + + std::vector mElements; + + std::vector mNames; + + std::vector mKeys; + + std::string mStringValue; + + std::string mModuleName; + + std::string mName; + + std::string mQualname; +}; diff --git a/typed_python/PyObjSnapshotGroupSearch.hpp b/typed_python/PyObjSnapshotGroupSearch.hpp new file mode 100644 index 000000000..fcbeddbff --- /dev/null +++ b/typed_python/PyObjSnapshotGroupSearch.hpp @@ -0,0 +1,147 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyObjSnapshot.hpp" + +// find strongly-connected groups of python objects and Type objects (as far as the +// compiler is concerned) +class PyObjSnapshotGroupSearch { +public: + PyObjSnapshotGroupSearch(PyObjGraphSnapshot* graph) : mGraph(graph) + { + } + + void add(PyObjSnapshot* o, bool insistNew=false) { + if (mSnapToOutGroupIx.find(o) != mSnapToOutGroupIx.end()) { + if (insistNew) { + throw std::runtime_error("We've already seen this element"); + } else { + return; + } + } + + pushGroup(o); + findAllGroups(); + } + + const std::vector > >& getGroups() const { + return mOutGroups; + } + +private: + void pushGroup(PyObjSnapshot* o) { + if (mInAGroup.find(o) != mInAGroup.end()) { + return; + } + + if (mSnapToOutGroupIx.find(o) != mSnapToOutGroupIx.end()) { + return; + } + + mInAGroup.insert(o); + + mGroups.push_back( + std::shared_ptr >( + new std::unordered_set() + ) + ); + + mGroupOutboundEdges.push_back( + std::shared_ptr >( + new std::vector() + ) + ); + + mGroups.back()->insert(o); + + o->visitOutbound([&](PyObjSnapshot* snap) { + if (snap->getGraph() == mGraph) { + mGroupOutboundEdges.back()->push_back(snap); + } + }); + } + + void findAllGroups() { + while (mGroups.size()) { + doOneStep(); + } + } + + void doOneStep() { + if (mGroupOutboundEdges.back()->size() == 0) { + // this group is finished.... + mOutGroups.push_back(mGroups.back()); + + for (auto gElt: *mGroups.back()) { + mSnapToOutGroupIx[gElt] = mOutGroups.size() - 1; + mInAGroup.erase(gElt); + } + + mGroups.pop_back(); + mGroupOutboundEdges.pop_back(); + } else { + // pop an outbound edge and see where does it go? + PyObjSnapshot* o = mGroupOutboundEdges.back()->back(); + + mGroupOutboundEdges.back()->pop_back(); + + if (mInAGroup.find(o) == mInAGroup.end()) { + // this is new + pushGroup(o); + } else { + // this is above us somewhere + while (mGroups.size() && mGroups.back()->find(o) == mGroups.back()->end()) { + // merge these two mGroups together + mGroups[mGroups.size() - 2]->insert( + mGroups.back()->begin(), mGroups.back()->end() + ); + mGroups.pop_back(); + + mGroupOutboundEdges[mGroupOutboundEdges.size() - 2]->insert( + mGroupOutboundEdges[mGroupOutboundEdges.size() - 2]->end(), + mGroupOutboundEdges.back()->begin(), + mGroupOutboundEdges.back()->end() + ); + mGroupOutboundEdges.pop_back(); + } + + if (!mGroups.size()) { + // this shouldn't ever happen, but we want to know if it does + throw std::runtime_error("Never found parent of object!"); + } + } + } + } + +private: + PyObjGraphSnapshot* mGraph; + + // everything in a group somewhere + std::unordered_set mInAGroup; + + // each group + std::vector > > mGroups; + + // for each group, the set of things it reaches out to + std::vector > > mGroupOutboundEdges; + + // the final groups we ended up with + std::vector > > mOutGroups; + + std::unordered_map mSnapToOutGroupIx; +}; diff --git a/typed_python/PyObjSnapshotMaker.cpp b/typed_python/PyObjSnapshotMaker.cpp new file mode 100644 index 000000000..b66f71cc6 --- /dev/null +++ b/typed_python/PyObjSnapshotMaker.cpp @@ -0,0 +1,220 @@ +#include "PyObjSnapshot.hpp" +#include "PyObjSnapshotMaker.hpp" + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionArg& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBinding& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBindingStep& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const MemberDefinition& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionGlobal& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(PyObject* val) +{ + Type* t = PyInstance::extractTypeFrom(val, true); + if (t) { + return internalize(t); + } + + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); + if (instanceType) { + return internalize( + InstanceRef( + ((PyInstance*)val)->dataPtr(), + instanceType + ) + ); + } + + auto it = mObjMapCache.find(val); + + if (it != mObjMapCache.end()) { + return it->second; + } + + mObjMapCache[val] = new PyObjSnapshot(mGraph); + + mObjMapCache[val]->becomeInternalizedOf(val, *this); + + return mObjMapCache[val]; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(Type* val) +{ + if (mLinkToInternal && val->getSnapshot()) { + return val->getSnapshot(); + } + + auto it = mTypeMapCache.find(val); + + if (it != mTypeMapCache.end()) { + return it->second; + } + + mTypeMapCache[val] = new PyObjSnapshot(mGraph); + + mTypeMapCache[val]->becomeInternalizedOf(val, *this); + + return mTypeMapCache[val]; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(InstanceRef val) +{ + Type* t = val.extractType(true); + if (t) { + return internalize(t); + } + + PyObject* o = val.extractPyobj(); + if (o) { + return internalize(o); + } + + auto it = mInstanceCache.find(val); + + if (it != mInstanceCache.end()) { + return it->second; + } + + mInstanceCache[val] = new PyObjSnapshot(mGraph); + + mInstanceCache[val]->becomeInternalizedOf(val, *this); + + return mInstanceCache[val]; +} diff --git a/typed_python/PyObjSnapshotMaker.hpp b/typed_python/PyObjSnapshotMaker.hpp new file mode 100644 index 000000000..6a751c004 --- /dev/null +++ b/typed_python/PyObjSnapshotMaker.hpp @@ -0,0 +1,89 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include +#include "Type.hpp" +#include "Instance.hpp" +#include "PythonTypeInternals.hpp" + +class PyObjGraphSnapshot; +class PyObjSnapshot; +class FunctionGlobal; +class FunctionOverload; + + +class PyObjSnapshotMaker { +public: + PyObjSnapshotMaker( + std::unordered_map& inObjMapCache, + std::unordered_map& inTypeMapCache, + std::unordered_map& inInstanceCache, + PyObjGraphSnapshot* inGraph, + bool inLinkBackToOriginalObject, + bool linkToInternal + ) : + mObjMapCache(inObjMapCache), + mTypeMapCache(inTypeMapCache), + mInstanceCache(inInstanceCache), + mGraph(inGraph), + mLinkBackToOriginalObject(inLinkBackToOriginalObject), + mLinkToInternal(linkToInternal) + { + } + + PyObjSnapshot* internalize(const std::string& def); + PyObjSnapshot* internalize(const MemberDefinition& def); + PyObjSnapshot* internalize(const FunctionGlobal& def); + PyObjSnapshot* internalize(const FunctionOverload& def); + PyObjSnapshot* internalize(const FunctionArg& def); + PyObjSnapshot* internalize(const ClosureVariableBinding& def); + PyObjSnapshot* internalize(const ClosureVariableBindingStep& def); + PyObjSnapshot* internalize(const std::vector& def); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::map& inBindings); + PyObjSnapshot* internalize(const std::map& inGlobals); + PyObjSnapshot* internalize(const std::map& inMethods); + PyObjSnapshot* internalize(const std::map& inMethods); + PyObjSnapshot* internalize(const std::vector& inMethods); + PyObjSnapshot* internalize(PyObject* val); + PyObjSnapshot* internalize(Type* val); + PyObjSnapshot* internalize(const Instance& val) { + return internalize(val.ref()); + } + PyObjSnapshot* internalize(InstanceRef val); + + bool linkBackToOriginalObject() const { + return mLinkBackToOriginalObject; + } + + PyObjGraphSnapshot* graph() const { + return mGraph; + } + +private: + std::unordered_map& mObjMapCache; + std::unordered_map& mTypeMapCache; + std::unordered_map& mInstanceCache; + PyObjGraphSnapshot* mGraph; + bool mLinkBackToOriginalObject; + bool mLinkToInternal; +}; + diff --git a/typed_python/PyPyObjGraphSnapshot.cpp b/typed_python/PyPyObjGraphSnapshot.cpp new file mode 100644 index 000000000..4d56f318c --- /dev/null +++ b/typed_python/PyPyObjGraphSnapshot.cpp @@ -0,0 +1,260 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyPyObjGraphSnapshot.hpp" + + +PyDoc_STRVAR(PyPyObjGraphSnapshot_doc, + "A snapshot of a collection of python objects.\n\n" +); + +PyMethodDef PyPyObjGraphSnapshot_methods[] = { + {"extractTypes", (PyCFunction)PyPyObjGraphSnapshot::extractTypes, METH_VARARGS | METH_KEYWORDS, NULL}, + {"getObjects", (PyCFunction)PyPyObjGraphSnapshot::getObjects, METH_VARARGS | METH_KEYWORDS, NULL}, + {"internalize", (PyCFunction)PyPyObjGraphSnapshot::internalize, METH_VARARGS | METH_KEYWORDS, NULL}, + {"resolveForwards", (PyCFunction)PyPyObjGraphSnapshot::resolveForwards, METH_VARARGS | METH_KEYWORDS, NULL}, + {"hashToObject", (PyCFunction)PyPyObjGraphSnapshot::hashToObject, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL} /* Sentinel */ +}; + + +/* static */ +void PyPyObjGraphSnapshot::dealloc(PyPyObjGraphSnapshot *self) +{ + if (self->mOwnsSnapshot) { + delete self->mGraphSnapshot; + } + + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyPyObjGraphSnapshot::newPyObjGraphSnapshot(PyObjGraphSnapshot* g, bool ownsIt) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)PyType_PyObjGraphSnapshot.tp_alloc(&PyType_PyObjGraphSnapshot, 0); + self->mGraphSnapshot = g; + self->mOwnsSnapshot = ownsIt; + + return (PyObject*)self; +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::hashToObject(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {"hash", NULL}; + + PyObject* hashPyobj = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", (char**)kwlist, &hashPyobj)) { + throw PythonExceptionSet(); + } + + if (!PyUnicode_Check(hashPyobj)) { + throw std::runtime_error("Invalid sha hash argument"); + } + + const char* hash = PyUnicode_AsUTF8(hashPyobj); + + ShaHash h = ShaHash::fromHexDigest(hash); + + PyObjSnapshot* s = self->mGraphSnapshot->snapshotForHash(h); + + if (!s) { + return incref(Py_None); + } + + return PyPyObjSnapshot::newPyObjSnapshot(s, graph); + }); +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::internalize(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {"sourceGraph", NULL}; + + PyObject* pySourceGraph = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", (char**)kwlist, &pySourceGraph)) { + throw PythonExceptionSet(); + } + + if (pySourceGraph->ob_type != &PyType_PyObjGraphSnapshot) { + throw std::runtime_error("Expected a PyObjGraphSnapshot for 'sourceGraph'"); + } + + PyPyObjGraphSnapshot* sourceGraph = (PyPyObjGraphSnapshot*)pySourceGraph; + + self->mGraphSnapshot->internalize(*sourceGraph->mGraphSnapshot, false); + return incref(Py_None); + }); +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::resolveForwards(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + throw PythonExceptionSet(); + } + + self->mGraphSnapshot->resolveForwards(); + + return incref(Py_None); + }); +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + throw PythonExceptionSet(); + } + + PyObjectStealer res(PySet_New(NULL)); + + for (auto o: self->mGraphSnapshot->getObjectsByIndex()) { + if (o->getType()) { + PySet_Add(res, (PyObject*)PyInstance::typeObj(o->getType())); + } + } + + return incref(res); + }); +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::getObjects(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + throw PythonExceptionSet(); + } + + PyObjectStealer res(PyList_New(0)); + + for (auto o: self->mGraphSnapshot->getObjectsByIndex()) { + PyList_Append(res, PyPyObjSnapshot::newPyObjSnapshot(o, graph)); + } + + return incref(res); + }); +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyPyObjGraphSnapshot* self; + + self = (PyPyObjGraphSnapshot*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mGraphSnapshot = nullptr; + self->mOwnsSnapshot = false; + } + + return (PyObject*)self; +} + +PyObject* PyPyObjGraphSnapshot::tp_repr(PyObject *selfObj) { + PyPyObjGraphSnapshot* pyGraph = (PyPyObjGraphSnapshot*)selfObj; + + return translateExceptionToPyObject([&]() { + return PyUnicode_FromString( + ( + "PyObjGraphSnapshot(" + + format(pyGraph->mGraphSnapshot->getObjects().size()) + + ")" + ).c_str() + ); + }); +} + + +/* static */ +int PyPyObjGraphSnapshot::init(PyPyObjGraphSnapshot *self, PyObject *args, PyObject *kwargs) +{ + self->mGraphSnapshot = new PyObjGraphSnapshot(); + self->mOwnsSnapshot = true; + return 0; +} + + +PyTypeObject PyType_PyObjGraphSnapshot = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "PyObjGraph2Snapshot", + .tp_basicsize = sizeof(PyPyObjGraphSnapshot), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyPyObjGraphSnapshot::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = PyPyObjGraphSnapshot::tp_repr, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = 0, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyPyObjGraphSnapshot_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyPyObjGraphSnapshot_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyPyObjGraphSnapshot::init, + .tp_alloc = 0, + .tp_new = PyPyObjGraphSnapshot::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyPyObjGraphSnapshot.hpp b/typed_python/PyPyObjGraphSnapshot.hpp new file mode 100644 index 000000000..1fad7c9c1 --- /dev/null +++ b/typed_python/PyPyObjGraphSnapshot.hpp @@ -0,0 +1,50 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" +#include "PyObjGraphSnapshot.hpp" + +class PyPyObjGraphSnapshot { +public: + PyObject_HEAD + + PyObjGraphSnapshot* mGraphSnapshot; + bool mOwnsSnapshot; + + static PyObject* tp_repr(PyObject *selfObj); + + static void dealloc(PyPyObjGraphSnapshot *self); + + static PyObject* hashToObject(PyObject* graph, PyObject *args, PyObject *kwargs); + + static PyObject* internalize(PyObject* graph, PyObject *args, PyObject *kwargs); + + static PyObject* resolveForwards(PyObject* graph, PyObject *args, PyObject *kwargs); + + static PyObject* extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs); + + static PyObject* getObjects(PyObject* graph, PyObject *args, PyObject *kwargs); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyObjGraphSnapshot(PyObjGraphSnapshot* o, bool ownsIt); + + static int init(PyPyObjGraphSnapshot *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_PyObjGraphSnapshot; diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp new file mode 100644 index 000000000..b11f93d18 --- /dev/null +++ b/typed_python/PyPyObjSnapshot.cpp @@ -0,0 +1,327 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyPyObjSnapshot.hpp" +#include "PyPyObjGraphSnapshot.hpp" +#include "PyObjGraphSnapshot.hpp" + +PyDoc_STRVAR(PyPyObjSnapshot_doc, + "A single object in a snapshot graph.\n\n" +); + +PyMethodDef PyPyObjSnapshot_methods[] = { + {"create", (PyCFunction)PyPyObjSnapshot::create, METH_VARARGS | METH_KEYWORDS | METH_STATIC, NULL}, + {NULL} /* Sentinel */ +}; + + +/* static */ +PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + int linkBack = 1; + static const char *kwlist[] = {"instance", "linkBackToOriginalObject", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|p", (char**)kwlist, &instance, &linkBack)) { + throw PythonExceptionSet(); + } + + std::unordered_map constantMapCache; + std::unordered_map constantMapCacheType; + std::unordered_map constantMapCacheInst; + + PyObjGraphSnapshot* graph = new PyObjGraphSnapshot(); + + PyObjSnapshotMaker maker( + constantMapCache, + constantMapCacheType, + constantMapCacheInst, + graph, + linkBack, + // don't link to the internal graph since we don't have + // that modeled correctly in the python wrappers yet. + false + ); + + PyObjSnapshot* object = maker.internalize(instance); + + PyObjectStealer graphObj(PyPyObjGraphSnapshot::newPyObjGraphSnapshot(graph, true)); + + return PyPyObjSnapshot::newPyObjSnapshot(object, graphObj); + }); +} + +/* static */ +void PyPyObjSnapshot::dealloc(PyPyObjSnapshot *self) +{ + decref(self->mElements); + decref(self->mKeys); + decref(self->mGraph); + decref(self->mByKey); + decref(self->mNamedElements); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyPyObjSnapshot::newPyObjSnapshot(PyObjSnapshot* g, PyObject* pyGraphPtr) { + PyPyObjSnapshot* self = (PyPyObjSnapshot*)PyType_PyObjSnapshot.tp_alloc(&PyType_PyObjSnapshot, 0); + + self->mPyobj = g; + self->mGraph = incref(pyGraphPtr); + self->mElements = nullptr; + self->mKeys = nullptr; + self->mByKey = nullptr; + self->mNamedElements = nullptr; + + return (PyObject*)self; +} + +/* static */ +PyObject* PyPyObjSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyPyObjSnapshot* self; + + self = (PyPyObjSnapshot*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mPyobj = nullptr; + self->mElements = nullptr; + self->mKeys = nullptr; + self->mGraph = nullptr; + self->mByKey = nullptr; + self->mNamedElements = nullptr; + } + + return (PyObject*)self; +} + +PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { + PyPyObjSnapshot* pySnap = (PyPyObjSnapshot*)selfObj; + + return translateExceptionToPyObject([&] { + if (!PyUnicode_Check(attrName)) { + throw std::runtime_error("Expected a string for attribute name"); + } + + std::string attr(PyUnicode_AsUTF8(attrName)); + + PyObjSnapshot* obj = pySnap->mPyobj; + + auto it = obj->namedElements().find(attr); + if (it != obj->namedElements().end()) { + return PyPyObjSnapshot::newPyObjSnapshot( + it->second, + it->second->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr + ); + } + + auto it2 = obj->namedInts().find(attr); + if (it2 != obj->namedInts().end()) { + return PyLong_FromLong(it2->second); + } + + if (attr == "pyobj") { + PyObject* o = obj->getPyObj(); + if (!o) { + throw std::runtime_error("PyObjSnapshot of kind " + obj->kindAsString() + " has no pyobj"); + } + return incref(o); + } + + if (attr == "kind") { + return PyUnicode_FromString(obj->kindAsString().c_str()); + } + + if (attr == "graph") { + return incref(pySnap->mGraph ? pySnap->mGraph : Py_None); + } + + if (attr == "shaHash" && obj->getGraph()) { + ShaHash hash = obj->getGraph()->hashFor(obj); + return PyUnicode_FromString(hash.digestAsHexString().c_str()); + } + + if (attr == "type" && obj->getType()) { + return incref((PyObject*)PyInstance::typeObj(obj->getType())); + } + + if (attr == "stringValue" && obj->isString()) { + return PyUnicode_FromString(obj->getStringValue().c_str()); + } + + if (attr == "name") { + return PyUnicode_FromString(obj->getName().c_str()); + } + + if (attr == "moduleName") { + return PyUnicode_FromString(obj->getModuleName().c_str()); + } + + if (attr == "elements") { + if (!pySnap->mElements) { + pySnap->mElements = PyList_New(0); + for (auto p: obj->elements()) { + PyList_Append( + pySnap->mElements, + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr + ) + ); + } + } + + return incref(pySnap->mElements); + } + + if (attr == "keys") { + if (!pySnap->mKeys) { + pySnap->mKeys = PyList_New(0); + for (auto p: obj->keys()) { + PyObjectStealer subPySnap( + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr + ) + ); + + PyList_Append(pySnap->mKeys, subPySnap); + } + } + + return incref(pySnap->mKeys); + } + + if (attr == "byKey") { + if (!pySnap->mByKey) { + pySnap->mByKey = PyDict_New(); + for (long k = 0; k < obj->keys().size(); k++) { + PyObjSnapshot* p = obj->elements()[k]; + + PyObjectStealer subPySnap( + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr + ) + ); + + PyDict_SetItem( + pySnap->mByKey, + obj->keys()[k]->getPyObj(), + subPySnap + ); + } + } + + return incref(pySnap->mByKey); + } + + if (attr == "namedElements") { + if (!pySnap->mNamedElements) { + pySnap->mNamedElements = PyDict_New(); + for (auto nameAndSnap: obj->namedElements()) { + PyObjSnapshot* p = nameAndSnap.second; + PyObjectStealer newSnap( + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr + ) + ); + PyDict_SetItemString( + pySnap->mNamedElements, + nameAndSnap.first.c_str(), + newSnap + ); + } + } + + return incref(pySnap->mNamedElements); + } + + return PyObject_GenericGetAttr(selfObj, attrName); + }); +} + +PyObject* PyPyObjSnapshot::tp_repr(PyObject *selfObj) { + PyPyObjSnapshot* self = (PyPyObjSnapshot*)selfObj; + + return translateExceptionToPyObject([&]() { + return PyUnicode_FromString(self->mPyobj->toString().c_str()); + }); +} + + +/* static */ +int PyPyObjSnapshot::init(PyPyObjSnapshot *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "PyObjSnapshot cannot be initialized directly"); + return -1; +} + + +PyTypeObject PyType_PyObjSnapshot = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "PyObjSnapshot", + .tp_basicsize = sizeof(PyPyObjSnapshot), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyPyObjSnapshot::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = PyPyObjSnapshot::tp_repr, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = PyPyObjSnapshot::tp_getattro, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyPyObjSnapshot_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyPyObjSnapshot_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyPyObjSnapshot::init, + .tp_alloc = 0, + .tp_new = PyPyObjSnapshot::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyPyObjSnapshot.hpp b/typed_python/PyPyObjSnapshot.hpp new file mode 100644 index 000000000..33515145d --- /dev/null +++ b/typed_python/PyPyObjSnapshot.hpp @@ -0,0 +1,51 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" + +class PyPyObjSnapshot { +public: + PyObject_HEAD + + PyObjSnapshot* mPyobj; + + // a graph object we're keeping alive + PyObject* mGraph; + + // caches of our 'elements/keys/byKey' members, which we produce on demand + PyObject* mElements; + PyObject* mKeys; + PyObject* mByKey; + PyObject* mNamedElements; + + static PyObject* tp_repr(PyObject *selfObj); + + static PyObject* create(PyObject* self, PyObject* args, PyObject* kwargs); + + static PyObject* tp_getattro(PyObject* selfObj, PyObject* attr); + + static void dealloc(PyPyObjSnapshot *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyObjSnapshot(PyObjSnapshot* o, PyObject* pyGraphPtr=nullptr); + + static int init(PyPyObjSnapshot *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_PyObjSnapshot; diff --git a/typed_python/PyPythonSubclassInstance.hpp b/typed_python/PyPythonSubclassInstance.hpp deleted file mode 100644 index b37b46c28..000000000 --- a/typed_python/PyPythonSubclassInstance.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/****************************************************************************** - Copyright 2017-2019 typed_python Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -#pragma once - -#include "PyInstance.hpp" - -class PyPythonSubclassInstance : public PyInstance { -public: - typedef PythonSubclass modeled_type; - - static void copyConstructFromPythonInstanceConcrete(PythonSubclass* eltType, instance_ptr tgt, PyObject* pyRepresentation, ConversionLevel level) { - std::pair typeAndPtr = extractTypeAndPtrFrom(pyRepresentation); - Type* argType = typeAndPtr.first; - instance_ptr argDataPtr = typeAndPtr.second; - - if (argType && Type::typesEquivalent(argType, eltType)) { - eltType->getBaseType()->copy_constructor(tgt, argDataPtr); - return; - } - - if (argType && Type::typesEquivalent(argType, eltType->getBaseType())) { - eltType->getBaseType()->copy_constructor(tgt, argDataPtr); - return; - } - - PyInstance::copyConstructFromPythonInstanceConcrete(eltType, tgt, pyRepresentation, level); - } - - PyObject* tp_getattr_concrete(PyObject* pyAttrName, const char* attrName) { - return specializeForType((PyObject*)this, [&](auto& subtype) { - return subtype.tp_getattr_concrete(pyAttrName, attrName); - }, type()->getBaseType()); - } - - int tp_setattr_concrete(PyObject* pyAttrName, PyObject* attrVal) { - return specializeForTypeReturningInt((PyObject*)this, [&](auto& subtype) { - return subtype.tp_setattr_concrete(pyAttrName, attrVal); - }, type()->getBaseType()); - } - - static bool pyValCouldBeOfTypeConcrete(modeled_type* eltType, PyObject* pyRepresentation, ConversionLevel level) { - Type* argType = extractTypeFrom(pyRepresentation->ob_type); - - if (argType && Type::typesEquivalent(argType, eltType)) { - return true; - } - - if (argType && Type::typesEquivalent(argType, eltType->getBaseType())) { - return true; - } - - return false; - } - - static void constructFromPythonArgumentsConcrete(Type* t, uint8_t* data, PyObject* args, PyObject* kwargs) { - constructFromPythonArguments(data, (Type*)t->getBaseType(), args, kwargs); - } - - Py_ssize_t mp_and_sq_length_concrete() { - return specializeForTypeReturningSizeT((PyObject*)this, [&](auto& subtype) { - return subtype.mp_and_sq_length_concrete(); - }, type()->getBaseType()); - } - - int pyInquiryConcrete(const char* op, const char* opErrRep) { - // op == '__bool__' - return 1; - } -}; diff --git a/typed_python/PySetInstance.cpp b/typed_python/PySetInstance.cpp index 582180cdc..218a1e463 100644 --- a/typed_python/PySetInstance.cpp +++ b/typed_python/PySetInstance.cpp @@ -1403,22 +1403,24 @@ bool PySetInstance::pyValCouldBeOfTypeConcrete(modeled_type* type, PyObject* pyR } PyMethodDef* PySetInstance::typeMethodsConcrete(Type* t) { - return new PyMethodDef[18]{{"add", (PyCFunction)PySetInstance::setAdd, METH_VARARGS, setAdd_doc}, - {"pop", (PyCFunction)PySetInstance::setPop, METH_VARARGS, setPop_doc}, - {"discard", (PyCFunction)PySetInstance::setDiscard, METH_VARARGS, setDiscard_doc}, - {"remove", (PyCFunction)PySetInstance::setRemove, METH_VARARGS, setRemove_doc}, - {"clear", (PyCFunction)PySetInstance::setClear, METH_VARARGS, setClear_doc}, - {"copy", (PyCFunction)PySetInstance::setCopy, METH_VARARGS, setCopy_doc}, - {"union", (PyCFunction)PySetInstance::setUnion, METH_VARARGS, setUnion_doc}, - {"update", (PyCFunction)PySetInstance::setUpdate, METH_VARARGS, setUpdate_doc}, - {"intersection", (PyCFunction)PySetInstance::setIntersection, METH_VARARGS, setIntersection_doc}, - {"intersection_update", (PyCFunction)PySetInstance::setIntersectionUpdate, METH_VARARGS, setIntersectionUpdate_doc}, - {"difference", (PyCFunction)PySetInstance::setDifference, METH_VARARGS, setDifference_doc}, - {"difference_update", (PyCFunction)PySetInstance::setDifferenceUpdate, METH_VARARGS, setDifferenceUpdate_doc}, - {"symmetric_difference", (PyCFunction)PySetInstance::setSymmetricDifference, METH_VARARGS, setSymmetricDifference_doc}, - {"symmetric_difference_update", (PyCFunction)PySetInstance::setSymmetricDifferenceUpdate, METH_VARARGS, setSymmetricDifferenceUpdate_doc}, - {"issubset", (PyCFunction)PySetInstance::setIsSubset, METH_VARARGS, setIsSubset_doc}, - {"issuperset", (PyCFunction)PySetInstance::setIsSuperset, METH_VARARGS, setIsSuperset_doc}, - {"isdisjoint", (PyCFunction)PySetInstance::setIsDisjoint, METH_VARARGS, setIsDisjoint_doc}, - {NULL, NULL}}; + return new PyMethodDef[18]{ + {"add", (PyCFunction)PySetInstance::setAdd, METH_VARARGS, setAdd_doc}, + {"pop", (PyCFunction)PySetInstance::setPop, METH_VARARGS, setPop_doc}, + {"discard", (PyCFunction)PySetInstance::setDiscard, METH_VARARGS, setDiscard_doc}, + {"remove", (PyCFunction)PySetInstance::setRemove, METH_VARARGS, setRemove_doc}, + {"clear", (PyCFunction)PySetInstance::setClear, METH_VARARGS, setClear_doc}, + {"copy", (PyCFunction)PySetInstance::setCopy, METH_VARARGS, setCopy_doc}, + {"union", (PyCFunction)PySetInstance::setUnion, METH_VARARGS, setUnion_doc}, + {"update", (PyCFunction)PySetInstance::setUpdate, METH_VARARGS, setUpdate_doc}, + {"intersection", (PyCFunction)PySetInstance::setIntersection, METH_VARARGS, setIntersection_doc}, + {"intersection_update", (PyCFunction)PySetInstance::setIntersectionUpdate, METH_VARARGS, setIntersectionUpdate_doc}, + {"difference", (PyCFunction)PySetInstance::setDifference, METH_VARARGS, setDifference_doc}, + {"difference_update", (PyCFunction)PySetInstance::setDifferenceUpdate, METH_VARARGS, setDifferenceUpdate_doc}, + {"symmetric_difference", (PyCFunction)PySetInstance::setSymmetricDifference, METH_VARARGS, setSymmetricDifference_doc}, + {"symmetric_difference_update", (PyCFunction)PySetInstance::setSymmetricDifferenceUpdate, METH_VARARGS, setSymmetricDifferenceUpdate_doc}, + {"issubset", (PyCFunction)PySetInstance::setIsSubset, METH_VARARGS, setIsSubset_doc}, + {"issuperset", (PyCFunction)PySetInstance::setIsSuperset, METH_VARARGS, setIsSuperset_doc}, + {"isdisjoint", (PyCFunction)PySetInstance::setIsDisjoint, METH_VARARGS, setIsDisjoint_doc}, + {NULL, NULL} + }; } diff --git a/typed_python/PyTemporaryReferenceTracer.cpp b/typed_python/PyTemporaryReferenceTracer.cpp index 046436a71..2afb86c96 100644 --- a/typed_python/PyTemporaryReferenceTracer.cpp +++ b/typed_python/PyTemporaryReferenceTracer.cpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2021 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -80,9 +80,29 @@ int PyTemporaryReferenceTracer::globalTraceFun(PyObject* dummyObj, PyFrameObject if (frameAction.lineNumber != PyFrame_GetLineNumber(frame) || forceProcess) { if (frameAction.action == TraceAction::ConvertTemporaryReference) { ((PyInstance*)frameAction.obj)->resolveTemporaryReference(); + decref(frameAction.obj); + } + else if (frameAction.action == TraceAction::Decref) { + decref(frameAction.obj); + } + else if (frameAction.action == TraceAction::Autoresolve) { + PyErrorStasher stashCurrentException; + + // attempt to autoresolve the type. Note that if we + // fail to do so, we just swallow the exception, which + // is not great, but we can't be throwing exceptions + // in here... + try { + frameAction.typ->tryToAutoresolve(); + } catch(std::exception& e) { + // just swallow it + std::cout << "Warning: autoresolve on " + << frameAction.typ->name() << " failed: " << e.what() << "\n"; + + } catch(PythonExceptionSet& e) { + PyErr_Clear(); + } } - - decref(frameAction.obj); } else { persistingActions.push_back(frameAction); } @@ -163,6 +183,27 @@ void PyTemporaryReferenceTracer::keepaliveForCurrentInstruction(PyObject* o, PyF installGlobalTraceHandlerIfNecessary(); } +void PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* t, PyFrameObject* f) { + if (!globalTracer.autoresolutionEnabled) { + return; + } + + // mark that we're going to trace + globalTracer.frameToActions[f].push_back( + FrameAction( + t, + TraceAction::Autoresolve, + PyFrame_GetLineNumber(f) + ) + ); + + if (globalTracer.mostRecentEmptyFrame == f) { + globalTracer.mostRecentEmptyFrame = nullptr; + } + + installGlobalTraceHandlerIfNecessary(); +} + void PyTemporaryReferenceTracer::traceObject(PyObject* o) { PyThreadState *tstate = PyThreadState_GET(); PyFrameObject *f = tstate->frame; @@ -172,6 +213,42 @@ void PyTemporaryReferenceTracer::traceObject(PyObject* o) { } } +Type* PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* o) { + if (!globalTracer.autoresolutionEnabled) { + return o; + } + + if (!o->isForwardDefined() || o->isResolved()) { + return o; + } + + PyThreadState *tstate = PyThreadState_GET(); + PyFrameObject *f = tstate->frame; + + // search upwards until we find a frame that's not running in typed_python.internals + auto isInternals = [&]() { + if (!PyUnicode_Check(f->f_code->co_filename)) { + return false; + } + + return endsWith(PyUnicode_AsUTF8(f->f_code->co_filename), "typed_python/internals.py"); + }; + + while (isInternals()) { + f = PyFrame_GetBack(f); + if (!f) { + throw std::runtime_error("Frame failed"); + } + } + + if (f) { + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(o, f); + o->registerContainingFrameForAutoresolve(f); + } + + return o; +} + void PyTemporaryReferenceTracer::keepaliveForCurrentInstruction(PyObject* o) { PyThreadState *tstate = PyThreadState_GET(); PyFrameObject *f = tstate->frame; diff --git a/typed_python/PyTemporaryReferenceTracer.hpp b/typed_python/PyTemporaryReferenceTracer.hpp index 75d6a278b..9620d42cd 100644 --- a/typed_python/PyTemporaryReferenceTracer.hpp +++ b/typed_python/PyTemporaryReferenceTracer.hpp @@ -21,7 +21,8 @@ enum class TraceAction { ConvertTemporaryReference, - Decref + Decref, + Autoresolve }; class PyTemporaryReferenceTracer { @@ -29,7 +30,8 @@ class PyTemporaryReferenceTracer { PyTemporaryReferenceTracer() : mostRecentEmptyFrame(nullptr), priorTraceFunc(nullptr), - priorTraceFuncArg(nullptr) + priorTraceFuncArg(nullptr), + autoresolutionEnabled(false) {} // perform an action on the first instruction where a @@ -39,16 +41,30 @@ class PyTemporaryReferenceTracer { public: FrameAction(PyObject* inO, TraceAction inA, int inLine) : obj(inO), + typ(nullptr), + action(inA), + lineNumber(inLine) + { + } + + FrameAction(Type* inT, TraceAction inA, int inLine) : + obj(nullptr), + typ(inT), action(inA), lineNumber(inLine) { } PyObject* obj; + Type* typ; TraceAction action; int lineNumber; }; + void enableTypeAutoresolution(bool enabled) { + autoresolutionEnabled = enabled; + } + std::unordered_map > frameToActions; std::unordered_map > codeObjectToExpressionLines; @@ -58,6 +74,8 @@ class PyTemporaryReferenceTracer { Py_tracefunc priorTraceFunc; + bool autoresolutionEnabled; + PyObject* priorTraceFuncArg; bool isLineNewStatement(PyObject* code, int line); @@ -72,6 +90,12 @@ class PyTemporaryReferenceTracer { static void traceObject(PyObject* o); + // on the next instruction of 'frame', attempt to autoresolve 't' + static void autoresolveOnNextInstruction(Type* t, PyFrameObject* frame); + + // on the next instruction of 'frame', attempt to autoresolve 't' + static Type* autoresolveOnNextInstruction(Type* t); + static void installGlobalTraceHandlerIfNecessary(); static void keepaliveForCurrentInstruction(PyObject* o); diff --git a/typed_python/PythonObjectOfTypeType.cpp b/typed_python/PythonObjectOfTypeType.cpp index c20c2d9dd..589a58918 100644 --- a/typed_python/PythonObjectOfTypeType.cpp +++ b/typed_python/PythonObjectOfTypeType.cpp @@ -506,20 +506,22 @@ bool PythonObjectOfType::cmp(instance_ptr left, instance_ptr right, int pyCompar } // static -PythonObjectOfType* PythonObjectOfType::Make(PyTypeObject* pyType, PyObject* givenType) { +PythonObjectOfType* PythonObjectOfType::Make(PyTypeObject* pyType) { PyEnsureGilAcquired getTheGil; - typedef std::pair keytype; + typedef PyTypeObject* keytype; static std::map m; - keytype key(pyType, givenType); + keytype key(pyType); auto it = m.find(key); if (it == m.end()) { + Type* t = new PythonObjectOfType(pyType); + it = m.insert( - std::make_pair(key, new PythonObjectOfType(pyType, givenType)) + std::make_pair(key, (PythonObjectOfType*)t->forwardResolvesTo()) ).first; } @@ -534,6 +536,11 @@ PythonObjectOfType* PythonObjectOfType::AnyPyObject() { return Make((PyTypeObject*)t); } +PythonObjectOfType* PythonObjectOfType::AnyPyDict() { + static PythonObjectOfType* t = Make((PyTypeObject*)&PyDict_Type); + return t; +} + PythonObjectOfType* PythonObjectOfType::AnyPyType() { PyTypeObject* t = (PyTypeObject*)staticPythonInstance( "typed_python.internals", "type" diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index 18e3954f0..de3602a64 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -113,22 +113,37 @@ class PyObjectHandleTypeBase : public Type { //refcount when we completely release a handle to a python object). class PythonObjectOfType : public PyObjectHandleTypeBase { public: - PythonObjectOfType(PyTypeObject* typePtr, PyObject* givenType) : + // clone pathway + PythonObjectOfType() : PyObjectHandleTypeBase(TypeCategory::catPythonObjectOfType) + { + } + + PythonObjectOfType(PyTypeObject* typePtr) : PyObjectHandleTypeBase(TypeCategory::catPythonObjectOfType) { mPyTypePtr = (PyTypeObject*)incref((PyObject*)typePtr); + m_is_forward_defined = true; + recomputeName(); + } - if (givenType) { - mGivenType = incref(givenType); - } else { - mGivenType = nullptr; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + mPyTypePtr = ((PythonObjectOfType*)forwardDefinitionOfSelf)->mPyTypePtr; + m_name = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_name; + m_size = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_size; + m_is_default_constructible = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_is_default_constructible; + } - m_name = std::string("PythonObjectOfType(") + mPyTypePtr->tp_name + ")"; + void initializeDuringDeserialization(PyTypeObject* typePtr) { + mPyTypePtr = (PyTypeObject*)incref((PyObject*)typePtr); + } - m_is_simple = false; + void updateInternalTypePointersConcrete( + const std::map& groupMap + ) { + } - endOfConstructorInitialization(); // finish initializing the type object. + Type* cloneForForwardResolutionConcrete() { + return new PythonObjectOfType(); } template @@ -137,10 +152,6 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { v.visitTopo((PyObject*)mPyTypePtr); } - bool isBinaryCompatibleWithConcrete(Type* other) { - return other == this; - } - template void _visitContainedTypes(const visitor_type& visitor) { } @@ -149,7 +160,11 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { void _visitReferencedTypes(const visitor_type& visitor) { } - bool _updateAfterForwardTypesChanged() { + std::string computeRecursiveNameConcrete(TypeStack& types) { + return std::string("PythonObjectOfType(") + mPyTypePtr->tp_name + ")"; + } + + void postInitializeConcrete() { m_size = sizeof(layout_type*); int isinst = PyObject_IsInstance(Py_None, (PyObject*)mPyTypePtr); @@ -159,10 +174,6 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { } m_is_default_constructible = isinst != 0; - - //none of these values can ever change, so we can just return - //because we don't need to be updated again. - return false; } int64_t refcount(instance_ptr self) const { @@ -213,10 +224,12 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { // construct a new Type. If 'givenType' is not NULL, then it's the type the user // gave us (and we inferred a real type from it based on the lookup table in internals.py) - static PythonObjectOfType* Make(PyTypeObject* pyType, PyObject* givenType=NULL); + static PythonObjectOfType* Make(PyTypeObject* pyType); static PythonObjectOfType* AnyPyObject(); + static PythonObjectOfType* AnyPyDict(); + static PythonObjectOfType* AnyPyType(); PyTypeObject* pyType() const { @@ -225,9 +238,4 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { private: PyTypeObject* mPyTypePtr; - - // this is the object we were given, which in some cases might not really - // be a type, but which users expect to refer to a type (for instance, a threading.RLock, - // or anything else in internals._nonTypesAcceptedAsTypes); - PyObject* mGivenType; }; diff --git a/typed_python/PythonSerializationContext.hpp b/typed_python/PythonSerializationContext.hpp index f36c87932..14b96cf8b 100644 --- a/typed_python/PythonSerializationContext.hpp +++ b/typed_python/PythonSerializationContext.hpp @@ -145,7 +145,7 @@ class PythonSerializationContext : public SerializationContext { void serializeNativeType(Type* nativeType, SerializationBuffer& b, size_t fieldNumber) const; - void serializeNativeTypeInner(Type* nativeType, SerializationBuffer& b, size_t fieldNumber) const; + void serializeNativeTypeInner(Type* nativeType, SerializationBuffer& b, size_t fieldNumber, bool checkIfObjectIsNamed) const; void serializeClassMembers(const std::vector& members, SerializationBuffer& b, int fieldNumber) const; @@ -163,7 +163,9 @@ class PythonSerializationContext : public SerializationContext { void deserializeClassClassMemberDict(std::map& dict, DeserializationBuffer& b, int wireType) const; - Type* deserializeNativeTypeInner(DeserializationBuffer& b, size_t wireType, bool actuallyProduceResult) const; + static Type* constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat, bool isForwardDefined = false); + + void deserializeNativeTypeIntoBlank(DeserializationBuffer& b, size_t wireType, Type* blankShell, std::string inName) const; Instance deserializeNativeInstance(DeserializationBuffer& b, size_t wireType) const; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index fce5b8c3d..de6db777f 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -573,6 +573,105 @@ PyObject* prepareBlankInstanceOfType(MutuallyRecursiveTypeGroup* outGroup, PyTyp return obj; } +/* static */ +Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat, bool isForwardDefined) { + Type* result = nullptr; + + if (typeCat == Type::TypeCategory::catValue) { + result = new Value(); + } + if (typeCat == Type::TypeCategory::catOneOf) { + result = new OneOfType(); + } + if (typeCat == Type::TypeCategory::catTupleOf) { + result = new TupleOfType(); + } + if (typeCat == Type::TypeCategory::catPointerTo) { + result = new PointerTo(); + } + if (typeCat == Type::TypeCategory::catListOf) { + result = new ListOfType(); + } + if (typeCat == Type::TypeCategory::catNamedTuple) { + result = new NamedTuple(); + } + if (typeCat == Type::TypeCategory::catTuple) { + result = new Tuple(); + } + if (typeCat == Type::TypeCategory::catDict) { + result = new DictType(); + } + if (typeCat == Type::TypeCategory::catConstDict) { + result = new ConstDictType(); + } + if (typeCat == Type::TypeCategory::catAlternative) { + result = new Alternative(); + } + if (typeCat == Type::TypeCategory::catConcreteAlternative) { + result = new ConcreteAlternative(); + } + if (typeCat == Type::TypeCategory::catPythonObjectOfType) { + result = new PythonObjectOfType(); + } + if (typeCat == Type::TypeCategory::catBoundMethod) { + result = new BoundMethod(); + } + if (typeCat == Type::TypeCategory::catAlternativeMatcher) { + result = new AlternativeMatcher(); + } + if (typeCat == Type::TypeCategory::catClass) { + result = new Class(); + } + if (typeCat == Type::TypeCategory::catHeldClass) { + result = new HeldClass(); + } + if (typeCat == Type::TypeCategory::catFunction) { + result = new Function(); + } + if (typeCat == Type::TypeCategory::catForward) { + result = new Forward(""); + } + if (typeCat == Type::TypeCategory::catSet) { + result = new SetType(); + } + if (typeCat == Type::TypeCategory::catTypedCell) { + result = new TypedCellType(); + } + if (typeCat == Type::TypeCategory::catRefTo) { + result = new RefTo(); + } + if (typeCat == Type::TypeCategory::catSubclassOf) { + result = new SubclassOfType(); + } + + if (result) { + result->markActivelyBeingDeserialized(isForwardDefined, "Unknown", "Unknown"); + return result; + } + + if (typeCat == Type::TypeCategory::catPyCell + || typeCat == Type::TypeCategory::catEmbeddedMessage + || typeCat == Type::TypeCategory::catNone + || typeCat == Type::TypeCategory::catBool + || typeCat == Type::TypeCategory::catUInt8 + || typeCat == Type::TypeCategory::catUInt16 + || typeCat == Type::TypeCategory::catUInt32 + || typeCat == Type::TypeCategory::catUInt64 + || typeCat == Type::TypeCategory::catInt8 + || typeCat == Type::TypeCategory::catInt16 + || typeCat == Type::TypeCategory::catInt32 + || typeCat == Type::TypeCategory::catInt64 + || typeCat == Type::TypeCategory::catString + || typeCat == Type::TypeCategory::catBytes + || typeCat == Type::TypeCategory::catFloat32 + || typeCat == Type::TypeCategory::catFloat64 + ) { + throw std::runtime_error("Can't produce a blank instance of a simple type"); + } + + throw std::runtime_error("Corrupt TypeCategory: can't produce a blank instance"); +} + MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecursiveTypeGroup( DeserializationBuffer& b, size_t inWireType ) const { @@ -590,7 +689,9 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur std::map indexOfObjToIndexOfType; std::map indicesOfNativeTypes; - std::map indicesOfNativeTypeCategories; + + std::map indicesOfNativeTypeNames; + std::map indicesWrittenAsObjectAndRep; std::map indicesWithSerializedBodies; @@ -613,7 +714,10 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur groupHash = ShaHash::fromDigest(b.readStringObject()); hasGroupHash = true; - outGroup = MutuallyRecursiveTypeGroup::getGroupFromIntendedHash(groupHash); + outGroup = MutuallyRecursiveTypeGroup::getGroupFromIntendedHash( + groupHash, + VisibilityType::Identity + ); if (memo != -1 && outGroup) { b.addCachedPointer(memo, (void*)outGroup, nullptr); @@ -625,7 +729,10 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } if (!outGroup) { - outGroup = MutuallyRecursiveTypeGroup::DeserializerGroup(groupHash); + outGroup = MutuallyRecursiveTypeGroup::DeserializerGroup( + groupHash, + VisibilityType::Identity + ); if (memo != -1) { b.addCachedPointer(memo, (void*)outGroup, nullptr); @@ -651,21 +758,22 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur assertWireTypesEqual(subSubWireType, WireType::BYTES); std::string fwdName = b.readStringObject(); - if (actuallyBuildGroup) { - Forward* f = Forward::Make(fwdName); - - outGroup->setIndexToObject(indexInGroup, f); - indicesOfNativeTypes[indexInGroup] = f; - } else { - indicesOfNativeTypes[indexInGroup] = nullptr; - } + indicesOfNativeTypeNames[indexInGroup] = fwdName; setSomething = true; } else if (fieldInIndex == 2 && kind == 0) { assertWireTypesEqual(subSubWireType, WireType::VARINT); - indicesOfNativeTypeCategories[indexInGroup] = Type::TypeCategory( - b.readUnsignedVarint() - ); + + auto typeCat = Type::TypeCategory(b.readUnsignedVarint()); + + if (actuallyBuildGroup) { + Type* type = constructBlankSubclassOfTypeCategory(typeCat); + outGroup->setIndexToObject(indexInGroup, type); + indicesOfNativeTypes[indexInGroup] = type; + } else { + indicesOfNativeTypes[indexInGroup] = nullptr; + } + } else if (fieldInIndex == 1 && kind == 1) { // this is the now-deprecated field code for sending a named object. @@ -916,66 +1024,69 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } } else if (indicesOfNativeTypes.find(indexInGroup) != indicesOfNativeTypes.end()) { - Type* t = deserializeNativeTypeInner(b, subWireType, actuallyBuildGroup); + Type* targetType = nullptr; if (actuallyBuildGroup) { - if (!t) { - throw std::runtime_error("Somehow, deserializeNativeTypeInner didn't return a type."); - } - if (!indicesOfNativeTypes[indexInGroup]) { throw std::runtime_error("indicesOfNativeTypes[indexInGroup] is somehow none"); } - if (!indicesOfNativeTypes[indexInGroup]->isForward()) { - throw std::runtime_error("indicesOfNativeTypes[indexInGroup] was already resolved"); - } - - if (t->isForward()) { - // we actually deserialized a forward here! this means everyone - // else is expecting this actual instance to be a forward, so we - // don't want to swap it out - ((Forward*)indicesOfNativeTypes[indexInGroup])->define( - ((Forward*)t)->getTarget() - ); - } else { - ((Forward*)indicesOfNativeTypes[indexInGroup])->define(t); - indicesOfNativeTypes[indexInGroup] = t; - // update the mutually recursive group or we'll end up with - // downstream consumers actually pulling out the forwards - outGroup->setIndexToObject(indexInGroup, t); - } + targetType = indicesOfNativeTypes[indexInGroup]; } + + deserializeNativeTypeIntoBlank( + b, + subWireType, + targetType, + indicesOfNativeTypeNames[indexInGroup] + ); } else { throw std::runtime_error("Corrupt MutuallyRecursiveTypeGroup group"); } }); } else if (fieldNumber == 4) { - // fill out the globals of function objects. we have to deserialize these - // in a separate final pass because they can have references to existing - // native types, and we need those objects to be filled out fully. - b.consumeCompoundMessage(wireType, [&](size_t indexInGroup, size_t subWireType) { - b.consumeCompoundMessage(subWireType, [&](size_t overloadIx, size_t subSubWireType) { - PyObjectStealer overloadGlobals(deserializePythonObject(b, subSubWireType)); - - if (actuallyBuildGroup) { - if (!indicesOfNativeTypes[indexInGroup] || - !indicesOfNativeTypes[indexInGroup]->isFunction()) { - throw std::runtime_error("Invalid function overload globals: not a function"); - } + b.consumeCompoundMessage(wireType, [&](size_t indexInGroup, size_t subWireType) {}); + + // now that we've constructed the type diagram, walk over all the types + // and finalize them. We have to do this before we wire function globals + // because sometime the function globals hold python references to type objects + // and the type objects have to be filled out completely and have PyType_Ready + // called on them before they are valid GC objects. + if (actuallyBuildGroup) { + // recompute type names + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->recomputeName(); + } - Function* f = (Function*)indicesOfNativeTypes[indexInGroup]; - if (overloadIx >= f->getOverloads().size()) { - throw std::runtime_error( - "Invalid function overload globals: overload index out of bounds" - ); + // do post-initialization + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto ixAndType: indicesOfNativeTypes) { + if (ixAndType.second->postInitialize()) { + anyUpdated = true; } + } + passCt += 1; - ((Function::Overload*)&f->getOverloads()[overloadIx])->setGlobals(overloadGlobals); + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > indicesOfNativeTypes.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); } - }); - }); + } + + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->finalizeType(); + } + + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->typeFinishedBeingDeserializedPhase1(); + } + } // now that we've set function globals, we need to go over all the classes and // held classes and make sure the versions of the function types they're actually @@ -1009,10 +1120,16 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur ); } - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)namedObj); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)namedObj, + VisibilityType::Identity + ); MutuallyRecursiveTypeGroup* group = ( - MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)namedObj).first + MutuallyRecursiveTypeGroup::groupAndIndexFor( + (PyObject*)namedObj, + VisibilityType::Identity + ).first ); if (!outGroup) { @@ -1023,7 +1140,7 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } } else if (fieldNumber == 6) { - // field 5 indicates that this is a mutually recursive type group + // field indicates that this is a mutually recursive type group // identified by a perfect factory PyObjectStealer factoryAndArgs(deserializePythonObject(b, wireType)); @@ -1047,10 +1164,16 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur throw PythonExceptionSet(); } - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)instance); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)instance, + VisibilityType::Identity + ); MutuallyRecursiveTypeGroup* group = ( - MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)instance).first + MutuallyRecursiveTypeGroup::groupAndIndexFor( + (PyObject*)instance, + VisibilityType::Identity + ).first ); if (!outGroup) { @@ -1083,6 +1206,10 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } if (actuallyBuildGroup) { + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->typeFinishedBeingDeserializedPhase2(); + } + outGroup->finalizeDeserializerGroup(); } @@ -1148,10 +1275,12 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b MutuallyRecursiveTypeGroup* group = nullptr; std::string name; PyObjectHolder typeFromRep; + int64_t memo = -1; int32_t indexInGroup = -1; int32_t kind = -1; int32_t which = -1; int32_t category = -1; + Type* forwardType = nullptr; b.consumeCompoundMessage(inWireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { @@ -1183,6 +1312,39 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b if (kind == 4 && fieldNumber == 2) { assertWireTypesEqual(wireType, WireType::VARINT); indexInGroup = b.readUnsignedVarint(); + } else + if (kind == 5 && fieldNumber == 1) { + assertWireTypesEqual(wireType, WireType::VARINT); + memo = b.readUnsignedVarint(); + forwardType = (Type*)b.lookupCachedPointer(memo); + } else + if (kind == 5 && fieldNumber == 2) { + assertWireTypesEqual(wireType, WireType::VARINT); + category = b.readUnsignedVarint(); + } else + if (kind == 5 && fieldNumber == 3) { + assertWireTypesEqual(wireType, WireType::BYTES); + name = b.readStringObject(); + } else + if (kind == 5 && fieldNumber == 4) { + if (memo == -1) { + throw std::runtime_error("Invalid forward type serialization: expected a memo"); + } + if (category == -1) { + throw std::runtime_error("Invalid forward type serialization: expected a category"); + } + if (forwardType) { + throw std::runtime_error("Invalid forward type serialization: type is double-defined"); + } + forwardType = constructBlankSubclassOfTypeCategory(Type::TypeCategory(category), true); + b.addCachedPointer(memo, (void*)forwardType, nullptr); + deserializeNativeTypeIntoBlank(b, wireType, forwardType, name); + + // at this point, we can finalize the type + forwardType->recomputeName(); + forwardType->finalizeType(); + forwardType->typeFinishedBeingDeserializedPhase1(); + forwardType->typeFinishedBeingDeserializedPhase2(); } else { throw std::runtime_error("Invalid nativeType: kind/fieldNumber error."); } @@ -1192,6 +1354,13 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b throw std::runtime_error("Invalid native type: no 'kind' field"); } + if (kind == 5) { + if (!forwardType) { + throw std::runtime_error("Invalid native type: forward body never given"); + } + return forwardType; + } + if (kind == 0) { if (category == -1) { throw std::runtime_error("Invalid inline type."); @@ -1275,7 +1444,7 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b ); } - Type* nativeType = PyInstance::extractTypeFrom(typeFromRep); + Type* nativeType = PyInstance::extractTypeFrom((PyTypeObject*)typeFromRep); if (!nativeType) { // the type representation could be a regular python class @@ -1383,28 +1552,29 @@ Instance PythonSerializationContext::deserializeNativeInstance(DeserializationBu return result; } -// if actuallyProduceResult == false, then we still need to walk over the objects to make sure -// that our memos are correctly populated, but we don't need to construct objects, which will prevent us -// from creating new objects and leaking them when deserializing classes we already hold. -Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuffer& b, size_t inWireType, bool actuallyProduceResult) const { +/* deserialize a native type of the apropriate category into an empty Type* that's been constructed */ +void PythonSerializationContext::deserializeNativeTypeIntoBlank( + DeserializationBuffer& b, + size_t inWireType, + Type* blankShell, + std::string inName +) const { PyEnsureGilAcquired acquireTheGil; int category = -1; - Type* namedType = nullptr; - std::vector types; std::vector names; int64_t whichIndex = -1; PyObjectHolder obj; Instance instance; - std::vector overloads; + std::vector overloads; Type* closureType = 0; int isEntrypoint = 0; int isNocompile = 0; - std::vector classBases; + std::vector classBases; bool classIsFinal = false; std::vector > alternativeMembers; std::map classMethods, classStatics, classClassMethods, classPropertyFunctions; @@ -1417,23 +1587,7 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff category = b.readUnsignedVarint(); } else if (wireType == WireType::BYTES) { - std::string objectName = b.readStringObject(); - - PyObjectStealer contextAsPyObj(PyInstance::extractPythonObject(mContextObj)); - - PyObjectStealer namedObj(PyObject_CallMethod(contextAsPyObj, "objectFromName", "s", objectName.c_str())); - - if (!namedObj) { - throw PythonExceptionSet(); - } - - if (!PyType_Check(namedObj)) { - throw std::runtime_error("Corrupt native type found: object named " + objectName + " was not a type."); - } - namedType = PyInstance::extractTypeFrom((PyTypeObject*)(PyObject*)namedObj); - if (!namedType) { - throw std::runtime_error("Corrupt native type found: object named " + objectName + " was not a native type."); - } + throw std::runtime_error("We shouldn't be using named serialization in an MRTG"); } else { throw std::runtime_error("Corrupt native type found: 0 field should be a category or a name"); } @@ -1441,17 +1595,23 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff if (category == -1) { throw std::runtime_error("Corrupt native type found."); } - if (category == Type::TypeCategory::catForward - || category == Type::TypeCategory::catBoundMethod - ) { + if (category == Type::TypeCategory::catBoundMethod) { if (fieldNumber == 1) { names.push_back(b.readStringObject()); } else if (fieldNumber == 2) { types.push_back(deserializeNativeType(b, wireType)); } } else - if (category == Type::TypeCategory::catAlternativeMatcher - ) { + if (category == Type::TypeCategory::catForward) { + if (fieldNumber == 1) { + names.push_back(b.readStringObject()); + } else if (fieldNumber == 2) { + types.push_back(deserializeNativeType(b, wireType)); + } else if (fieldNumber == 3) { + obj.steal(deserializePythonObject(b, wireType)); + } + } else + if (category == Type::TypeCategory::catAlternativeMatcher) { if (fieldNumber == 1) { types.push_back(deserializeNativeType(b, wireType)); } @@ -1466,6 +1626,13 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff deserializeAlternativeMembers(alternativeMembers, b, wireType); } else if (fieldNumber == 4) { deserializeClassFunDict(names.back(), classMethods, b, wireType); + } else if (fieldNumber == 5) { + b.consumeCompoundMessage(wireType, [&](int which, size_t wireType2) { + types.push_back(deserializeNativeType(b, wireType2)); + if (!types.back()->isConcreteAlternative()) { + throw std::runtime_error("Expected a concrete alternative"); + } + }); } } else if (category == Type::TypeCategory::catHeldClass) { @@ -1497,11 +1664,13 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff t->name() + " of category " + t->getTypeCategoryString() ); } - classBases.push_back(((HeldClass*)t)->getClassType()); + classBases.push_back((HeldClass*)t); }); } else if (fieldNumber == 8) { assertWireTypesEqual(wireType, WireType::VARINT); classIsFinal = b.readUnsignedVarint(); + } else if (fieldNumber == 10) { + types.push_back(deserializeNativeType(b, wireType)); } } else if (category == Type::TypeCategory::catFunction) { @@ -1522,7 +1691,7 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff } else { overloads.push_back( - Function::Overload::deserialize(*this, b, wireType) + FunctionOverload::deserialize(*this, b, wireType) ); } } else @@ -1537,7 +1706,6 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff category == Type::TypeCategory::catRefTo || category == Type::TypeCategory::catSubclassOf || category == Type::TypeCategory::catTuple || - (category == Type::TypeCategory::catPythonSubclass && fieldNumber == 1) || //named tuples alternate between strings for names and type values (category == Type::TypeCategory::catNamedTuple && (fieldNumber % 2 == 0)) || //alternatives encode one type exactly @@ -1548,8 +1716,6 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff else if (category == Type::TypeCategory::catNamedTuple) { assertWireTypesEqual(wireType, WireType::BYTES); names.push_back(b.readStringObject()); - } else if (category == Type::TypeCategory::catPythonSubclass && fieldNumber == 2) { - obj.steal(deserializePythonObject(b, wireType)); } else if (category == Type::TypeCategory::catPythonObjectOfType && fieldNumber == 1) { obj.steal(deserializePythonObject(b, wireType)); } else if (category == Type::TypeCategory::catValue && fieldNumber == 1) { @@ -1563,48 +1729,60 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff } }); - // if the type was just named in the codebase, return it - if (namedType) { - return namedType; + if (!blankShell) { + // we don't actually need to construct this thing + return; } - if (!actuallyProduceResult) { - return nullptr; - } - - // otherwise, construct it from the pieces we were given. - Type* resultType = nullptr; - if (category == Type::TypeCategory::catClass) { if (types.size() != 1) { throw std::runtime_error("Corrupt 'Class' encountered: can't have multiple corresponding HeldClass objects."); } - if (types[0]->getTypeCategory() != Type::TypeCategory::catHeldClass) { + if (!types[0]->isHeldClass()) { throw std::runtime_error( - "Corrupt 'Class' encountered: HeldClass is a " + - Type::categoryToString(types[0]->getTypeCategory()) + " not a HeldClass" + "Corrupt 'Class' encountered: HeldClass is a " + types[0]->getTypeCategoryString() + + " not a HeldClass" ); } - resultType = ((HeldClass*)types[0])->getClassType(); + if (!blankShell->isClass()) { + throw std::runtime_error("Shell is not a Class"); + } + + ((Class*)blankShell)->initializeDuringDeserialization(inName, (HeldClass*)types[0]); } else if (category == Type::TypeCategory::catAlternative) { if (names.size() != 2) { throw std::runtime_error("Corrupt 'Alternative' encountered: invalid number of names"); } - resultType = Alternative::Make( + if (!blankShell->isAlternative()) { + throw std::runtime_error("Shell is not an Alternative"); + } + + ((Alternative*)blankShell)->initializeDuringDeserialization( names[0], names[1], alternativeMembers, - classMethods + classMethods, + types ); } else if (category == Type::TypeCategory::catForward) { - if (names.size() != 1 || types.size() != 1) { + if (names.size() != 1 || types.size() > 1) { throw std::runtime_error("Corrupt Forward"); } - resultType = Forward::Make(names[0]); - ((Forward*)resultType)->define(types[0]); + if (!blankShell->isForward()) { + throw std::runtime_error("Shell is not a Forward"); + } + + ((Forward*)blankShell)->setName(names[0]); + + if (obj) { + ((Forward*)blankShell)->setCellOrDict(obj); + } + if (types.size()) { + ((Forward*)blankShell)->define(types[0]); + } } else if (category == Type::TypeCategory::catHeldClass) { if (names.size() != 1) { @@ -1616,9 +1794,29 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff classClassMembersRaw[nameAndObj.first] = incref((PyObject*)nameAndObj.second); } - resultType = Class::Make( - names[0], - classBases, + if (!blankShell->isHeldClass()) { + throw std::runtime_error("Shell is not a HeldClass"); + } + + if (types.size() != 1) { + throw std::runtime_error("Corrupt 'HeldClass' - needed a single type, not " + format(types.size())); + } + + if (!types[0]->isClass()) { + throw std::runtime_error("Corrupt 'HeldClass' - needed a Class, not " + types[0]->getTypeCategoryString()); + } + + std::vector heldBases; + for (auto b: classBases) { + if (!b->isHeldClass()) { + throw std::runtime_error("Non-held base encountered: its " + b->getTypeCategoryString()); + } + heldBases.push_back((HeldClass*)b); + } + + ((HeldClass*)blankShell)->initializeDuringDeserialization( + "Held(" + names[0] + ")", + heldBases, classIsFinal, classMembers, classMethods, @@ -1626,16 +1824,19 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff classPropertyFunctions, classClassMembersRaw, classClassMethods, - // the methodOf should already be set in the class - false - )->getHeldClass(); + (Class*)types[0] + ); } else if (category == Type::TypeCategory::catFunction) { if (names.size() != 3) { throw std::runtime_error("Badly structured 'Function' encountered."); } - resultType = Function::Make( + if (!blankShell->isFunction()) { + throw std::runtime_error("Shell is not a Function"); + } + + ((Function*)blankShell)->initializeDuringDeserialization( names[0], names[1], names[2], @@ -1645,83 +1846,50 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff isNocompile ); } - else if (category == Type::TypeCategory::catUInt8) { - resultType = ::UInt8::Make(); - } - else if (category == Type::TypeCategory::catUInt16) { - resultType = ::UInt16::Make(); - } - else if (category == Type::TypeCategory::catUInt32) { - resultType = ::UInt32::Make(); - } - else if (category == Type::TypeCategory::catUInt64) { - resultType = ::UInt64::Make(); - } - else if (category == Type::TypeCategory::catInt8) { - resultType = ::Int8::Make(); - } - else if (category == Type::TypeCategory::catInt16) { - resultType = ::Int16::Make(); - } - else if (category == Type::TypeCategory::catInt32) { - resultType = ::Int32::Make(); - } - else if (category == Type::TypeCategory::catInt64) { - resultType = ::Int64::Make(); - } - else if (category == Type::TypeCategory::catEmbeddedMessage) { - resultType = ::EmbeddedMessageType::Make(); - } - else if (category == Type::TypeCategory::catFloat32) { - resultType = ::Float32::Make(); - } - else if (category == Type::TypeCategory::catFloat64) { - resultType = ::Float64::Make(); - } - else if (category == Type::TypeCategory::catNone) { - resultType = ::NoneType::Make(); - } - else if (category == Type::TypeCategory::catBytes) { - resultType = ::BytesType::Make(); - } - else if (category == Type::TypeCategory::catString) { - resultType = ::StringType::Make(); - } - else if (category == Type::TypeCategory::catPyCell) { - resultType = ::PyCellType::Make(); - } - else if (category == Type::TypeCategory::catBool) { - resultType = ::Bool::Make(); - } - else if (category == Type::TypeCategory::catOneOf) { - resultType = ::OneOfType::Make(types); - } - else if (category == Type::TypeCategory::catValue) { - resultType = ::Value::Make(instance); - } else if (category == Type::TypeCategory::catTupleOf) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: TupleOf needs exactly 1 type."); } - resultType = ::TupleOfType::Make(types[0]); + + if (!blankShell->isTupleOf()) { + throw std::runtime_error("Shell is not a TupleOf"); + } + + ((TupleOfType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catListOf) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: ListOf needs exactly 1 type."); } - resultType = ::ListOfType::Make(types[0]); + + if (!blankShell->isListOf()) { + throw std::runtime_error("Shell is not a ListOf"); + } + + ((ListOfType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catTypedCell) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: TypedCell needs exactly 1 type."); } - resultType = ::TypedCellType::Make(types[0]); + + if (!blankShell->isTypedCell()) { + throw std::runtime_error("Shell is not a TypedCell"); + } + + ((TypedCellType*)blankShell)->initializeDuringDeserialization(types[0]); + } else if (category == Type::TypeCategory::catPointerTo) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: PointerTo needs exactly 1 type."); } - resultType = ::PointerTo::Make(types[0]); + + if (!blankShell->isPointerTo()) { + throw std::runtime_error("Shell is not a PointerTo"); + } + + ((PointerTo*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catBoundMethod) { if (types.size() != 1) { @@ -1730,67 +1898,121 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff if (names.size() != 1) { throw std::runtime_error("Invalid native type: BoundMethod needs exactly 1 name."); } - resultType = ::BoundMethod::Make(types[0], names[0]); + + if (!blankShell->isBoundMethod()) { + throw std::runtime_error("Shell is not a BoundMethod"); + } + + ((BoundMethod*)blankShell)->initializeDuringDeserialization(names[0], types[0]); } else if (category == Type::TypeCategory::catAlternativeMatcher) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: AternativeMatcher needs exactly 1 type."); } - resultType = ::AlternativeMatcher::Make(types[0]); + + if (!blankShell->isAlternativeMatcher()) { + throw std::runtime_error("Shell is not a AlternativeMatcher"); + } + + ((AlternativeMatcher*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catSubclassOf) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: SubclassOf needs exactly 1 type."); } - resultType = ::SubclassOfType::Make(types[0]); + + if (!blankShell->isSubclassOf()) { + throw std::runtime_error("Shell is not a SubclassOf"); + } + + ((SubclassOfType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catRefTo) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: RefTo needs exactly 1 type."); } - resultType = ::RefTo::Make(types[0]); + + if (!blankShell->isRefTo()) { + throw std::runtime_error("Shell is not a RefTo"); + } + + ((RefTo*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catNamedTuple) { - resultType = ::NamedTuple::Make(types, names); + if (!blankShell->isNamedTuple()) { + throw std::runtime_error("Shell is not a NamedTuple"); + } + + ((NamedTuple*)blankShell)->initializeDuringDeserialization(types, names); } else if (category == Type::TypeCategory::catTuple) { - resultType = ::Tuple::Make(types); + if (!blankShell->isTuple()) { + throw std::runtime_error("Shell is not a Tuple"); + } + + ((Tuple*)blankShell)->initializeDuringDeserialization(types); } else if (category == Type::TypeCategory::catSet) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: Set needs exactly 1 type."); } - resultType = ::SetType::Make(types[0]); + + if (!blankShell->isSet()) { + throw std::runtime_error("Shell is not a Set"); + } + + ((SetType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catDict) { if (types.size() != 2) { throw std::runtime_error("Invalid native type: Dict needs exactly 2 types."); } - resultType = ::DictType::Make(types[0], types[1]); + + if (!blankShell->isDict()) { + throw std::runtime_error("Shell is not a Dict"); + } + + ((DictType*)blankShell)->initializeDuringDeserialization(types[0], types[1]); } else if (category == Type::TypeCategory::catConstDict) { if (types.size() != 2) { throw std::runtime_error("Invalid native type: ConstDict needs exactly 2 types."); } - resultType = ::ConstDictType::Make(types[0], types[1]); - } - else if (category == Type::TypeCategory::catPythonSubclass) { - if (!obj || !PyType_Check(obj)) { - throw std::runtime_error("Invalid native type: PythonSubclass needs a python type."); - } - if (types.size() != 1) { - throw std::runtime_error("Invalid native type: PythonSubclass needs a native type."); + if (!blankShell->isConstDict()) { + throw std::runtime_error("Shell is not a ConstDict"); } - resultType = ::PythonSubclass::Make(types[0], (PyTypeObject*)(PyObject*)obj); + ((ConstDictType*)blankShell)->initializeDuringDeserialization(types[0], types[1]); } else if (category == Type::TypeCategory::catPythonObjectOfType) { if (!obj || !PyType_Check(obj)) { throw std::runtime_error("Invalid native type: PythonObjectOfType needs a python type."); } - resultType = ::PythonObjectOfType::Make((PyTypeObject*)(PyObject*)obj); + if (!blankShell->isPythonObjectOfType()) { + throw std::runtime_error("Shell is not a PythonObjectOfType"); + } + + ((PythonObjectOfType*)blankShell)->initializeDuringDeserialization( + (PyTypeObject*)(PyObject*)obj + ); + } + else if (category == Type::TypeCategory::catValue) { + if (!blankShell->isValue()) { + throw std::runtime_error("Shell is not a Value"); + } + + ((Value*)blankShell)->initializeDuringDeserialization(instance); + } + else if (category == Type::TypeCategory::catOneOf) { + if (!blankShell->isOneOf()) { + throw std::runtime_error("Shell is not a OneOf"); + } + + ((OneOfType*)blankShell)->initializeDuringDeserialization( + types + ); } else if (category == Type::TypeCategory::catConcreteAlternative) { if (types.size() != 1) { @@ -1810,16 +2032,16 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff throw std::runtime_error("corrupt data: invalid alternative specified"); } - resultType = a->concreteSubtype(whichIndex); - } else { - throw std::runtime_error("Invalid native type category"); - } + if (!blankShell->isConcreteAlternative()) { + throw std::runtime_error("Shell is not a ConcreteAlternative"); + } - if (!resultType) { - throw std::runtime_error("Corrupt nativeType."); + ((ConcreteAlternative*)blankShell)->initializeDuringDeserialization(whichIndex, a); + } else { + throw std::runtime_error( + "Cant initialize a blank instance of " + blankShell->getTypeCategoryString() + ); } - - return resultType; } template diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 34282de78..87e5db1e7 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -41,7 +41,10 @@ void PythonSerializationContext::serializePythonObject(PyObject* o, Serializatio // because otherwise we'd create a dependency during serialization on _whether_ the // group was already created, which would introduce a nondeterminism into serilization // based on the compiler state - auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor(o); + auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor( + o, + VisibilityType::Identity + ); // if we have a group, check whether we've already memoized this group in the stream // this prevents us from initiating the serialization of a new group here. @@ -248,8 +251,14 @@ void PythonSerializationContext::serializePythonObjectByNameOrRepresentation(PyO if (PyType_Check(o)) { // if its a Type object, this is the place to put it into a mutually recursive type // group - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(o); - auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor(o); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + o, + VisibilityType::Identity + ); + auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor( + o, + VisibilityType::Identity + ); b.writeBeginCompound(FieldNumbers::RECURSIVE_OBJECT); serializeMutuallyRecursiveTypeGroup(groupAndIndex.first, b, 0); @@ -337,6 +346,34 @@ bool isSimpleType(Type* t) { void PythonSerializationContext::serializeNativeType(Type* nativeType, SerializationBuffer& b, size_t fieldNumber) const { PyEnsureGilAcquired getTheGil; + if (nativeType->isForwardDefined() || nativeType->isForward()) { + // forward types are neither stateful nor interned. So, we can serialize them + // with a memo and not worry about needing to intern them at any point. + b.writeBeginCompound(fieldNumber); + + // the 5 indicates that this is a forward native type. + b.writeUnsignedVarintObject(0, 5); + + uint32_t id; + bool isNew; + std::tie(id, isNew) = b.cachePointer(nativeType, nullptr); + + // write the memo first + b.writeUnsignedVarintObject(1, id); + + if (isNew) { + // then write the actual object if it's the first + // time we're writing it. note we write the type category first + // so that we can produce a blank instance before proceeding into + // the recursion + b.writeUnsignedVarintObject(2, nativeType->getTypeCategory()); + b.writeStringObject(3, nativeType->name()); + serializeNativeTypeInner(nativeType, b, 4, false); + } + + b.writeEndCompound(); + return; + } # if TP_VERBOSE_SERIALIZE ScopedIndenter s; @@ -353,7 +390,7 @@ void PythonSerializationContext::serializeNativeType(Type* nativeType, Serializa b.writeUnsignedVarintObject(0, 0); b.writeUnsignedVarintObject(1, nativeType->getTypeCategory()); } else - if (nativeType->getTypeCategory() == Type::TypeCategory::catConcreteAlternative && + if (nativeType->isConcreteAlternative() && (name = getNameForPyObj((PyObject*)PyInstance::typeObj(nativeType->getBaseType()))).size()) { // this is an inline named concrete alternative b.writeUnsignedVarintObject(0, 1); @@ -393,6 +430,10 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec throw std::runtime_error("Can't serialize a mutually recursive type group with an invalid hash."); } + if (group->visibilityType() != VisibilityType::Identity) { + throw std::runtime_error("shouldn't be serializing a compiler identity group."); + } + b.writeBeginCompound(fieldNumber); // group memo uint32_t id; @@ -413,7 +454,8 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec # if TP_VERBOSE_SERIALIZE ScopedIndenter s; - std::cout << s.prefix() << "Serializing fresh MRTG: " << group->hash().digestAsHexString() << "\n"; + std::cout << s.prefix() << "Serializing fresh MRTG: " << group->hash().digestAsHexString() + << " with " << group->getIndexToObject().size() << " items" << "\n"; # endif // group hash if (shouldSerializeHashSequence()) { @@ -700,7 +742,7 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec } } else { Type* toSerialize = obj.type(); - serializeNativeTypeInner(toSerialize, b, index); + serializeNativeTypeInner(toSerialize, b, index, true); } }; @@ -794,28 +836,7 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec b.writeEndCompound(); -# if TP_VERBOSE_SERIALIZE - std::cout << s.prefix() << "Serializing used globals\n"; -# endif - b.writeBeginCompound(4); - - for (auto& indexAndObj: group->getIndexToObject()) { - if (indexAndObj.second.type() && indexAndObj.second.type()->isFunction()) { - // we need to serialize the 'globals' dict of each function object - Function* f = (Function*)indexAndObj.second.type(); - - b.writeBeginCompound(indexAndObj.first); - - for (long k = 0; k < f->getOverloads().size(); k++) { - PyObjectStealer globals(f->getOverloads()[k].getUsedGlobals()); - serializePythonObject(globals, b, k); - } - - b.writeEndCompound(); - } - } - b.writeEndCompound(); b.writeEndCompound(); @@ -824,7 +845,8 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec void PythonSerializationContext::serializeNativeTypeInner( Type* nativeType, SerializationBuffer& b, - size_t fieldNumber + size_t fieldNumber, + bool checkIfObjectIsNamed ) const { b.writeBeginCompound(fieldNumber); @@ -840,75 +862,81 @@ void PythonSerializationContext::serializeNativeTypeInner( << nativeType->name() << " of cat " << nativeType->getTypeCategoryString() << "\n"; # endif - PyEnsureGilAcquired acquireTheGil; + if (checkIfObjectIsNamed) { + PyEnsureGilAcquired acquireTheGil; - std::string nameForObject = getNameForPyObj((PyObject*)PyInstance::typeObj(nativeType)); + std::string nameForObject = getNameForPyObj((PyObject*)PyInstance::typeObj(nativeType)); - if (nameForObject.size()) { - b.writeStringObject(0, nameForObject); - b.writeEndCompound(); - return; + if (nameForObject.size()) { + b.writeStringObject(0, nameForObject); + b.writeEndCompound(); + return; + } } b.writeUnsignedVarintObject(0, nativeType->getTypeCategory()); - if (nativeType->getTypeCategory() == Type::TypeCategory::catConcreteAlternative) { + if (nativeType->isConcreteAlternative()) { serializeNativeType(nativeType->getBaseType(), b, 1); b.writeUnsignedVarintObject(2, ((ConcreteAlternative*)nativeType)->which()); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catAlternative) { + } else if (nativeType->isAlternative()) { Alternative* altType = (Alternative*)nativeType; b.writeStringObject(1, altType->name()); b.writeStringObject(2, altType->moduleName()); serializeAlternativeMembers(altType->subtypes(), b, 3); serializeClassFunDict(altType->getMethods(), b, 4); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catSet) { + + b.writeBeginCompound(5); + for (long k = 0; k < altType->subtypes().size(); k++) { + serializeNativeType(altType->concreteSubtype(k), b, 1); + } + b.writeEndCompound(); + + } else if (nativeType->isSet()) { serializeNativeType(((SetType*)nativeType)->keyType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catConstDict) { + } else if (nativeType->isConstDict()) { serializeNativeType(((ConstDictType*)nativeType)->keyType(), b, 1); serializeNativeType(((ConstDictType*)nativeType)->valueType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catDict) { + } else if (nativeType->isDict()) { serializeNativeType(((DictType*)nativeType)->keyType(), b, 1); serializeNativeType(((DictType*)nativeType)->valueType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPythonSubclass) { - serializeNativeType(((PythonSubclass*)nativeType)->baseType(), b, 1); - serializePythonObject((PyObject*)((PythonSubclass*)nativeType)->pyType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTupleOf) { + } else if (nativeType->isTupleOf()) { serializeNativeType(((TupleOfType*)nativeType)->getEltType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPointerTo) { + } else if (nativeType->isPointerTo()) { serializeNativeType(((PointerTo*)nativeType)->getEltType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catRefTo) { + } else if (nativeType->isRefTo()) { serializeNativeType(((RefTo*)nativeType)->getEltType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catSubclassOf) { + } else if (nativeType->isSubclassOf()) { serializeNativeType(((SubclassOfType*)nativeType)->getSubclassOf(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catBoundMethod) { + } else if (nativeType->isBoundMethod()) { b.writeStringObject(1, ((BoundMethod*)nativeType)->getFuncName()); serializeNativeType(((BoundMethod*)nativeType)->getFirstArgType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catAlternativeMatcher) { + } else if (nativeType->isAlternativeMatcher()) { serializeNativeType(((AlternativeMatcher*)nativeType)->getAlternative(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catListOf) { + } else if (nativeType->isListOf()) { serializeNativeType(((ListOfType*)nativeType)->getEltType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTypedCell) { + } else if (nativeType->isTypedCell()) { serializeNativeType(((TypedCellType*)nativeType)->getHeldType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTuple) { + } else if (nativeType->isTuple()) { size_t index = 1; for (auto t: ((CompositeType*)nativeType)->getTypes()) { serializeNativeType(t, b, index++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catNamedTuple) { + } else if (nativeType->isNamedTuple()) { size_t index = 1; for (long k = 0; k < ((CompositeType*)nativeType)->getTypes().size(); k++) { b.writeStringObject(index++, ((CompositeType*)nativeType)->getNames()[k]); serializeNativeType(((CompositeType*)nativeType)->getTypes()[k], b, index++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catOneOf) { + } else if (nativeType->isOneOf()) { size_t index = 1; for (auto t: ((OneOfType*)nativeType)->getTypes()) { serializeNativeType(t, b, index++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPythonObjectOfType) { + } else if (nativeType->isPythonObjectOfType()) { serializePythonObject((PyObject*)((PythonObjectOfType*)nativeType)->pyType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catValue) { + } else if (nativeType->isValue()) { b.writeBeginCompound(1); Instance i = ((Value*)nativeType)->value(); @@ -917,7 +945,7 @@ void PythonSerializationContext::serializeNativeTypeInner( PyEnsureGilReleased releaseTheGil; i.type()->serialize(i.data(), b, 1); b.writeEndCompound(); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catFunction) { + } else if (nativeType->isFunction()) { Function* ftype = (Function*)nativeType; serializeNativeType(ftype->getClosureType(), b, 1); @@ -938,15 +966,18 @@ void PythonSerializationContext::serializeNativeTypeInner( for (auto& overload: ftype->getOverloads()) { overload.serialize(*this, b, whichIndex++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catForward) { + } else if (nativeType->isForward()) { b.writeStringObject(1, nativeType->name()); if (((Forward*)nativeType)->getTarget()) { serializeNativeType(((Forward*)nativeType)->getTarget(), b, 2); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catClass) { + if (((Forward*)nativeType)->getCellOrDict()) { + serializePythonObject(((Forward*)nativeType)->getCellOrDict(), b, 3); + } + } else if (nativeType->isClass()) { serializeNativeType(((Class*)nativeType)->getHeldClass(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catHeldClass) { + } else if (nativeType->isHeldClass()) { HeldClass* cls = (HeldClass*)nativeType; // note that we serialize the name of the class (not the held class) @@ -977,6 +1008,8 @@ void PythonSerializationContext::serializeNativeTypeInner( b.writeEndCompound(); b.writeUnsignedVarintObject(8, cls->isFinal() ? 1 : 0); + + serializeNativeType(cls->getClassType(), b, 10); } else { throw std::runtime_error( "Can't serialize native type " + nativeType->name() diff --git a/typed_python/PythonSubclassType.cpp b/typed_python/PythonSubclassType.cpp deleted file mode 100644 index 2929e3467..000000000 --- a/typed_python/PythonSubclassType.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/****************************************************************************** - Copyright 2017-2019 typed_python Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -#include "AllTypes.hpp" - -bool PythonSubclass::isBinaryCompatibleWithConcrete(Type* other) { - Type* nonPyBase = m_base; - while (nonPyBase->getTypeCategory() == TypeCategory::catPythonSubclass) { - nonPyBase = nonPyBase->getBaseType(); - } - - Type* otherNonPyBase = other; - while (otherNonPyBase->getTypeCategory() == TypeCategory::catPythonSubclass) { - otherNonPyBase = otherNonPyBase->getBaseType(); - } - - return nonPyBase->isBinaryCompatibleWith(otherNonPyBase); -} - -// static -PythonSubclass* PythonSubclass::Make(Type* base, PyTypeObject* pyType) { - PyEnsureGilAcquired getTheGil; - - static std::map m; - - auto it = m.find(pyType); - - if (it == m.end()) { - it = m.insert( - std::make_pair(pyType, new PythonSubclass(base, pyType)) - ).first; - } - - if (it->second->getBaseType() != base) { - throw std::runtime_error( - "Expected to find the same base type. Got " - + it->second->getBaseType()->name() + " != " + base->name() - ); - } - - return it->second; -} - - -typed_python_hash_type PythonSubclass::hash(instance_ptr left) { - if (m_hashFun) { - PyEnsureGilAcquired acquireTheGil; - - PyObjectStealer selfAsObject(PyInstance::initialize(this, [&](instance_ptr selfData) { - this->copy_constructor(selfData, left); - })); - - PyObjectStealer result( - PyObject_CallFunctionObjArgs( - m_hashFun, - //the typecast is necessary since this is a varargs function, and so the - //C-style calling convention doesn't know to transform the PyObjectStealer - //into a PyObject* - (PyObject*)selfAsObject, - NULL - ) - ); - - if (!result) { - throw PythonExceptionSet(); - } - - if (!PyLong_Check(result)) { - throw std::runtime_error("Expected " + this->name() + ".__hash__ to return an integer"); - } - - return PyLong_AsLong(result); - } - - return m_base->hash(left); -} - -void PythonSubclass::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { - if (!isStr && m_reprFun) { - PyEnsureGilAcquired acquireTheGil; - - PyObjectStealer selfAsObject(PyInstance::initialize(this, [&](instance_ptr selfData) { - this->copy_constructor(selfData, self); - })); - - PyObjectStealer result( - PyObject_CallFunctionObjArgs( - m_reprFun, - //the typecast is necessary since this is a varargs function, and so the - //C-style calling convention doesn't know to transform the PyObjectStealer - //into a PyObject* - (PyObject*)selfAsObject, - NULL - ) - ); - - if (!result) { - throw PythonExceptionSet(); - } - - if (!PyUnicode_Check(result)) { - throw std::runtime_error("Expected " + this->name() + ".__repr__ to return a string"); - } - - stream << PyUnicode_AsUTF8(result); - return; - } - - m_base->repr(self, stream, isStr); -} diff --git a/typed_python/PythonSubclassType.hpp b/typed_python/PythonSubclassType.hpp deleted file mode 100644 index 5e229a056..000000000 --- a/typed_python/PythonSubclassType.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/****************************************************************************** - Copyright 2017-2019 typed_python Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -#pragma once - -#include "Type.hpp" - -class PythonSubclass : public Type { -public: - PythonSubclass(Type* base, PyTypeObject* typePtr) : - Type(TypeCategory::catPythonSubclass) - { - m_base = base; - mTypeRep = (PyTypeObject*)incref((PyObject*)typePtr); - m_name = typePtr->tp_name; - - m_reprFun = PyObject_GetAttrString((PyObject*)typePtr, "__repr__"); - - if (!m_reprFun) { - PyErr_Clear(); - } - - // make sure we don't get the __repr__ from our own implementation - // in the base class. We want to get the subclass' version. - if (!PyFunction_Check(m_reprFun)) { - m_reprFun = nullptr; - } - - m_hashFun = PyObject_GetAttrString((PyObject*)typePtr, "__hash__"); - - if (!m_hashFun) { - PyErr_Clear(); - } - - if (!PyFunction_Check(m_hashFun)) { - m_hashFun = nullptr; - } - - endOfConstructorInitialization(); // finish initializing the type object. - } - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitHash(ShaHash(1, m_typeCategory)); - v.visitName(m_name); - v.visitTopo(m_base); - v.visitTopo((PyObject*)mTypeRep); - if (mTypeRep->tp_dict) { - v.visitHash(ShaHash(1)); - v.visitDict((PyObject*)mTypeRep->tp_dict); - } else { - v.visitHash(ShaHash(0)); - } - } - - bool isBinaryCompatibleWithConcrete(Type* other); - - template - void _visitContainedTypes(const visitor_type& visitor) { - visitor(m_base); - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - visitor(m_base); - } - - bool _updateAfterForwardTypesChanged() { - size_t size = m_base->bytecount(); - bool is_default_constructible = m_base->is_default_constructible(); - - bool anyChanged = ( - size != m_size || - m_is_default_constructible != is_default_constructible - ); - - m_size = size; - m_is_default_constructible = is_default_constructible; - - return anyChanged; - } - - typed_python_hash_type hash(instance_ptr left); - - template - void serialize(instance_ptr self, buf_t& buffer, size_t fieldNumber) { - m_base->serialize(self, buffer, fieldNumber); - } - - void deepcopyConcrete( - instance_ptr dest, - instance_ptr src, - DeepcopyContext& context - ) { - m_base->deepcopy(dest, src, context); - } - - template - void visitMROConcrete(const visitor_type& v) { - v(this); - m_base->visitMROConcrete(v); - } - - size_t deepBytecountConcrete(instance_ptr instance, std::unordered_set& alreadyVisited, std::set* outSlabs) { - return m_base->deepBytecount(instance, alreadyVisited, outSlabs); - } - - template - void deserialize(instance_ptr self, buf_t& buffer, size_t wireType) { - m_base->deserialize(self, buffer, wireType); - } - - void repr(instance_ptr self, ReprAccumulator& stream, bool isStr); - - bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { - return m_base->cmp(left, right, pyComparisonOp, suppressExceptions); - } - - bool isPODConcrete() { - return m_base->isPOD(); - } - - void constructor(instance_ptr self) { - m_base->constructor(self); - } - - void destroy(instance_ptr self) { - m_base->destroy(self); - } - - void copy_constructor(instance_ptr self, instance_ptr other) { - m_base->copy_constructor(self, other); - } - - void assign(instance_ptr self, instance_ptr other) { - m_base->assign(self, other); - } - - static PythonSubclass* Make(Type* base, PyTypeObject* pyType); - - Type* baseType() const { - return m_base; - } - - PyTypeObject* pyType() const { - return mTypeRep; - } - -private: - PyObject* m_reprFun; - - PyObject* m_hashFun; -}; diff --git a/typed_python/PythonTypeInternals.hpp b/typed_python/PythonTypeInternals.hpp new file mode 100644 index 000000000..09a242289 --- /dev/null +++ b/typed_python/PythonTypeInternals.hpp @@ -0,0 +1,41 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include + +// the definition of a python property which we can use to read and write its internals +// despite it being an internal detail. +typedef struct { + PyObject_HEAD + PyObject *prop_get; + PyObject *prop_set; + PyObject *prop_del; + PyObject *prop_doc; + + #if PY_MINOR_VERSION >= 10 + PyObject *prop_name; + #endif + + int getter_doc; +} JustLikeAPropertyObject; + +typedef struct { + PyObject_HEAD + PyObject *cm_callable; + PyObject *cm_dict; +} JustLikeAClassOrStaticmethod; diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index f808a856b..2442e266d 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -19,18 +19,22 @@ #include "Type.hpp" class RefTo : public Type { -protected: +public: typedef void* instance; -public: + // construct a non-forward defined refto + RefTo() : Type(TypeCategory::catRefTo) + { + m_size = sizeof(instance); + m_is_default_constructible = true; + } + RefTo(Type* t) : Type(TypeCategory::catRefTo), m_element_type(t) { - m_size = sizeof(instance); - m_is_default_constructible = false; - - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; + recomputeName(); } template @@ -39,45 +43,58 @@ class RefTo : public Type { v.visitTopo(m_element_type); } - void _updateTypeMemosAfterForwardResolution() { - RefTo::Make(m_element_type, this); - } - + static RefTo* Make(Type* elt) { + if (elt->isForwardDefined()) { + return new RefTo(elt); + } - static RefTo* Make(Type* elt, RefTo* knownType = nullptr) { PyEnsureGilAcquired getTheGil; - if (elt->getTypeCategory() != Type::TypeCategory::catHeldClass) { - throw std::runtime_error("RefTo only valid on HeldClass types"); + static std::map memo; + + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - static std::map m; + RefTo* res = new RefTo(elt); + RefTo* concrete = (RefTo*)res->forwardResolvesTo(); - auto it = m.find(elt); - if (it == m.end()) { - it = m.insert(std::make_pair(elt, knownType ? knownType : new RefTo(elt))).first; - } + memo[elt] = concrete; + return concrete; + } - return it->second; + const char* docConcrete() { + return "RefTo is deprecated."; + } + + void initializeDuringDeserialization(Type* eltType) { + m_element_type = eltType; } bool isPODConcrete() { return true; } - bool _updateAfterForwardTypesChanged() { - std::string name = "RefTo(" + m_element_type->name(true) + ")"; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "RefTo(" + m_element_type->computeRecursiveName(typeStack) + ")"; + } - if (m_is_recursive_forward) { - name = m_recursive_name; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_element_type = ((RefTo*)forwardDefinitionOfSelf)->m_element_type; + } - bool anyChanged = name != m_name; + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_element_type, groupMap); + } - m_name = name; - m_stripped_name = ""; + Type* cloneForForwardResolutionConcrete() { + return new RefTo(); + } - return anyChanged; + void postInitializeConcrete() { + m_size = sizeof(instance); + m_is_default_constructible = false; } template diff --git a/typed_python/RegisterTypes.hpp b/typed_python/RegisterTypes.hpp index a30f26e51..bff20161e 100644 --- a/typed_python/RegisterTypes.hpp +++ b/typed_python/RegisterTypes.hpp @@ -191,24 +191,12 @@ class RegisterType : public Type { { m_size = sizeof(T); m_is_default_constructible = true; - - endOfConstructorInitialization(); // finish initializing the type object. - } - - bool isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; } bool isPODConcrete() { return true; } - bool _updateAfterForwardTypesChanged() { return false; } - template void _visitReferencedTypes(const visitor_type& v) {} @@ -281,6 +269,9 @@ class RegisterType : public Type { ) { copy_constructor(dest, src); } + + // no op + void postInitializeConcrete() {} }; @@ -292,7 +283,10 @@ class Bool : public RegisterType { Bool() : RegisterType(TypeCategory::catBool) { m_name = "bool"; - m_doc = Bool_doc; + } + + const char* docConcrete() { + return Bool_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -313,7 +307,10 @@ class UInt8 : public RegisterType { UInt8() : RegisterType(TypeCategory::catUInt8) { m_name = "UInt8"; - m_doc = UInt8_doc; + } + + const char* docConcrete() { + return UInt8_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -334,7 +331,10 @@ class UInt16 : public RegisterType { UInt16() : RegisterType(TypeCategory::catUInt16) { m_name = "UInt16"; - m_doc = UInt16_doc; + } + + const char* docConcrete() { + return UInt16_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -352,7 +352,10 @@ class UInt32 : public RegisterType { UInt32() : RegisterType(TypeCategory::catUInt32) { m_name = "UInt32"; - m_doc = UInt32_doc; + } + + const char* docConcrete() { + return UInt32_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -370,7 +373,10 @@ class UInt64 : public RegisterType { UInt64() : RegisterType(TypeCategory::catUInt64) { m_name = "UInt64"; - m_doc = UInt64_doc; + } + + const char* docConcrete() { + return UInt64_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -387,9 +393,12 @@ class Int8 : public RegisterType { public: Int8() : RegisterType(TypeCategory::catInt8) { - m_name = "Int8"; - m_doc = Int8_doc; m_size = 1; // Why only specify this here and not in other specializations? + m_name = "Int8"; + } + + const char* docConcrete() { + return Int8_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -407,7 +416,10 @@ class Int16 : public RegisterType { Int16() : RegisterType(TypeCategory::catInt16) { m_name = "Int16"; - m_doc = Int16_doc; + } + + const char* docConcrete() { + return Int16_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -425,7 +437,10 @@ class Int32 : public RegisterType { Int32() : RegisterType(TypeCategory::catInt32) { m_name = "Int32"; - m_doc = Int32_doc; + } + + const char* docConcrete() { + return Int32_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -443,7 +458,10 @@ class Int64 : public RegisterType { Int64() : RegisterType(TypeCategory::catInt64) { m_name = "int"; - m_doc = Int64_doc; + } + + const char* docConcrete() { + return Int64_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -461,7 +479,10 @@ class Float32 : public RegisterType { Float32() : RegisterType(TypeCategory::catFloat32) { m_name = "Float32"; - m_doc = Float32_doc; + } + + const char* docConcrete() { + return Float32_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -479,7 +500,10 @@ class Float64 : public RegisterType { Float64() : RegisterType(TypeCategory::catFloat64) { m_name = "float"; - m_doc = Float64_doc; + } + + const char* docConcrete() { + return Float64_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/SetType.cpp b/typed_python/SetType.cpp index 6671e8fe3..d5fb560e0 100644 --- a/typed_python/SetType.cpp +++ b/typed_python/SetType.cpp @@ -3,18 +3,26 @@ #include -SetType* SetType::Make(Type* eltype, SetType* knownType) { - PyEnsureGilAcquired getTheGil; +SetType* SetType::Make(Type* eltype) { + if (eltype->isForwardDefined()) { + return new SetType(eltype); + } - static std::map m; + PyEnsureGilAcquired getTheGil; - auto it = m.find(eltype); + static std::map memo; - if (it == m.end()) { - it = m.insert(std::make_pair(eltype, knownType ? knownType : new SetType(eltype))).first; + auto it = memo.find(eltype); + if (it != memo.end()) { + return it->second; } - return it->second; + SetType* res = new SetType(eltype); + SetType* concrete = (SetType*)res->forwardResolvesTo(); + + memo[eltype] = concrete; + + return concrete; } bool SetType::discard(instance_ptr self, instance_ptr key) { @@ -65,22 +73,6 @@ int64_t SetType::size(instance_ptr self) const { return record.hash_table_count; } -bool SetType::_updateAfterForwardTypesChanged() { - std::string name = "Set(" + m_key_type->name(true) + ")"; - m_size = sizeof(void*); - m_is_default_constructible = true; - m_bytes_per_el = m_key_type->bytecount(); - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool anyChanged = name != m_name; - m_name = name; - m_stripped_name = ""; - return anyChanged; -} - int64_t SetType::refcount(instance_ptr self) const { hash_table_layout& record = **(hash_table_layout**)self; return record.refcount; diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index f8e73e88c..a275f5ac5 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -27,13 +27,25 @@ PyDoc_STRVAR(Set_doc, ); class SetType : public Type { - public: +public: + SetType() : Type(TypeCategory::catSet) + { + } + SetType(Type* eltype) : Type(TypeCategory::catSet) , m_key_type(eltype) { - m_doc = Set_doc; - endOfConstructorInitialization(); + m_is_forward_defined = true; + recomputeName(); + } + + const char* docConcrete() { + return Set_doc; + } + + void initializeDuringDeserialization(Type* keyType) { + m_key_type = keyType; } template @@ -41,16 +53,39 @@ class SetType : public Type { visitor(m_key_type); } - void _updateTypeMemosAfterForwardResolution() { - SetType::Make(m_key_type, this); - } - template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); v.visitTopo(m_key_type); } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "Set(" + + m_key_type->computeRecursiveName(typeStack) + + ")"; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_key_type = ((SetType*)forwardDefinitionOfSelf)->m_key_type; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_key_type, groupMap); + } + + Type* cloneForForwardResolutionConcrete() { + return new SetType(); + } + + void postInitializeConcrete() { + m_size = sizeof(void*); + m_is_default_constructible = true; + } + + void finalizeTypeConcrete() { + m_bytes_per_el = m_key_type->bytecount(); + } + instance_ptr insertKey(instance_ptr self, instance_ptr key); instance_ptr lookupKey(instance_ptr self, instance_ptr key) const; bool discard(instance_ptr self, instance_ptr key); @@ -60,7 +95,6 @@ class SetType : public Type { void copy_constructor(instance_ptr self, instance_ptr other); void repr(instance_ptr self, ReprAccumulator& stream, bool isStr); int64_t refcount(instance_ptr self) const; - bool _updateAfterForwardTypesChanged(); int64_t size(instance_ptr self) const; bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions = false); @@ -87,7 +121,7 @@ class SetType : public Type { } } -void deepcopyConcrete( + void deepcopyConcrete( instance_ptr dest, instance_ptr src, DeepcopyContext& context @@ -233,10 +267,10 @@ void deepcopyConcrete( } } - public: - static SetType* Make(Type* eltype, SetType* knownType=nullptr); +public: + static SetType* Make(Type* eltype); - private: +private: Type* m_key_type; size_t m_bytes_per_el; bool subset(instance_ptr left, instance_ptr right); diff --git a/typed_python/ShaHash.hpp b/typed_python/ShaHash.hpp index 237b1a3a5..a74335cb9 100644 --- a/typed_python/ShaHash.hpp +++ b/typed_python/ShaHash.hpp @@ -122,6 +122,29 @@ class ShaHash { return res; } + static ShaHash fromHexDigest(std::string s) { + if (s.size() != sizeof(ShaHash) * 2 + 2) { + throw std::runtime_error("Incorrect size for hash-digest"); + } + + ShaHash res; + uint8_t* srcDataPtr = (uint8_t*)&s[2]; + uint8_t* destDataPtr = (uint8_t*)&res; + + auto fromHex = [&](uint8_t u) { + if (u >= '0' && u <= '9') { return u - '0'; } + if (u >= 'a' && u <= 'f') { return 10 + u - 'a'; } + if (u >= 'A' && u <= 'F') { return 10 + u - 'A'; } + return 0; + }; + + for (int i = 0; i < sizeof(ShaHash); i++) { + destDataPtr[i] = fromHex(srcDataPtr[i * 2]) * 16 + fromHex(srcDataPtr[i * 2 + 1]); + } + + return res; + } + // create a 'poison' sha hash that, like a nan in float-land, // always produces another poison sha hash when added. We can use this // to indicate that a hash is 'bad' in some dimension. @@ -189,3 +212,19 @@ inline ShaHash operator+(const ShaHash& l, const ShaHash& r) { return tr; } + +namespace std { + template<> + struct hash { + typedef ShaHash argument_type; + typedef std::size_t result_type; + + result_type operator()(argument_type const& s) const noexcept { + return ((size_t)s[0] + (((size_t)s[1]) << 32)) + ^ ((size_t)s[2] + (((size_t)s[3]) << 32)) + ^ (size_t)s[3]; + } + }; +} + + diff --git a/typed_python/StringType.hpp b/typed_python/StringType.hpp index 49f943eca..3cb918187 100644 --- a/typed_python/StringType.hpp +++ b/typed_python/StringType.hpp @@ -41,8 +41,6 @@ class StringType : public Type { m_name = "str"; m_is_default_constructible = true; m_size = sizeof(void*); - - endOfConstructorInitialization(); // finish initializing the type object. } template @@ -117,14 +115,6 @@ class StringType : public Type { static layout* createFromString(std::string s); - bool isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; - } - template void _visitReferencedTypes(const visitor_type& v) {} @@ -344,4 +334,6 @@ class StringType : public Type { static bool to_int64(layout* s, int64_t* value); static bool to_float64(layout* s, double* value); + + void postInitializeConcrete() {}; }; diff --git a/typed_python/SubclassOfType.cpp b/typed_python/SubclassOfType.cpp index fe4d8ba85..91645e89a 100644 --- a/typed_python/SubclassOfType.cpp +++ b/typed_python/SubclassOfType.cpp @@ -16,35 +16,6 @@ #include "AllTypes.hpp" -bool SubclassOfType::_updateAfterForwardTypesChanged() { - size_t size = sizeof(Type*); - std::string name = computeName(); - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool is_default_constructible = true; - - bool anyChanged = ( - size != m_size || - name != m_name || - is_default_constructible != m_is_default_constructible - ); - - m_size = size; - m_name = name; - m_stripped_name = ""; - - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - -std::string SubclassOfType::computeName() const { - return "SubclassOf(" + m_subclassOf->name(true) + ")"; -} - void SubclassOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { stream << "name() + "'>"; } @@ -89,16 +60,24 @@ void SubclassOfType::assign(instance_ptr self, instance_ptr other) { } // static -SubclassOfType* SubclassOfType::Make(Type* subclassOf, SubclassOfType* knownType) { - PyEnsureGilAcquired getTheGil; +SubclassOfType* SubclassOfType::Make(Type* subclassOf) { + if (subclassOf->isForwardDefined()) { + return new SubclassOfType(subclassOf); + } - static std::map m; + PyEnsureGilAcquired getTheGil; - auto it = m.find(subclassOf); + static std::map memo; - if (it == m.end()) { - it = m.insert(std::make_pair(subclassOf, knownType ? knownType : new SubclassOfType(subclassOf))).first; + auto it = memo.find(subclassOf); + if (it != memo.end()) { + return it->second; } - return it->second; + SubclassOfType* res = new SubclassOfType(subclassOf); + SubclassOfType* concrete = (SubclassOfType*)res->forwardResolvesTo(); + + memo[subclassOf] = concrete; + + return concrete; } diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index 6e65aa19e..cc4b6a952 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -26,13 +26,41 @@ PyDoc_STRVAR(SubclassOf_doc, class SubclassOfType : public Type { public: + SubclassOfType() : Type(TypeCategory::catSubclassOf) + { + } + SubclassOfType(Type* subclassOf) noexcept : Type(TypeCategory::catSubclassOf), m_subclassOf(subclassOf) { - m_doc = SubclassOf_doc; + m_is_forward_defined = true; + recomputeName(); + } + + const char* docConcrete() { + return SubclassOf_doc; + } + + void initializeDuringDeserialization(Type* subclassOf) { + m_subclassOf = subclassOf; + } + + void postInitializeConcrete() { + m_is_default_constructible = true; + m_size = sizeof(Type*); + } - endOfConstructorInitialization(); // finish initializing the type object. + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_subclassOf = ((SubclassOfType*)forwardDefinitionOfSelf)->m_subclassOf; + } + + Type* cloneForForwardResolutionConcrete() { + return new SubclassOfType(); + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_subclassOf, groupMap); } template @@ -41,10 +69,6 @@ class SubclassOfType : public Type { v.visitTopo(m_subclassOf); } - bool isBinaryCompatibleWithConcrete(Type* other) { - return false; - } - template void _visitContainedTypes(const visitor_type& visitor) { visitor(m_subclassOf); @@ -55,13 +79,15 @@ class SubclassOfType : public Type { _visitContainedTypes(visitor); } - bool _updateAfterForwardTypesChanged(); - bool isPODConcrete() { return true; } - std::string computeName() const; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "SubclassOf(" + + m_subclassOf->computeRecursiveName(typeStack) + + ")"; + } void deepcopyConcrete( instance_ptr dest, @@ -104,11 +130,7 @@ class SubclassOfType : public Type { return m_subclassOf; } - void _updateTypeMemosAfterForwardResolution() { - SubclassOfType::Make(m_subclassOf, this); - } - - static SubclassOfType* Make(Type* subclassOf, SubclassOfType* knownType = nullptr); + static SubclassOfType* Make(Type* subclassOf); private: Type* m_subclassOf; diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 911f36724..9993c5a30 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -16,31 +16,6 @@ #include "AllTypes.hpp" -bool TupleOrListOfType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - TupleOfType* otherO = (TupleOfType*)other; - - return m_element_type->isBinaryCompatibleWith(otherO->m_element_type); -} - -bool TupleOrListOfType::_updateAfterForwardTypesChanged() { - std::string name = (m_is_tuple ? "TupleOf(" : "ListOf(") + m_element_type->name(true) + ")"; - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool anyChanged = name != m_name; - - m_name = name; - m_stripped_name = ""; - - return anyChanged; -} - void TupleOrListOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { PushReprState isNew(stream, self); @@ -156,42 +131,79 @@ bool TupleOrListOfType::cmp(instance_ptr left, instance_ptr right, int pyCompari return cmpResultToBoolForPyOrdering(pyComparisonOp,0); } -// static -TupleOfType* TupleOfType::Make(Type* elt, TupleOfType* knownType) { - PyEnsureGilAcquired getTheGil; - static std::map m; - auto it = m.find(elt); +Type* TupleOrListOfType::cloneForForwardResolutionConcrete() { + if (m_is_tuple) { + return new TupleOfType(); + } else { + return new ListOfType(); + } +} - if (it == m.end()) { - if (knownType == nullptr) { - knownType = new TupleOfType(elt); - } +void TupleOrListOfType::initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_element_type = ((TupleOrListOfType*)forwardDefinitionOfSelf)->m_element_type; +} - it = m.insert(std::make_pair(elt, knownType)).first; +void TupleOrListOfType::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + auto it = groupMap.find(m_element_type); + if (it != groupMap.end()) { + m_element_type = it->second; } +} - return it->second; +std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { + if (m_is_tuple) { + return "TupleOf(" + m_element_type->computeRecursiveName(stack) + ")"; + } else { + return "ListOf(" + m_element_type->computeRecursiveName(stack) + ")"; + } } // static -ListOfType* ListOfType::Make(Type* elt, ListOfType* knownType) { +TupleOfType* TupleOfType::Make(Type* elt) { PyEnsureGilAcquired getTheGil; - static std::map m; + if (elt->isForwardDefined()) { + return new TupleOfType(elt); + } - auto it = m.find(elt); + static std::map memo; - if (it == m.end()) { - if (knownType == nullptr) { - knownType = new ListOfType(elt); - } + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; + } - it = m.insert(std::make_pair(elt, knownType)).first; + TupleOfType* res = new TupleOfType(elt); + TupleOfType* concrete = (TupleOfType*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; +} + +// static +ListOfType* ListOfType::Make(Type* elt) { + PyEnsureGilAcquired getTheGil; + + if (elt->isForwardDefined()) { + return new ListOfType(elt); } - return it->second; + static std::map memo; + + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; + } + + ListOfType* res = new ListOfType(elt); + ListOfType* concrete = (ListOfType*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } int64_t TupleOrListOfType::count(instance_ptr self) const { diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 64e7c30d9..4a9514d06 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -32,19 +32,29 @@ class TupleOrListOfType : public Type { typedef layout* layout_ptr; -public: + // this is the non-forward clone pathway + TupleOrListOfType(bool isTuple) : + Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), + m_element_type(nullptr), + m_is_tuple(isTuple) + { + m_is_default_constructible = true; + } + TupleOrListOfType(Type* type, bool isTuple) : Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), m_element_type(type), m_is_tuple(isTuple) { - m_size = sizeof(void*); + m_is_forward_defined = true; m_is_default_constructible = true; - endOfConstructorInitialization(); // finish initializing the type object. + recomputeName(); } - bool isBinaryCompatibleWithConcrete(Type* other); + void initializeDuringDeserialization(Type* elementType) { + m_element_type = elementType; + } template void _visitContainedTypes(const visitor_type& visitor) { @@ -62,8 +72,6 @@ class TupleOrListOfType : public Type { v.visitTopo(m_element_type); } - bool _updateAfterForwardTypesChanged(); - //serialize, but don't write a count template void serializeStream(instance_ptr self, buf_t& buffer) { @@ -587,9 +595,22 @@ class TupleOrListOfType : public Type { } + Type* cloneForForwardResolutionConcrete(); + + void initializeFromConcrete(Type* forwardDefinitionOfSelf); + + void postInitializeConcrete() { + m_size = sizeof(layout*); + }; + + std::string computeRecursiveNameConcrete(TypeStack& typeStack); + + void updateInternalTypePointersConcrete( + const std::map& groupMap + ); + protected: Type* m_element_type; - bool m_is_tuple; }; @@ -600,18 +621,24 @@ PyDoc_STRVAR(ListOf_doc, ); class ListOfType : public TupleOrListOfType { + friend class TupleOrListOfType; + public: + ListOfType() : TupleOrListOfType(false) + { + } + ListOfType(Type* type) : TupleOrListOfType(type, false) { - m_doc = ListOf_doc; } - static ListOfType* Make(Type* elt, ListOfType* knownType=nullptr); - void _updateTypeMemosAfterForwardResolution() { - ListOfType::Make(m_element_type, this); + const char* docConcrete() { + return ListOf_doc; } + static ListOfType* Make(Type* elt); + void setSizeUnsafe(instance_ptr self, size_t count); void append(instance_ptr self, instance_ptr other); @@ -659,17 +686,21 @@ PyDoc_STRVAR(TupleOf_doc, ); class TupleOfType : public TupleOrListOfType { + friend class TupleOrListOfType; + public: + TupleOfType() : TupleOrListOfType(true) + { + } + + // forward form TupleOfType(Type* type) : TupleOrListOfType(type, true) { - m_doc = TupleOf_doc; } - void _updateTypeMemosAfterForwardResolution() { - TupleOfType::Make(m_element_type, this); + const char* docConcrete() { + return TupleOf_doc; } - // get a memoized TupleOfType. If 'knownType', then install this type - // if not already known. - static TupleOfType* Make(Type* elt, TupleOfType* knownType = nullptr); + static TupleOfType* Make(Type* elt); }; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 77bf22c00..2a97abd03 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2019 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -211,40 +211,6 @@ void Type::assign(instance_ptr self, instance_ptr other) { this->check([&](auto& subtype) { subtype.assign(self, other); } ); } -bool Type::isBinaryCompatibleWith(Type* other) { - if (other == this) { - return true; - } - - if (isSubclassOf(other)) { - return true; - } - - while (other->getTypeCategory() == TypeCategory::catPythonSubclass) { - other = other->getBaseType(); - } - - auto it = mIsBinaryCompatible.find(other); - if (it != mIsBinaryCompatible.end()) { - return it->second != BinaryCompatibilityCategory::Incompatible; - } - - //mark that we are recursing through this datastructure. we don't want to - //loop indefinitely. - mIsBinaryCompatible[other] = BinaryCompatibilityCategory::Checking; - - bool isCompatible = this->check([&](auto& subtype) { - return subtype.isBinaryCompatibleWithConcrete(other); - }); - - mIsBinaryCompatible[other] = isCompatible ? - BinaryCompatibilityCategory::Compatible : - BinaryCompatibilityCategory::Incompatible - ; - - return isCompatible; -} - bool Type::canConvertToTrivially(Type* otherType) { if (typesEquivalent(this, otherType)) { return true; @@ -288,125 +254,547 @@ bool Type::canConvertToTrivially(Type* otherType) { return false; } -void Type::endOfConstructorInitialization() { - visitReferencedTypes([&](Type* &t) { - while (t->getTypeCategory() == TypeCategory::catForward && ((Forward*)t)->getTarget()) { - t = ((Forward*)t)->getTarget(); +void reachableUnresolvedTypes(Type* root, std::set& outTypes) { + PyObjGraphSnapshot graph; + std::unordered_map objMapCache; + std::unordered_map typeMapCache; + std::unordered_map instanceCache; + + // this finds every reachable python objects from 'root', with the search terminating at + // named module objects and named module dicts. This is enough information to completely + // characterize a type's "identity hash", since there can be at most one version of a + // given named module per program. + PyObjSnapshotMaker snapMaker( + objMapCache, + typeMapCache, + instanceCache, + &graph, + true, + true + ); + + snapMaker.internalize(root); + + std::unordered_set typeSnaps; + graph.getTypes(typeSnaps); + + for (auto snap: typeSnaps) { + Type* t = snap->getType(); + if (!t) { + throw std::runtime_error( + "Snap of kind " + snap->kindAsString() + " didn't produce a type" + ); } - if (t == this) { - return; + if (t->isForwardDefined() && !t->isResolved()) { + outTypes.insert(t); } + } +} - if (t->getTypeCategory() == TypeCategory::catForward) { - m_referenced_forwards.insert((Forward*)t); - ((Forward*)t)->markIndirectForwardUse(this); - } else { - for (auto referencedT: t->getReferencedForwards()) { - m_referenced_forwards.insert(referencedT); - referencedT->markIndirectForwardUse(this); - } +bool Type::looksResolvable(bool unambiguously) { + if (!m_is_forward_defined) { + return true; + } + + if (m_forward_resolves_to) { + return true; + } + + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + return false; } - }); + } - visitContainedTypes([&](Type* t) { - if (t->getTypeCategory() == TypeCategory::catForward) { - m_contained_forwards.insert((Forward*)t); - } else { - for (auto containedT: t->getContainedForwards()) { - m_contained_forwards.insert(containedT); + if (unambiguously) { + for (auto t: typesNeedingResolution) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols(false)) { + return false; } } - }); - - if (!m_referenced_forwards.size()) { - forwardTypesAreResolved(); - } else { - updateAfterForwardTypesChanged(); } -} -void Type::forwardTypesAreResolved() { - m_resolved = true; + return true; +} - updateAfterForwardTypesChanged(); +void Type::assertForwardsResolvedSufficientlyToInstantiateConcrete() { + if (!m_is_forward_defined) { + return; + } - if (m_is_simple) { - bool isSimple = true; + if (m_forward_resolves_to) { + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be instantiated. However, " + + "has already been resolved, so it needs only to be replaced with a concrete " + + "type by calling resolveForwardDeclaredType and it can be used." + ); + } - visitReferencedTypes([&](Type* t) { if (!t->m_is_simple) isSimple = false; }); + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; - if (!isSimple) { - std::function markNotSimple([&](Type* t) { - if (t->m_is_simple) { - t->m_is_simple = false; - markNotSimple(t); - } - }); + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); - markNotSimple(this); + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be autoresolved because " + + "it refers to forward " + t->nameWithModule() + " which has no definition." + ); } } - if (mTypeRep) { - updateTypeRepForType(this, mTypeRep); + for (auto t: typesNeedingResolution) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols(false)) { + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be autoresolved because " + + "it refers to forward " + t->nameWithModule() + " which has a reference to " + + "symbol '" + ((Function*)t)->firstUnresolvedSymbol(false) + "' which is " + + "unresolved." + ); + } } + + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be instantiated. However, " + + "it is resolvable, so it needs only to be replaced with a concrete " + + "type by calling resolveForwardDeclaredType and it can be used." + ); } -void Type::forwardResolvedTo(Forward* forward, Type* resolvedTo) { - // make sure we reference this forward properly - if (m_referenced_forwards.find(forward) == m_referenced_forwards.end()) { - throw std::runtime_error( - "Internal error: we are supposed to reference forward " + - forward->name() + " but we don't have it marked." - ); +bool Type::assertResolvable(bool unambiguously) { + if (!m_is_forward_defined) { + return true; + } + + if (m_forward_resolves_to) { + return true; } - // swap out the type representation. we shouldn't reference this forward any more - visitReferencedTypes([&](Type* &subtype) { - if (subtype == forward) { - subtype = resolvedTo; + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + throw std::runtime_error("Forward " + t->name() + " has no definition."); } - }); + } - // update the 'containment' forward graph - if (m_contained_forwards.find(forward) != m_contained_forwards.end()) { - m_contained_forwards.erase(forward); - for (auto contained: resolvedTo->getContainedForwards()) { - if (contained != forward) { - m_contained_forwards.insert(contained); + if (unambiguously) { + for (auto t: typesNeedingResolution) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols(false)) { + throw std::runtime_error( + "Function " + t->nameWithModule() + " has unresolved symbol '" + + ((Function*)t)->firstUnresolvedSymbol(false) + "'" + ); } } } - m_referenced_forwards.erase(forward); + return true; +} + +// try to resolve this forward type. If we can't, we'll throw an exception. On exit, +// we will have thrown, or m_forward_resolves_to will be populated. +void Type::attemptToResolve() { + if (m_forward_resolves_to) { + return; + } + + if (!m_is_forward_defined) { + return; + } + + if (::getenv("TP_NEW_INTERNING") && ::getenv("TP_NEW_INTERNING")[0]) { + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + // compute a snapshot starting with us as the root + PyObjGraphSnapshot graph(this); + + // build a map from our existing types into the graph since the graph is going + // to change + std::map typeToSnap; + for (auto snap: graph.getObjects()) { + if (snap->getType()) { + typeToSnap[snap->getType()] = snap; + } + } + if (typeToSnap.find(this) == typeToSnap.end()) { + throw std::runtime_error("Somehow, we're not in the graph?"); + } + + // take the graph and remove any forwards from it. We will now have a graph where + // no object points directly to a forward anymore. Each forward will point to its + // resolution, and every PyObjSnapshot has a link back to the original object or + // type that it came from. + graph.resolveForwards(); + + // recompute the names of all the recursive types in here, which we have to do before + // rehydrating anything + graph.recomputeNames(); + + // now take every non-forward node and make a version of it in the internal graph, + // if it doesn't already exist. Then rehydrate it. At this point, we have an interned + // copy of every single type and pyobject that was present in this graph. + graph.internalize(); + + // now we need to link the new graph back to our types. + // Every forward Type in our current graph needs to be pointed to the resolved type. + for (auto typeAndSnap: typeToSnap) { + PyObjSnapshot* internalizedSnap = graph.internal().snapshotForHash( + graph.hashFor(typeAndSnap.second) + ); + + if (!internalizedSnap) { + throw std::runtime_error("Somehow our Type didn't map to a type in the internalized graph"); + } + + Type* target = internalizedSnap->getType(); - if (resolvedTo->getTypeCategory() == TypeCategory::catForward) { - Forward* tgtForward = (Forward*)resolvedTo; + if (target && target->isForward()) { + target = ((Forward*)target)->getTargetTransitive(); + } - if (tgtForward->getTarget()) { - throw std::runtime_error("Forwards must not resolve to other forwards that are already resolved!"); + if (target) { + if (typeAndSnap.first->m_forward_resolves_to) { + if (typeAndSnap.first->m_forward_resolves_to != target) { + throw std::runtime_error("This type is already resolved and not to our target!"); + } + } else { + typeAndSnap.first->m_forward_resolves_to = target; + } + } } - m_referenced_forwards.insert(tgtForward); + if (!m_forward_resolves_to) { + throw std::runtime_error("Somehow, we didn't resolve???"); + } } else { - for (auto referenced: resolvedTo->getReferencedForwards()) { - if (referenced != forward) { - referenced->markIndirectForwardUse(this); - m_referenced_forwards.insert(referenced); + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + std::set existingReferencedTypes; + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + throw std::runtime_error( + "Forward defined as " + t->nameWithModule() + " has not been defined." + ); + } + } + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget()) { + ((Forward*)t)->define( + ((Forward*)t)->lambdaDefinition() + ); + } + } + + + // for each type that we have defined in our graph, what type are we mapping it to? + // for Forwards of primitive types like ListOf or TupleOf that can see themselves, this + // this will be a 'named copy' of the type depending on the Forward name. + std::map resolutionMapping; + + // for each type we end up defining, which type did it come from + std::map resolutionSource; + + // allocate new target type bodies for all regular types in the graph + // that are forward declared. + for (auto t: typesNeedingResolution) { + if (!t->isForward()) { + // we're resolving to this type directly + resolutionMapping[t] = t->cloneForForwardResolution(); + resolutionSource[resolutionMapping[t]] = t; + } + } + + // now ensure that the target for any forward is the underlying type + // note that we don't need a source for any of these since nobody will end up + // actually having a forward in their graph + for (auto t: typesNeedingResolution) { + if (t->isForward()) { + Forward* f = (Forward*)t; + + Type* tgt = f->getTargetTransitive(); + + if (!tgt) { + throw std::runtime_error("somehow this forward doesn't have a target"); + } + + if (typesNeedingResolution.find(tgt) == typesNeedingResolution.end()) { + resolutionMapping[t] = tgt; + existingReferencedTypes.insert(tgt); + } else { + // resolve this forward to whatever its target resolves to + resolutionMapping[t] = resolutionMapping[tgt]; + } + } + } + + // copy all the types. At this point, they all know that they are not forwards + // but none of them is ready to be identity-hashed yet. They all will be in a state of + // where m_needs_post_init is true. + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->initializeFrom(typeAndSource.second); + typeAndSource.first->updateInternalTypePointers(resolutionMapping); + } + + // allow each Function type to build CompilerVisiblePythonObjects for its globals + // after this, any FunctionGlobals will refer to Constants not cells + for (auto typeAndSource: resolutionSource) { + if (typeAndSource.first->isFunction()) { + ((Function*)typeAndSource.first)->internalizeConstants(resolutionMapping); + } + } + + // cause each type to recompute its name. We have to do this in a single pass + // without any types being aware of their final name before we run the post initialize + // step. Otherwise, it's possible for the names to depend on the order of initialization + // and we want them to be stable. + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->recomputeName(); + } + + // cause each type to post-initialize itself, which lets it update internal bytecounts and + // default initialization flags + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto typeAndSource: resolutionSource) { + if (typeAndSource.first->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > resolutionSource.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } + } + + // let each type update any internal caches it might need before it gets instantiated + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->finalizeType(); + } + + // std::cout << "resolving group:\n"; + // for (auto typeAndSource: resolutionSource) { + // std::cout << " " << TypeOrPyobj(typeAndSource.first).name() << " from " << TypeOrPyobj(typeAndSource.second).name() << "\n"; + // } + + // now internalize the types by their hash. For each type, we compute a hash + // and then look to see if we've seen it before. We build a lookup table from + // each existing type to the internalized type, and then do the same process we did + // above to map any roots across. + + std::map resolvedToInternal; // each 'resolved' type what should it be replaced with + + std::set newTypes; + std::set redundantTypes; + + for (auto typeAndSource: resolutionSource) { + ShaHash h = typeAndSource.first->identityHash(); + + auto it = mInternalizedIdentityHashToType.find(h); + if (it == mInternalizedIdentityHashToType.end()) { + // this is a new type. We're going to put it in the memo and + // update it so that it ponts + newTypes.insert(typeAndSource.first); + } else { + resolvedToInternal[typeAndSource.first] = it->second; + redundantTypes.insert(typeAndSource.first); + } + } + + // update all the new types to no longer look at the redundant types + for (auto t: newTypes) { + t->updateInternalTypePointers(resolvedToInternal); + mInternalizedIdentityHashToType[t->identityHash()] = t; + resolvedToInternal[t] = t; + } + + for (auto t: redundantTypes) { + t->markRedundant(); + } + + // tell each source type which type it actually resolves to. We're holding the GIL so + // nobody should see anything until we finish this process. + for (auto typeAndTarget: resolutionMapping) { + auto it = resolvedToInternal.find(typeAndTarget.second); + if (it == resolvedToInternal.end()) { + // this must be an existing type + if (existingReferencedTypes.find(typeAndTarget.second) == existingReferencedTypes.end()) { + throw std::runtime_error( + "Failed to get an internalization of Type " + + typeAndTarget.second->name() + " and it wasn't a pre-existing type either." + ); + } + + // we resolve to it directly + typeAndTarget.first->m_forward_resolves_to = typeAndTarget.second; + } else { + typeAndTarget.first->m_forward_resolves_to = it->second; } } - if (m_referenced_forwards.size() == 0) { - forwardTypesAreResolved(); + if (!m_forward_resolves_to) { + throw std::runtime_error("Somehow, we didn't resolve???"); + } + + for (auto typeAndTarget: resolutionMapping) { + if (typeAndTarget.first->isForwardDefined()) { + typeAndTarget.second->addForwardDefinition(typeAndTarget.first); + } } } +} - this->check([&](auto& subtype) { - subtype._updateTypeMemosAfterForwardResolution(); - }); +void Type::internalize() { + if (mIdentityHash == ShaHash()) { + throw std::runtime_error("This type should already have been hashed!"); + } + + if (mInternalizedIdentityHashToType.find(mIdentityHash) + != mInternalizedIdentityHashToType.end()) { + throw std::runtime_error("This type is already internalized!"); + } + + mInternalizedIdentityHashToType[mIdentityHash] = this; } +void Type::tryToAutoresolve() { + // some types don't need resolution + if (!m_is_forward_defined) { + return; + } + + // some types are already resolved + if (m_forward_resolves_to) { + return; + } + + if (!looksResolvable(true)) { + return; + } + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + if (::getenv("TP_AUTORESOLVE_VERBOSE") && ::getenv("TP_AUTORESOLVE_VERBOSE")[0]) { + std::cout << "try to autoresolve " << TypeOrPyobj(this).name() << ":\n"; + for (auto t: typesNeedingResolution) { + std::cout << " " << TypeOrPyobj(t).name() << "\n"; + } + } + + attemptToResolve(); + + for (auto t: typesNeedingResolution) { + t->attemptAutoresolveWrite(); + } + + for (auto t: typesNeedingResolution) { + if (t->isForward()) { + ((Forward*)t)->installDefinitionIfLambda(); + } + + if (t->isFunction()) { + ((Function*)t)->autoresolveGlobals(typesNeedingResolution); + } + } +} + +void Type::attemptAutoresolveWrite() { + if (!m_is_forward_defined || !m_forward_resolves_to) { + return; + } + + if (!mAutoresolveFrameOwners.size()) { + return; + } + + long updateCount = 0; + + for (auto f: mAutoresolveFrameOwners) { + // force the locals to be populated with a dict so we can muck with them + PyFrame_FastToLocals((PyFrameObject*)f); + + PyObject* locals = ((PyFrameObject*)f)->f_locals; + + if (locals && PyDict_Check(locals)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + + PyObject* ownTypeObj = (PyObject*)PyInstance::typeObj(this); + PyObject* resolvedTypeObj = (PyObject*)PyInstance::typeObj(m_forward_resolves_to); + + bool updated = false; + while (PyDict_Next(locals, &pos, &key, &value)) { + if (value == ownTypeObj) { + PyDict_SetItem(locals, key, resolvedTypeObj); + updated = true; + } + } + + if (updated) { + updateCount += 1; + PyFrame_LocalsToFast((PyFrameObject*)f, 0); + } + } + + decref(f); + } + + mAutoresolveFrameOwners.clear(); + + if (::getenv("TP_AUTORESOLVE_VERBOSE") && ::getenv("TP_AUTORESOLVE_VERBOSE")[0]) { + std::cout << "Autoresolve wrote " << name() << " into " << updateCount << " stackframe slots\n"; + } +} + +void Type::typeFinishedBeingDeserializedPhase1() { + if (m_is_being_deserialized) { + // during deserialization we have to defer anything that actually + // looks hard at the type and copies data into the PyTypeObject + // because its not ready yet. This allows us to refer to the object + // without it being finished. + PyTypeObject* typeObj = PyInstance::typeObj(this); + + PyInstance::finalizePyTypeObjectPhase1(this, typeObj); + } +} + +void Type::typeFinishedBeingDeserializedPhase2() { + if (m_is_being_deserialized) { + // during deserialization we have to defer anything that actually + // looks hard at the type and copies data into the PyTypeObject + // because its not ready yet. This allows us to refer to the object + // without it being finished. + PyTypeObject* typeObj = PyInstance::typeObj(this); + + PyInstance::finalizePyTypeObjectPhase2(this, typeObj); + + m_is_being_deserialized = false; + } +} PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { int num_args = 0; @@ -430,3 +818,6 @@ PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { curResolver = resolver; return curResolver; } + +//static +std::map Type::mInternalizedIdentityHashToType; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index cb62247b5..34c5a1388 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2020 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -39,9 +39,11 @@ #include "MutuallyRecursiveTypeGroup.hpp" #include "Slab.hpp" #include "DeepcopyContext.hpp" +#include "TypeStack.hpp" class SerializationBuffer; class DeserializationBuffer; +class PyObjSnapshot; class Type; class NoneType; @@ -59,7 +61,6 @@ class Float64; class StringType; class BytesType; class OneOfType; -class SubclassOfType; class Value; class TupleOfType; class PointerTo; @@ -72,8 +73,8 @@ class ConstDictType; class Alternative; class ConcreteAlternative; class AlternativeMatcher; -class PythonSubclass; class PythonObjectOfType; +class SubclassOfType; class Class; class HeldClass; class Function; @@ -88,8 +89,6 @@ typedef uint8_t* instance_ptr; typedef void (*compiled_code_entrypoint)(instance_ptr, instance_ptr*); -void updateTypeRepForType(Type* t, PyTypeObject* pyType); - PyObject* getOrSetTypeResolver(PyObject* module = nullptr, PyObject* args = nullptr); enum class Maybe { @@ -129,7 +128,6 @@ class Type { catConstDict = 22, catAlternative = 23, catConcreteAlternative = 24, //concrete Alternative subclass - catPythonSubclass = 25, //subclass of a typed_python type catPythonObjectOfType = 26, //a python object that matches 'isinstance' on a particular type catBoundMethod = 27, catAlternativeMatcher = 37, @@ -177,10 +175,22 @@ class Type { return m_typeCategory == catSet; } + bool isPointerTo() const { + return m_typeCategory == catPointerTo; + } + bool isConstDict() const { return m_typeCategory == catConstDict; } + bool isAlternativeMatcher() const { + return m_typeCategory == catAlternativeMatcher; + } + + bool isBoundMethod() const { + return m_typeCategory == catBoundMethod; + } + bool isNamedTuple() const { return m_typeCategory == catNamedTuple; } @@ -243,6 +253,10 @@ class Type { ); } + bool isEmbeddedMessage() const { + return m_typeCategory == catEmbeddedMessage; + } + bool isRegister() const { return ( m_typeCategory == catBool @@ -303,39 +317,32 @@ class Type { ); } - const std::string& name(bool stripQualname=false) const { - if (stripQualname) { - if (!m_stripped_name.size()) { - m_stripped_name = qualname_to_name(m_name); - } - - return m_stripped_name; - } + const std::string& name() const { return m_name; } std::string moduleName() { - return this->check([&](auto& subtype) { - return subtype.moduleNameConcrete(); - }); + return m_module_name; } - std::string moduleNameConcrete() { - return "builtins"; + std::string nameWithModule() { + if (m_module_name.size()) { + return m_module_name + "." + m_name; + } + + return m_name; } - std::string nameWithModule() { + const char* doc() { return this->check([&](auto& subtype) { - return subtype.nameWithModuleConcrete(); + return subtype.docConcrete(); }); } - std::string nameWithModuleConcrete() { - return name(); - } - - const char* doc() const { - return m_doc; + const char* docConcrete() { + throw std::runtime_error( + "No docstring provided for " + name() + " of category " + getTypeCategoryString() + ); } size_t bytecount() const { @@ -487,7 +494,6 @@ class Type { if (category == Type::TypeCategory::catConstDict) { return "ConstDict"; } if (category == Type::TypeCategory::catAlternative) { return "Alternative"; } if (category == Type::TypeCategory::catConcreteAlternative) { return "ConcreteAlternative"; } - if (category == Type::TypeCategory::catPythonSubclass) { return "PythonSubclass"; } if (category == Type::TypeCategory::catBoundMethod) { return "BoundMethod"; } if (category == Type::TypeCategory::catAlternativeMatcher) { return "AlternativeMatcher"; } if (category == Type::TypeCategory::catClass) { return "Class"; } @@ -560,8 +566,6 @@ class Type { return f(*(Alternative*)this); case catConcreteAlternative: return f(*(ConcreteAlternative*)this); - case catPythonSubclass: - return f(*(PythonSubclass*)this); case catPythonObjectOfType: return f(*(PythonObjectOfType*)this); case catClass: @@ -585,7 +589,32 @@ class Type { case catSubclassOf: return f(*(SubclassOfType*)this); default: - throw std::runtime_error("Invalid type found"); + throw std::runtime_error("Invalid type found: " + format((int)m_typeCategory)); + } + } + + template + static void adaptTypeVisitor(const visitor_type& vis, concrete_type*& ioTypePtr) { + Type* t = ioTypePtr; + vis(t); + if (t != (Type*)ioTypePtr) { + if (t->getTypeCategory() != ioTypePtr->getTypeCategory() && !ioTypePtr->isForward()) { + throw std::runtime_error( + "Somehow, visitor changed the type category from " + + ioTypePtr->getTypeCategoryString() + " to " + t->getTypeCategoryString() + ); + } + + ioTypePtr = (concrete_type*)t; + } + } + + template + static void updateTypeRefFromGroupMap(T*& toUpdate, const std::map& groupMap) { + auto it = groupMap.find(toUpdate); + + if (it != groupMap.end()) { + toUpdate = (T*)it->second; } } @@ -593,25 +622,21 @@ class Type { return m_base; } + size_t typeLevel() const { + if (!m_base) { + return 0; + } + + return m_base->typeLevel() + 1; + } + // are these types equivalent up to identity? This should be // preferred over t1 == t2 for comparing types in cases where // we don't need _exact_ identity equiality, because there are // some pathways where the identical type object can get created // (usually through deserialization) static bool typesEquivalent(Type* t1, Type* t2) { - if (t1 == t2) { - return true; - } - - if (!t1 || !t2) { - return false; - } - - if (t1->m_typeCategory != t2->m_typeCategory) { - return false; - } - - return t1->identityHash() == t2->identityHash(); + return t1 == t2; } //this checks _strict_ subclass. X is not a subclass of itself. @@ -661,8 +686,8 @@ class Type { } void assertForwardsResolved() const { - if (!m_resolved) { - throw std::logic_error("Type " + m_name + " has unresolved forwards."); + if (m_is_forward_defined) { + throw std::logic_error("Type " + m_name + " is a forward"); } } @@ -672,9 +697,7 @@ class Type { }); } - void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { - assertForwardsResolved(); - } + void assertForwardsResolvedSufficientlyToInstantiateConcrete(); template void _visitReferencedTypes(const visitor_type& v) {} @@ -738,43 +761,12 @@ class Type { }); } - // dispatch to the actual subtype - bool updateAfterForwardTypesChanged() { - return this->check([&](auto& subtype) { - return subtype._updateAfterForwardTypesChanged(); - }); - } - - // subtype-specific calculation - bool _updateAfterForwardTypesChanged() { return false; } - - // update our T::Make memos to reflect the fact that our - // types may have referred to forward that are now replaced. - // the type memos will have us listed with the Forward as one - // of the arguments to the 'Make' function, but now we'll have - // the resolved circular type dependency, and we need that memo - // to resolve to 'this' as well. - // - // subtypes are expected to specialize this function - void _updateTypeMemosAfterForwardResolution() {} - - // called when a downstream type has changed in some way. - // this may recalculate our on-disk size, our name, or some other - // feature of the type that depends on the forward delcarations - // below us. - void forwardTypesAreResolved(); - - void buildMutuallyRecursiveTypeCycle(); - - // called after each type has initialized its internals - void endOfConstructorInitialization(); - // call subtype.copy_constructor void copy_constructor(instance_ptr self, instance_ptr other); template void copy_constructor(int64_t count, const ptr_func_dest& ptrToTarget, const ptr_func_src& ptrToSrc) { - assertForwardsResolved(); + assertForwardsResolvedSufficientlyToInstantiate(); this->check([&](auto& subtype) { for (long k = 0; k < count; k++) { @@ -798,36 +790,6 @@ class Type { return m_is_default_constructible; } - bool resolved() const { - return m_resolved; - } - - bool isBinaryCompatibleWith(Type* other); - - bool isBinaryCompatibleWithConcrete(Type* other) { - return false; - } - - bool isSimple() const { - return m_is_simple; - } - - const std::set& getReferencedForwards() const { - return m_referenced_forwards; - } - - const std::set& getContainedForwards() const { - return m_contained_forwards; - } - - void forwardResolvedTo(Forward* forward, Type* resolvedTo); - - void setNameAndIndexForRecursiveType(std::string nameOverride, int index) { - m_recursive_name = nameOverride; - m_is_recursive_forward = true; - m_recursive_forward_index = index; - } - // are we guaranteed we can convert to this other type at the 'Signature' level bool canConvertToTrivially(Type* otherType); @@ -842,7 +804,10 @@ class Type { *****/ MutuallyRecursiveTypeGroup* getRecursiveTypeGroup() { if (!mTypeGroup) { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(this); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + this, + VisibilityType::Identity + ); if (!mTypeGroup) { throw std::runtime_error("Somehow, this type is not in a type group!"); @@ -880,29 +845,104 @@ class Type { ShaHash identityHash() { // if we've never initialized our hash if (mIdentityHash == ShaHash()) { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(this); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + this, + VisibilityType::Identity + ); - mIdentityHash = MutuallyRecursiveTypeGroup::shaHash(this); + mIdentityHash = MutuallyRecursiveTypeGroup::shaHash( + this, + VisibilityType::Identity + ); } return mIdentityHash; } - void setRecursiveTypeGroup(MutuallyRecursiveTypeGroup* group, int32_t index) { + void setRecursiveTypeGroup(MutuallyRecursiveTypeGroup* group, int32_t index, ShaHash hash) { + if (group->visibilityType() != VisibilityType::Identity) { + throw std::runtime_error("A Type's MutuallyRecursiveTypeGroup needs to be an Identity group"); + } mTypeGroup = group; mRecursiveTypeGroupIndex = index; + mIdentityHash = hash; } int64_t getRecursiveTypeGroupIndex() const { return mRecursiveTypeGroupIndex; } - int64_t getRecursiveForwardIndex() { - if (!m_is_recursive_forward) { - return -1; + bool isForwardDefined() const { + return m_is_forward_defined; + } + + bool isResolved() const { + if (!m_is_forward_defined) { + return true; } - return m_recursive_forward_index; + return m_forward_resolves_to != nullptr; + } + + bool looksResolvable(bool unambiguously); + bool assertResolvable(bool unambiguously); + + Type* forwardResolvesTo() { + if (!m_is_forward_defined) { + return this; + } + + if (!m_forward_resolves_to) { + attemptToResolve(); + } + + return m_forward_resolves_to; + } + + // attempt to autoresolve this type. This should + // always suceed and assumes we are holding the gil. + void tryToAutoresolve(); + + bool isRedundant() { + return m_is_redundant; + } + + // indicate that this frame may contain a reference to this object in its locals + void registerContainingFrameForAutoresolve(PyFrameObject* frame) { + mAutoresolveFrameOwners.push_back(incref((PyObject*)frame)); + } + + void attemptAutoresolveWrite(); + + void setIsForwardDefined(bool isForwardDefined) { + m_is_forward_defined = isForwardDefined; + } + + void markActivelyBeingDeserialized(bool isForwardDefined, std::string inModuleName, std::string inName) { + m_is_being_deserialized = true; + m_is_forward_defined = isForwardDefined; + m_name = inName; + m_module_name = inModuleName; + } + + bool isActivelyBeingDeserialized() { + return m_is_being_deserialized; + } + + bool isTypeObjReady() { + return m_type_obj_ready; + } + + void markTypeObjReady() { + m_type_obj_ready = true; + } + + + void typeFinishedBeingDeserializedPhase1(); + void typeFinishedBeingDeserializedPhase2(); + + const std::vector& getForwardDefinitions() const { + return mForwardDefinitions; } protected: @@ -911,58 +951,43 @@ class Type { m_size(0), m_is_default_constructible(false), m_name("Undefined"), - m_doc(nullptr), mTypeRep(nullptr), m_base(nullptr), - m_is_simple(true), - m_resolved(false), - m_is_recursive_forward(false), - m_recursive_forward_index(-1), mTypeGroup(nullptr), - mRecursiveTypeGroupIndex(-1) + mRecursiveTypeGroupIndex(-1), + m_is_forward_defined(false), + m_forward_resolves_to(nullptr), + m_is_redundant(false), + m_is_being_deserialized(false), + mSnapshot(nullptr), + m_type_obj_ready(false) {} + bool m_type_obj_ready; + + PyObjSnapshot* mSnapshot; + TypeCategory m_typeCategory; + // this class is actively being deserialized. + bool m_is_being_deserialized; + size_t m_size; bool m_is_default_constructible; - std::string m_recursive_name; - std::string m_name; - mutable std::string m_stripped_name; - - const char* m_doc; + std::string m_module_name; PyTypeObject* mTypeRep; Type* m_base; - // 'simple' types are those that have no reference to the python interpreter - bool m_is_simple; - - bool m_resolved; - - // were we defined as a recursive Forward type? - bool m_is_recursive_forward; - - // if we are recursive, then an integer indicating the order in - // which we were resolved, which is useful when trying to - // order the type graph - int64_t m_recursive_forward_index; + std::vector mAutoresolveFrameOwners; - enum BinaryCompatibilityCategory { Incompatible, Checking, Compatible }; - - std::map mIsBinaryCompatible; - - // a set of forward types that we need to be resolved before we - // could be resolved - std::set m_referenced_forwards; - - // a subset of m_referenced_forwards that we directly contain - std::set m_contained_forwards; + // the original forwards that defined this Type + std::vector mForwardDefinitions; // a sha-hash that uniquely identifies this type. If this value is // the same for two types, then they should be indistinguishable except @@ -973,4 +998,140 @@ class Type { MutuallyRecursiveTypeGroup* mTypeGroup; int32_t mRecursiveTypeGroupIndex; + + // is there a Forward reachable in our object graph? This is a permanent feature of + // the type object + bool m_is_forward_defined; + + // if we are a Forward, what do we resolve to? This will be null if we have not + // fully resolved ourselves and will, for valid types, become not-null pointing to + // the actual type we resolve to. + Type* m_forward_resolves_to; + + // have we been made 'redundant'? + bool m_is_redundant; + + // try to resolve this forward type. If we can't, we'll throw an exception. On exit, + // we will have thrown, or m_forward_resolves_to will be populated. + void attemptToResolve(); + + // this is a fully-resolved type that we created but aren't using. + // it should just leak and never be used again. + void markRedundant() { + m_is_redundant = true; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + throw std::runtime_error("Type " + name() + " didn't define initializeFromConcrete"); + } + + // update internal Type pointers to point into a new group + void updateInternalTypePointers(const std::map& groupMap) { + this->check([&](auto& subtype) { + return subtype.updateInternalTypePointersConcrete(groupMap); + }); + } + + void updateInternalTypePointersConcrete( + const std::map& groupMap + ) { + throw std::runtime_error("Type " + name() + " didn't define updateInternalTypePointersConcrete"); + } + + // produce a copy of ourself + Type* cloneForForwardResolution() { + return this->check([&](auto& subtype) { + return subtype.cloneForForwardResolutionConcrete(); + }); + } + + Type* cloneForForwardResolutionConcrete() { + throw std::runtime_error("Type " + name() + " didn't define cloneForForwardResolutionConcrete"); + } + + void postInitializeConcrete() { + throw std::runtime_error( + "Type " + name() + " of cat " + getTypeCategoryString() + " didn't implement postInitializeConcrete" + ); + } + + void finalizeTypeConcrete() {} + + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return m_name; + } + + static std::map mInternalizedIdentityHashToType; + +public: + void setSnapshot(PyObjSnapshot* snapshot) { + mSnapshot = snapshot; + } + + PyObjSnapshot* getSnapshot() const { + return mSnapshot; + } + + void addForwardDefinition(Type* t) { + if (!t->isForwardDefined()) { + throw std::runtime_error("Can't add a non-forward defined type as a reverse Forward link"); + } + + if (isForwardDefined()) { + throw std::runtime_error("Can't add a forward definition to a forward type - it should be resolved!"); + } + + mForwardDefinitions.push_back(t); + } + + void recomputeName() { + TypeStack stack; + m_name = computeRecursiveName(stack); + } + + void internalize(); + + // initialize ourself as a copy of 'forwardDefinitionOfSelf' where none of the types + // will have a forward definition reference. The instance must have been constructed + // using 'cloneForForwardResolution'. If the recursiveNameOverride is not None, then + // use that name for the type instead of the computed name. + void initializeFrom(Type* forwardDefinitionOfSelf) { + this->check([&](auto& subtype) { + return subtype.initializeFromConcrete(forwardDefinitionOfSelf); + }); + } + + // finish initializing the type assuming no forward types are reachable + bool postInitialize() { + size_t oldSize = m_size; + bool oldIsDefaultInit = m_is_default_constructible; + + this->check([&](auto& subtype) { + subtype.postInitializeConcrete(); + }); + + // return whether we changed + return m_size != oldSize || m_is_default_constructible != oldIsDefaultInit; + } + + // update any caches in the type after we've walked over it + void finalizeType() { + this->check([&](auto& subtype) { + subtype.finalizeTypeConcrete(); + }); + } + + std::string computeRecursiveName(TypeStack& stack) { + long index = stack.indexOf(this); + + if (index != -1) { + return "^" + format(index); + } + + PushTypeStack addSelf(stack, this); + + return this->check([&](auto& subtype) { + return subtype.computeRecursiveNameConcrete(stack); + }); + } }; diff --git a/typed_python/TypeOrPyobj.cpp b/typed_python/TypeOrPyobj.cpp index d1ad38c12..127914936 100644 --- a/typed_python/TypeOrPyobj.cpp +++ b/typed_python/TypeOrPyobj.cpp @@ -34,6 +34,7 @@ TypeOrPyobj::TypeOrPyobj(PyObject* o) : if (PyType_Check(mPyObj)) { mType = PyInstance::extractTypeFrom((PyTypeObject*)mPyObj, true); + if (mType) { mPyObj = nullptr; return; @@ -91,7 +92,8 @@ TypeOrPyobj TypeOrPyobj::withoutIntern(PyObject* o) { std::string TypeOrPyobj::name() const { if (mType) { std::ostringstream s; - s << "name() + " of cat " + Type::categoryToString(mType->getTypeCategory()) + "@" << (void*)mType << ">"; + s << "<" << (mType->isForwardDefined() ? "Forward":"") + << "Type " << mType->name() + " of cat " + Type::categoryToString(mType->getTypeCategory()) + "@" << (void*)mType << ">"; return s.str(); } diff --git a/typed_python/TypeStack.hpp b/typed_python/TypeStack.hpp new file mode 100644 index 000000000..9ae61b81a --- /dev/null +++ b/typed_python/TypeStack.hpp @@ -0,0 +1,78 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include + +class Type; + + +// model a call stack of types above us in a search. Clients can ask if a type +// is above us in the stack and if so, how many levels up, using 'indexOf' +// and push new types to the stack using PushTypeStack. The stack can't have +// duplicates. +template +class PtrStack { +public: + long indexOf(T* t) { + auto it = mTypeIndices.find(t); + + if (it == mTypeIndices.end()) { + return -1; + } + + return mTypeIndices.size() - it->second - 1; + } + + void push(T* t) { + if (mTypeIndices.find(t) != mTypeIndices.end()) { + throw std::runtime_error("Type is already in the type stack"); + } + size_t s = mTypeIndices.size(); + + mTypeIndices[t] = s; + } + + void pop(T* t) { + mTypeIndices.erase(t); + } + +private: + std::map mTypeIndices; +}; + +typedef PtrStack TypeStack; + + +template +class PushPtrStack { +public: + PushPtrStack(PtrStack& stack, T* t) : mStack(stack), mT(t) { + stack.push(mT); + } + + ~PushPtrStack() { + mStack.pop(mT); + } + +private: + PtrStack& mStack; + T* mT; +}; + + +typedef PushPtrStack PushTypeStack; diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index 867af2a14..e18b854e4 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -31,19 +31,45 @@ class TypedCellType : public Type { typedef layout* layout_ptr; + TypedCellType() : Type(TypeCategory::catTypedCell) + { + } + TypedCellType(Type* heldType) : Type(TypeCategory::catTypedCell), mHeldType(heldType) { - m_name = std::string("TypedCell(") + heldType->name(true) + ")"; + m_is_forward_defined = true; + recomputeName(); + } - m_is_simple = false; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "TypedCell(" + mHeldType->computeRecursiveName(typeStack) + ")"; + } - m_size = sizeof(layout*); + const char* docConcrete() { + return "TypedCell: a cell held in a fully-typed function closure."; + } + void initializeDuringDeserialization(Type* heldType) { + mHeldType = heldType; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + mHeldType = ((TypedCellType*)forwardDefinitionOfSelf)->mHeldType; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(mHeldType, groupMap); + } + + void postInitializeConcrete() { + m_size = sizeof(layout*); m_is_default_constructible = true; + } - endOfConstructorInitialization(); // finish initializing the type object. + Type* cloneForForwardResolutionConcrete() { + return new TypedCellType(); } template @@ -52,10 +78,6 @@ class TypedCellType : public Type { v.visitTopo(mHeldType); } - bool isBinaryCompatibleWithConcrete(Type* other) { - return other == this; - } - template void _visitContainedTypes(const visitor_type& visitor) { } @@ -65,25 +87,10 @@ class TypedCellType : public Type { visitor(mHeldType); } - bool _updateAfterForwardTypesChanged() { - std::string newName = std::string("TypedCell(") + mHeldType->name(true) + ")"; - - bool nameChanged = newName != m_name; - - m_name = newName; - m_stripped_name = ""; - - return nameChanged; - } - int64_t refcount(instance_ptr self) const { return getLayoutPtr(self)->refcount; } - void _updateTypeMemosAfterForwardResolution() { - TypedCellType::Make(mHeldType, this); - } - layout_ptr& getLayoutPtr(instance_ptr self) const { return *(layout**)self; } @@ -238,24 +245,25 @@ class TypedCellType : public Type { return getLayoutPtr(self)->data; } - static TypedCellType* Make(Type* t, TypedCellType* knownType = nullptr) { - PyEnsureGilAcquired getTheGil; + static TypedCellType* Make(Type* elt) { + if (elt->isForwardDefined()) { + return new TypedCellType(elt); + } - typedef Type* keytype; + PyEnsureGilAcquired getTheGil; - static std::map m; + static std::map memo; - auto it = m.find(t); - if (it == m.end()) { - it = m.insert( - std::make_pair( - t, - knownType ? knownType : new TypedCellType(t) - ) - ).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + TypedCellType* res = new TypedCellType(elt); + TypedCellType* concrete = (TypedCellType*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } Type* getHeldType() const { diff --git a/typed_python/TypedClosureBuilder.hpp b/typed_python/TypedClosureBuilder.hpp index 240defc78..d4dfd0291 100644 --- a/typed_python/TypedClosureBuilder.hpp +++ b/typed_python/TypedClosureBuilder.hpp @@ -87,7 +87,7 @@ class TypedClosureBuilder { if (PyFunction_Check(o)) { PyObjectStealer name(PyObject_GetAttrString(o, "__name__")); - Function* funcType = PyFunctionInstance::convertPythonObjectToFunctionType(name, o, false, true); + Function* funcType = PyFunctionInstance::convertPythonObjectToFunctionType(name, nullptr, o, false, true); if (!funcType) { throw PythonExceptionSet(); @@ -332,7 +332,7 @@ class TypedClosureBuilder { instance_ptr closureData = ((PyInstance*)(PyObject*)funcObj)->dataPtr(); Tuple* closureType = (Tuple*)funcT->getClosureType(); - std::vector newOverloads; + std::vector newOverloads; if (closureType->getTypes().size() != funcT->getOverloads().size()) { throw std::runtime_error("Untyped closures should have one element per overload."); @@ -354,7 +354,7 @@ class TypedClosureBuilder { return funcT->replaceClosure(mClosureType)->replaceOverloads(newOverloads); } - Function::Overload mapOverload(const Function::Overload& overload, NamedTuple* untypedClosure, instance_ptr untypedClosureData) { + FunctionOverload mapOverload(const FunctionOverload& overload, NamedTuple* untypedClosure, instance_ptr untypedClosureData) { std::map bindings; for (long cellIx = 0; cellIx < untypedClosure->getTypes().size(); cellIx++) { @@ -408,7 +408,7 @@ class TypedClosureBuilder { } PyObject* buildFinalResult() { - Function* outType = (Function*)mResolvedFunctionTypes[Path()]; + Function* outType = (Function*)mResolvedFunctionTypes[Path()]->forwardResolvesTo(); if (!outType) { throw std::runtime_error("Somehow, we don't have a function at the root of the closure converter"); diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 3862973d1..20901c089 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -1,29 +1,93 @@ /****************************************************************************** - Copyright 2017-2023 typed_python Authors + Copyright 2017-2023 typed_python Authors - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. ******************************************************************************/ #include "AllTypes.hpp" +Value::Value() : + Type(TypeCategory::catValue), + mValueAsPyobj(nullptr) +{ + m_size = 0; + m_is_default_constructible = true; +} + -Value::Value(const Instance& instance) : +Value::Value(const Instance& instance, PyObject* valueAsPyobj) : Type(TypeCategory::catValue), - mInstance(instance) + mInstance(instance), + mValueAsPyobj(incref(valueAsPyobj)) { m_size = 0; m_is_default_constructible = true; + m_is_forward_defined = true; + recomputeName(); +} + +Type* Value::Make(Instance i) { + PyEnsureGilAcquired getTheGil; + + // if its a forward declared Type then we need to behave differently + if (i.type()->isPythonObjectOfType() && + PyType_Check( + PyObjectHandleTypeBase::getPyObj(i.data()) + ) + ) { + PyObject* obj = PyObjectHandleTypeBase::getPyObj(i.data()); + Type* t = PyInstance::extractTypeFrom((PyTypeObject*)obj, true); + + if (t) { + static std::map typeMemo; + + auto it = typeMemo.find(t); + + if (it != typeMemo.end()) { + return it->second; + } + + if (t->isForwardDefined()) { + typeMemo[t] = new Value(i, obj); + } else { + typeMemo[t] = (Value*)(new Value(i, obj))->forwardResolvesTo(); + } + + return typeMemo[t]; + } + } + + static std::map instanceMemo; + + auto it = instanceMemo.find(i); + + if (it == instanceMemo.end()) { + PyObject* obj = PyInstance::extractPythonObject(i); + + if (!obj) { + throw std::runtime_error("Failed to convert an instance of type " + i.type()->name() + " to a PyObject!"); + } + + Value* resolved = (Value*)((new Value(i, obj))->forwardResolvesTo()); + it = instanceMemo.insert(std::make_pair(i, resolved)).first; + } + + return it->second; +} + +std::string Value::computeRecursiveNameConcrete(TypeStack& typeStack) +{ if ( mInstance.type()->isPythonObjectOfType() && PyType_Check( @@ -32,7 +96,14 @@ Value::Value(const Instance& instance) : ) { PyObject* obj = PyObjectHandleTypeBase::getPyObj(mInstance.data()); - std::string name(((PyTypeObject*)obj)->tp_name); + Type* t = PyInstance::extractTypeFrom((PyTypeObject*)obj, true); + + std::string name; + if (t) { + name = t->computeRecursiveName(typeStack); + } else { + name = ((PyTypeObject*)obj)->tp_name; + } auto ix = name.rfind('.'); @@ -42,12 +113,50 @@ Value::Value(const Instance& instance) : ix += 1; } - m_name = "Value(" + name.substr(ix) + ")"; + return "Value(" + name.substr(ix) + ")"; } else { - m_name = mInstance.repr(); + return mInstance.repr(); } +} + +void Value::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + Type* ownType = mInstance.extractType(true); + + auto it = groupMap.find(ownType); + + if (it != groupMap.end()) { + mInstance = Instance::create( + (PyObject*)PyInstance::typeObj(it->second) + ); + + mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } + } +} + +void Value::initializeDuringDeserialization(Instance i) { + mInstance = i; - m_doc = Value_doc; mValueAsPyobj = PyInstance::extractPythonObject(mInstance); - endOfConstructorInitialization(); // finish initializing the type object. + + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } +} + + +void Value::initializeFromConcrete(Type* forwardDefinitionOfSelf) { + mInstance = ((Value*)forwardDefinitionOfSelf)->mInstance.clone(); + mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } } diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index f75f8dcea..01b4eb623 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -20,12 +20,17 @@ PyDoc_STRVAR(Value_doc, "Value(x) -> type representing the single immutable value x" - ); +); class Value : public Type { public: - bool isBinaryCompatibleWithConcrete(Type* other) { - return this == other; + Value(const Instance& instance, PyObject* valueAsPyobj); + + // this is the clone-pathway + Value(); + + const char* docConcrete() { + return Value_doc; } template @@ -39,6 +44,11 @@ class Value : public Type { if (t != mInstance.type()) { throw std::runtime_error("visitor shouldn't have changed the type of a Value"); } + + t = mInstance.extractType(); + if (t) { + visitor(t); + } } bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { @@ -97,22 +107,23 @@ class Value : public Type { return mInstance; } - static Type* Make(Instance i) { - PyEnsureGilAcquired getTheGil; + static Type* Make(Instance i); - static std::map m; + void initializeDuringDeserialization(Instance i); + + void postInitializeConcrete() {} - auto it = m.find(i); + Type* cloneForForwardResolutionConcrete() { + return new Value(); + } - if (it == m.end()) { - it = m.insert(std::make_pair(i, new Value(i))).first; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf); - return it->second; - } + std::string computeRecursiveNameConcrete(TypeStack& typeStack); -private: - Value(const Instance& instance); + void updateInternalTypePointersConcrete( + const std::map& groupMap + ); Instance mInstance; PyObject* mValueAsPyobj; diff --git a/typed_python/VisibilityType.hpp b/typed_python/VisibilityType.hpp new file mode 100644 index 000000000..dbcaa8b0c --- /dev/null +++ b/typed_python/VisibilityType.hpp @@ -0,0 +1,39 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + + +enum class VisibilityType { + // compute the identity of the object as far as the interpreter is concerned, + // excluding any notion of pointer identity. This is used to memoize Type instances. + // We don't look into module-level members when computing this value, so this is valid + // as soon as a type object is constructed + Identity = 1, + // compute the identity of the object as far as the compiler itself is concerned. + // this is stronger than the notion of the Identity hash because the + Compiler = 2, +}; + +inline std::string visibilityTypeToStr(VisibilityType v) { + if (v == VisibilityType::Identity) { + return "Identity"; + } + if (v == VisibilityType::Compiler) { + return "Compiler"; + } + return "Unknown"; +} diff --git a/typed_python/__init__.py b/typed_python/__init__.py index ca8495770..4ff4edc2c 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.2.6" +__version__ = "0.2.7" # incremented every time the way we serialize a fully-typed # TP object changes. Consumers who only serialize fully-typed @@ -38,22 +38,24 @@ ) from typed_python._types import bytecount, refcount from typed_python.module import Module -from typed_python.type_function import TypeFunction from typed_python.hash import sha_hash -from typed_python.SerializationContext import SerializationContext -from typed_python.type_filter import TypeFilter from typed_python.compiler.typeof import TypeOf from typed_python._types import ( Forward, TupleOf, ListOf, Tuple, NamedTuple, OneOf, ConstDict, SubclassOf, Alternative, Value, serialize, deserialize, serializeStream, deserializeStream, PointerTo, RefTo, Dict, validateSerializedObject, validateSerializedObjectStream, decodeSerializedObject, getOrSetTypeResolver, Set, Class, Type, BoundMethod, - TypedCell, pointerTo, refTo, copy, identityHash, PythonObjectOfType, + TypedCell, pointerTo, refTo, copy, compilerHash, PythonObjectOfType, deepBytecount, deepcopy, deepcopyContiguous, totalBytesAllocatedInSlabs, deepBytecountAndSlabs, Slab, totalBytesAllocatedOnFreeStore, ModuleRepresentation, - setGilReleaseThreadLoopSleepMicroseconds + setGilReleaseThreadLoopSleepMicroseconds, + isForwardDefined, + typeLooksResolvable, + resolveForwardDefinedType, + identityHash, + forwardDefinitionsFor ) import typed_python._types as _types import threading @@ -71,6 +73,10 @@ EmbeddedMessage = _types.EmbeddedMessage() PyCell = _types.PyCell() +_types._enableTypeAutoresolution(True) + +from typed_python.type_function import TypeFunction +from typed_python.SerializationContext import SerializationContext # noqa from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa from typed_python.generator import Generator # noqa diff --git a/typed_python/_runtime.cpp b/typed_python/_runtime.cpp index 6a0d8cddd..6f7933668 100644 --- a/typed_python/_runtime.cpp +++ b/typed_python/_runtime.cpp @@ -478,7 +478,7 @@ extern "C" { } StringType::layout* np_typePtrToName(Type* t) { - return StringType::createFromString(t->name(true)); + return StringType::createFromString(t->name()); } StringType::layout* np_typePtrToQualName(Type* t) { diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index c2b52dfb6..545ab6b06 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -1,6 +1,5 @@ - /****************************************************************************** - Copyright 2017-2019 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,263 +35,339 @@ #include "UnicodeProps.hpp" #include "PyTemporaryReferenceTracer.hpp" #include "PySlab.hpp" +#include "PyFunctionOverload.hpp" +#include "PyFunctionGlobal.hpp" +#include "PyPyObjSnapshot.hpp" +#include "PyPyObjGraphSnapshot.hpp" #include "PyModuleRepresentation.hpp" #include "_types.hpp" #include "CompilerVisibleObjectVisitor.hpp" PyObject *MakeTupleOrListOfType(PyObject* nullValue, PyObject* args, bool isTuple) { - std::vector types; + return translateExceptionToPyObject([&]() { + std::vector types; - if (!unpackTupleToTypes(args, types)) { - return nullptr; - } + if (!unpackTupleToTypes(args, types)) { + throw PythonExceptionSet(); + } - if (types.size() != 1) { - if (isTuple) { - PyErr_SetString(PyExc_TypeError, "TupleOfType takes 1 positional argument."); - } else { - PyErr_SetString(PyExc_TypeError, "ListOfType takes 1 positional argument."); + if (types.size() != 1) { + if (isTuple) { + PyErr_SetString(PyExc_TypeError, "TupleOfType takes 1 positional argument."); + } else { + PyErr_SetString(PyExc_TypeError, "ListOfType takes 1 positional argument."); + } + throw PythonExceptionSet(); } - return NULL; - } - return incref( - (PyObject*)PyInstance::typeObj( - isTuple ? (TupleOrListOfType*)TupleOfType::Make(types[0]) : (TupleOrListOfType*)ListOfType::Make(types[0]) - ) - ); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + isTuple ? (TupleOrListOfType*)TupleOfType::Make(types[0]) : (TupleOrListOfType*)ListOfType::Make(types[0]) + ) + ) + ); + }); } PyObject *MakePointerToType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "PointerTo takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "PointerTo takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "PointerTo needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "PointerTo needs a type."); + throw PythonExceptionSet(); + } - return incref((PyObject*)PyInstance::typeObj(PointerTo::Make(t))); + return incref((PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + PointerTo::Make(t) + ) + ) + ); + }); } PyObject *MakeRefToType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "RefTo takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "RefTo takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "RefTo needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "RefTo needs a type."); + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]{ - return incref((PyObject*)PyInstance::typeObj(RefTo::Make(t))); + return translateExceptionToPyObject([&]{ + return incref((PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + RefTo::Make(t) + ) + )); + }); }); } PyObject *MakeSubclassOfType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "SubclassOf takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "SubclassOf takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "SubclassOf needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "SubclassOf needs a type."); + throw PythonExceptionSet(); + } - // types that can't be subclassed just produce values - if ((t->isClass() && !((Class*)t)->isFinal()) || - (t->isAlternative() && !((Class*)t)->isConcreteAlternative())) { - return translateExceptionToPyObject([&]{ - return incref((PyObject*)PyInstance::typeObj(SubclassOfType::Make(t))); - }); - } + // types that can't be subclassed just produce values + if ((t->isClass() && !((Class*)t)->isFinal()) || + (t->isAlternative() && !((Class*)t)->isConcreteAlternative())) { + return translateExceptionToPyObject([&]{ + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + SubclassOfType::Make(t) + ) + )); + }); + } - return MakeValueType(nullValue, args); + return MakeValueType(nullValue, args); + }); } PyObject *MakeTypedCellType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "TypedCell takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "TypedCell takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "TypedCell needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "TypedCell needs a type."); + throw PythonExceptionSet(); + } - return incref((PyObject*)PyInstance::typeObj(TypedCellType::Make(t))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + TypedCellType::Make(t) + ) + ) + ); + }); } PyObject *MakeTupleOfType(PyObject* nullValue, PyObject* args) { - return MakeTupleOrListOfType(nullValue, args, true); + return translateExceptionToPyObject([&]() { + return MakeTupleOrListOfType(nullValue, args, true); + }); } PyObject *MakeListOfType(PyObject* nullValue, PyObject* args) { - return MakeTupleOrListOfType(nullValue, args, false); + return translateExceptionToPyObject([&]() { + return MakeTupleOrListOfType(nullValue, args, false); + }); } PyObject *MakeTupleType(PyObject* nullValue, PyObject* args) { - std::vector types; - if (!unpackTupleToTypes(args, types)) { - return NULL; - } + return translateExceptionToPyObject([&]() { + std::vector types; + if (!unpackTupleToTypes(args, types)) { + throw PythonExceptionSet(); + } - return incref((PyObject*)PyInstance::typeObj(Tuple::Make(types))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + Tuple::Make(types) + ) + ) + ); + }); } PyObject *MakeConstDictType(PyObject* nullValue, PyObject* args) { - std::vector types; - for (long k = 0; k < PyTuple_Size(args); k++) { - PyObjectHolder item(PyTuple_GetItem(args,k)); - types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); - if (not types.back()) { - return NULL; + return translateExceptionToPyObject([&]() { + std::vector types; + for (long k = 0; k < PyTuple_Size(args); k++) { + PyObjectHolder item(PyTuple_GetItem(args,k)); + types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); + if (not types.back()) { + throw PythonExceptionSet(); + } } - } - if (types.size() != 2) { - PyErr_SetString(PyExc_TypeError, "ConstDict accepts two arguments"); - return NULL; - } + if (types.size() != 2) { + PyErr_SetString(PyExc_TypeError, "ConstDict accepts two arguments"); + throw PythonExceptionSet(); + } - PyObject* typeObj = (PyObject*)PyInstance::typeObj( - ConstDictType::Make(types[0],types[1]) + PyObject* typeObj = (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + ConstDictType::Make(types[0],types[1]) + ) ); - return incref(typeObj); + return incref(typeObj); + }); } PyObject* MakeSetType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args)!=1) { - PyErr_SetString(PyExc_TypeError, "Set takes 1 positional arguments"); - return NULL; - } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "Set needs a type."); - return NULL; - } - SetType* setT = SetType::Make(t); - return incref((PyObject*)PyInstance::typeObj(setT)); + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args)!=1) { + PyErr_SetString(PyExc_TypeError, "Set takes 1 positional arguments"); + throw PythonExceptionSet(); + } + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + if (!t) { + PyErr_SetString(PyExc_TypeError, "Set needs a type."); + throw PythonExceptionSet(); + } + SetType* setT = SetType::Make(t); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(setT) + ) + ); + }); } PyObject *MakeDictType(PyObject* nullValue, PyObject* args) { - std::vector types; - for (long k = 0; k < PyTuple_Size(args); k++) { - PyObjectHolder item(PyTuple_GetItem(args,k)); - types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); - if (not types.back()) { - return NULL; + return translateExceptionToPyObject([&]() { + std::vector types; + for (long k = 0; k < PyTuple_Size(args); k++) { + PyObjectHolder item(PyTuple_GetItem(args,k)); + types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); + if (not types.back()) { + throw PythonExceptionSet(); + } } - } - if (types.size() != 2) { - PyErr_SetString(PyExc_TypeError, "Dict accepts two arguments"); - return NULL; - } + if (types.size() != 2) { + PyErr_SetString(PyExc_TypeError, "Dict accepts two arguments"); + throw PythonExceptionSet(); + } - return incref( - (PyObject*)PyInstance::typeObj( - DictType::Make(types[0],types[1]) + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + DictType::Make(types[0],types[1]) + ) ) ); + }); } PyObject *MakeOneOfType(PyObject* nullValue, PyObject* args) { - std::vector types; - for (long k = 0; k < PyTuple_Size(args); k++) { - PyObjectHolder item(PyTuple_GetItem(args,k)); - - Type* t = PyInstance::tryUnwrapPyInstanceToType(item); - - if (t) { - types.push_back(t); - } else { - PyErr_Format(PyExc_TypeError, - "Type arguments must be types or simple values (like ints, strings, etc.), not %S. " - "If you need a more complex value (such as a type object itself), wrap it in 'Value'.", - (PyObject*)item - ); + return translateExceptionToPyObject([&]() { + std::vector types; + for (long k = 0; k < PyTuple_Size(args); k++) { + PyObjectHolder item(PyTuple_GetItem(args,k)); + + Type* t = PyInstance::tryUnwrapPyInstanceToType(item); + + if (t) { + types.push_back(t); + } else { + PyErr_Format(PyExc_TypeError, + "Type arguments must be types or simple values (like ints, strings, etc.), not %S. " + "If you need a more complex value (such as a type object itself), wrap it in 'Value'.", + (PyObject*)item + ); - return NULL; + throw PythonExceptionSet(); + } } - } - PyObject* typeObj = (PyObject*)PyInstance::typeObj(OneOfType::Make(types)); + PyObject* typeObj = (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + OneOfType::Make(types) + ) + ); - return incref(typeObj); + return incref(typeObj); + }); } PyObject *MakeNamedTupleType(PyObject* nullValue, PyObject* args, PyObject* kwargs) { - if (args && PyTuple_Check(args) && PyTuple_Size(args)) { - PyErr_SetString(PyExc_TypeError, "NamedTuple takes no positional arguments."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (args && PyTuple_Check(args) && PyTuple_Size(args)) { + PyErr_SetString(PyExc_TypeError, "NamedTuple takes no positional arguments."); + throw PythonExceptionSet(); + } - std::vector > namesAndTypes; + std::vector > namesAndTypes; - if (kwargs) { - PyObject *key, *value; - Py_ssize_t pos = 0; + if (kwargs) { + PyObject *key, *value; + Py_ssize_t pos = 0; - while (PyDict_Next(kwargs, &pos, &key, &value)) { - if (!PyUnicode_Check(key)) { - PyErr_SetString(PyExc_TypeError, "NamedTuple keywords are supposed to be strings."); - return NULL; - } + while (PyDict_Next(kwargs, &pos, &key, &value)) { + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, "NamedTuple keywords are supposed to be strings."); + throw PythonExceptionSet(); + } - namesAndTypes.push_back( - std::make_pair( - PyUnicode_AsUTF8(key), - PyInstance::unwrapTypeArgToTypePtr(value) - ) - ); + namesAndTypes.push_back( + std::make_pair( + PyUnicode_AsUTF8(key), + PyInstance::unwrapTypeArgToTypePtr(value) + ) + ); - if (not namesAndTypes.back().second) { - return NULL; + if (not namesAndTypes.back().second) { + throw PythonExceptionSet(); + } } } - } - if (PY_MINOR_VERSION <= 5) { - //we cannot rely on the ordering of 'kwargs' here because of the python version, so - //we sort it. this will be a problem for anyone running some processes using different - //python versions that share python code. - std::sort(namesAndTypes.begin(), namesAndTypes.end()); - } + if (PY_MINOR_VERSION <= 5) { + //we cannot rely on the ordering of 'kwargs' here because of the python version, so + //we sort it. this will be a problem for anyone running some processes using different + //python versions that share python code. + std::sort(namesAndTypes.begin(), namesAndTypes.end()); + } - std::vector names; - std::vector types; + std::vector names; + std::vector types; - for (auto p: namesAndTypes) { - names.push_back(p.first); - types.push_back(p.second); - } + for (auto p: namesAndTypes) { + names.push_back(p.first); + types.push_back(p.second); + } - return incref((PyObject*)PyInstance::typeObj(NamedTuple::Make(types, names))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + NamedTuple::Make(types, names) + ) + ) + ); + }); } @@ -454,7 +529,7 @@ PyObject *getCodeGlobalDotAccesses(PyObject* nullValue, PyObject* args, PyObject std::vector > v; - Function::Overload::extractDottedGlobalAccessesFromCode( + FunctionOverload::extractDottedGlobalAccessesFromCode( (PyCodeObject*)pyCodeObject, v ); @@ -603,18 +678,20 @@ PyObject* initializeGlobalStatics(PyObject* nullValue, PyObject* args, PyObject* return NULL; } - // initialize all global references to modules. - // we can't initialize these lazily, because it may happen when we are - // doing something that prevents importing modules, like handling - // an exception - internalsModule(); - runtimeModule(); - builtinsModule(); - sysModule(); - osModule(); - weakrefModule(); + return translateExceptionToPyObject([&]() { + // initialize all global references to modules. + // we can't initialize these lazily, because it may happen when we are + // doing something that prevents importing modules, like handling + // an exception + internalsModule(); + runtimeModule(); + builtinsModule(); + sysModule(); + osModule(); + weakrefModule(); - return incref(Py_None); + return incref(Py_None); + }); } PyObject* isValidArithmeticConversion(PyObject* nullValue, PyObject* args, PyObject* kwargs) @@ -792,7 +869,11 @@ PyObject *MakeValueType(PyObject* nullValue, PyObject* args) { Type* type = PyInstance::tryUnwrapPyInstanceToValueType(arg, true); if (type) { - return incref((PyObject*)PyInstance::typeObj(type)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(type) + ) + ); } PyErr_Format(PyExc_TypeError, "Couldn't convert %S to a value", (PyObject*)arg); @@ -817,7 +898,11 @@ PyObject *MakeBoundMethodType(PyObject* nullValue, PyObject* args) { Type* resType = BoundMethod::Make(t0, PyUnicode_AsUTF8(a1)); - return incref((PyObject*)PyInstance::typeObj(resType)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(resType) + ) + ); } PyObject *MakeAlternativeMatcherType(PyObject* nullValue, PyObject* args) { @@ -841,241 +926,309 @@ PyObject *MakeAlternativeMatcherType(PyObject* nullValue, PyObject* args) { Type* resType = AlternativeMatcher::Make((Alternative*)t0); - return incref((PyObject*)PyInstance::typeObj(resType)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(resType) + ) + ); } PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 6 && PyTuple_Size(args) != 2) { - PyErr_SetString(PyExc_TypeError, "Function takes 2 or 6 arguments"); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 6 && PyTuple_Size(args) != 2) { + PyErr_SetString(PyExc_TypeError, "Function takes 2 or 6 arguments"); + throw PythonExceptionSet(); + } - Function* resType; + Function* resType; - if (PyTuple_Size(args) == 2) { - PyObjectHolder a0(PyTuple_GetItem(args, 0)); - PyObjectHolder a1(PyTuple_GetItem(args, 1)); + if (PyTuple_Size(args) == 2) { + if (PyUnicode_Check(PyTuple_GetItem(args, 1))) { + PyObjectHolder a0(PyTuple_GetItem(args, 0)); + Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); - Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); - Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); + if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); + throw PythonExceptionSet(); + } - if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); - return NULL; - } - if (!t1 || t1->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_SetString(PyExc_TypeError, "Expected second argument to be a function"); - return NULL; - } + if (PyUnicode_AsUTF8(PyTuple_GetItem(args, 1)) == std::string("Entrypoint")) { + resType = ((Function*)t0)->withEntrypoint(true); + } else if (PyUnicode_AsUTF8(PyTuple_GetItem(args, 1)) == std::string("Nocompile")) { + resType = ((Function*)t0)->withEntrypoint(true); + } else { + PyErr_SetString(PyExc_TypeError, "Expected string argument to be Entrypoint or Nocompile"); + throw PythonExceptionSet(); + } + } else { + PyObjectHolder a0(PyTuple_GetItem(args, 0)); + PyObjectHolder a1(PyTuple_GetItem(args, 1)); - resType = Function::merge((Function*)t0, (Function*)t1); - } else { - PyObjectHolder nameObj(PyTuple_GetItem(args, 0)); - if (!PyUnicode_Check(nameObj)) { - PyErr_SetString(PyExc_TypeError, "First arg should be a string."); - return NULL; - } - PyObjectHolder qualnameObj(PyTuple_GetItem(args, 1)); - if (!PyUnicode_Check(qualnameObj)) { - PyErr_SetString(PyExc_TypeError, "First arg should be a string."); - return NULL; - } - PyObjectHolder retType(PyTuple_GetItem(args, 2)); - PyObjectHolder funcObj(PyTuple_GetItem(args, 3)); - PyObjectHolder argTuple(PyTuple_GetItem(args, 4)); + Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); + Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); - int assumeClosureGlobal = PyObject_IsTrue(PyTuple_GetItem(args, 5)); + if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); + throw PythonExceptionSet(); + } + if (!t1 || t1->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected second argument to be a function"); + throw PythonExceptionSet(); + } - if (assumeClosureGlobal == -1) { - return NULL; - } + resType = Function::merge((Function*)t0, (Function*)t1); + } + } else { + PyObjectHolder nameObj(PyTuple_GetItem(args, 0)); + if (!PyUnicode_Check(nameObj)) { + PyErr_SetString(PyExc_TypeError, "First arg should be a string."); + throw PythonExceptionSet(); + } + PyObjectHolder qualnameObj(PyTuple_GetItem(args, 1)); + if (!PyUnicode_Check(qualnameObj)) { + PyErr_SetString(PyExc_TypeError, "First arg should be a string."); + throw PythonExceptionSet(); + } + PyObjectHolder retType(PyTuple_GetItem(args, 2)); + PyObjectHolder funcObj(PyTuple_GetItem(args, 3)); + PyObjectHolder argTuple(PyTuple_GetItem(args, 4)); - if (!PyFunction_Check(funcObj)) { - PyErr_SetString(PyExc_TypeError, "Third arg should be a function object"); - return NULL; - } + int assumeClosureGlobal = PyObject_IsTrue(PyTuple_GetItem(args, 5)); - Type* rType = 0; - PyObject* rSignature = 0; + if (assumeClosureGlobal == -1) { + throw PythonExceptionSet(); + } - if (retType != Py_None) { - rType = PyInstance::unwrapTypeArgToTypePtr(retType); - if (!rType) { - if (!PyCallable_Check(retType)) { - PyErr_SetString(PyExc_TypeError, "Expected second argument to be None, a type, or a callable type signature function"); - return NULL; - } - PyErr_Clear(); - rSignature = retType; + if (!PyFunction_Check(funcObj)) { + PyErr_SetString(PyExc_TypeError, "Third arg should be a function object"); + throw PythonExceptionSet(); } - } - if (!PyTuple_Check(argTuple)) { - PyErr_SetString(PyExc_TypeError, "Expected fourth argument to be a tuple of args"); - return NULL; - } + Type* rType = 0; + PyObject* rSignature = 0; - std::vector argList; + if (retType != Py_None) { + rType = PyInstance::unwrapTypeArgToTypePtr(retType); + if (!rType) { + if (!PyCallable_Check(retType)) { + PyErr_SetString(PyExc_TypeError, "Expected second argument to be None, a type, or a callable type signature function"); + throw PythonExceptionSet(); + } + PyErr_Clear(); + rSignature = retType; + } + } - for (long k = 0; k < PyTuple_Size(argTuple); k++) { - PyObjectHolder kTup(PyTuple_GetItem(argTuple, k)); - if (!PyTuple_Check(kTup) || PyTuple_Size(kTup) != 5) { - PyErr_SetString(PyExc_TypeError, "Argtuple elements should be tuples of five things."); - return NULL; + if (!PyTuple_Check(argTuple)) { + PyErr_SetString(PyExc_TypeError, "Expected fourth argument to be a tuple of args"); + throw PythonExceptionSet(); } - PyObjectHolder k0(PyTuple_GetItem(kTup, 0)); - PyObjectHolder k1(PyTuple_GetItem(kTup, 1)); - PyObjectHolder k2(PyTuple_GetItem(kTup, 2)); - PyObjectHolder k3(PyTuple_GetItem(kTup, 3)); - PyObjectHolder k4(PyTuple_GetItem(kTup, 4)); + std::vector argList; - if (!PyUnicode_Check(k0)) { - PyErr_Format(PyExc_TypeError, "Argument %S has a name which is not a string.", (PyObject*)k0); - return NULL; - } + for (long k = 0; k < PyTuple_Size(argTuple); k++) { + PyObjectHolder kTup(PyTuple_GetItem(argTuple, k)); + if (!PyTuple_Check(kTup) || PyTuple_Size(kTup) != 5) { + PyErr_SetString(PyExc_TypeError, "Argtuple elements should be tuples of five things."); + throw PythonExceptionSet(); + } + + PyObjectHolder k0(PyTuple_GetItem(kTup, 0)); + PyObjectHolder k1(PyTuple_GetItem(kTup, 1)); + PyObjectHolder k2(PyTuple_GetItem(kTup, 2)); + PyObjectHolder k3(PyTuple_GetItem(kTup, 3)); + PyObjectHolder k4(PyTuple_GetItem(kTup, 4)); - Type* argT = nullptr; - if (k1 != Py_None) { - argT = PyInstance::unwrapTypeArgToTypePtr(k1); - if (!argT) { - PyErr_Format(PyExc_TypeError, "Argument %S has a type argument %S which should be None or a Type.", k0->ob_type, k1->ob_type); - return NULL; + if (!PyUnicode_Check(k0)) { + PyErr_Format(PyExc_TypeError, "Argument %S has a name which is not a string.", (PyObject*)k0); + throw PythonExceptionSet(); } - } - if ((k3 != Py_True && k3 != Py_False) || (k4 != Py_True && k4 != Py_False)) { - PyErr_Format(PyExc_TypeError, "Argument %S has a malformed type tuple", (PyObject*)k0); - return NULL; - } + Type* argT = nullptr; + if (k1 != Py_None) { + argT = PyInstance::unwrapTypeArgToTypePtr(k1); + if (!argT) { + PyErr_Format(PyExc_TypeError, "Argument %S has a type argument %S which should be None or a Type.", k0->ob_type, k1->ob_type); + throw PythonExceptionSet(); + } + } - PyObject* val = nullptr; - if (k2 != Py_None) { - if (!PyTuple_Check(k2) || PyTuple_Size(k2) != 1) { + if ((k3 != Py_True && k3 != Py_False) || (k4 != Py_True && k4 != Py_False)) { PyErr_Format(PyExc_TypeError, "Argument %S has a malformed type tuple", (PyObject*)k0); - return NULL; + throw PythonExceptionSet(); } - val = PyTuple_GetItem(k2,0); - } + PyObject* val = nullptr; + if (k2 != Py_None) { + if (!PyTuple_Check(k2) || PyTuple_Size(k2) != 1) { + PyErr_Format(PyExc_TypeError, "Argument %S has a malformed type tuple", (PyObject*)k0); + throw PythonExceptionSet(); + } - if (val) { - incref(val); - } + val = PyTuple_GetItem(k2,0); + } - argList.push_back(Function::FunctionArg( - PyUnicode_AsUTF8(k0), - argT, - val, - k3 == Py_True, - k4 == Py_True - )); - } + if (val) { + incref(val); + } - std::string moduleName = ""; + argList.push_back(FunctionArg( + PyUnicode_AsUTF8(k0), + argT, + val, + k3 == Py_True, + k4 == Py_True + )); + } - if (PyObject_HasAttrString(funcObj, "__module__")) { - PyObject* pyModulename = PyObject_GetAttrString(funcObj, "__module__"); + std::string moduleName = ""; - if (!pyModulename) { - return NULL; - } + if (PyObject_HasAttrString(funcObj, "__module__")) { + PyObject* pyModulename = PyObject_GetAttrString(funcObj, "__module__"); - if (PyUnicode_Check(pyModulename)) { - moduleName = PyUnicode_AsUTF8(pyModulename); - } + if (!pyModulename) { + throw PythonExceptionSet(); + } - decref(pyModulename); - } + if (PyUnicode_Check(pyModulename)) { + moduleName = PyUnicode_AsUTF8(pyModulename); + } - std::vector overloads; + decref(pyModulename); + } - std::vector closureVarnames; - std::vector closureVarTypes; - std::map globalsInCells; - std::map closureBindings; + std::vector overloads; - PyObject* closure = PyFunction_GetClosure(funcObj); + std::vector closureVarnames; + std::vector closureTupleVarnames; + std::vector globalsInClosureVarnames; + std::vector closureVarTypes; + std::map closureBindings; + std::map globals; - if (closure) { - PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); + PyObject* closure = PyFunction_GetClosure(funcObj); - if (!coFreevars) { - return NULL; - } + if (closure) { + PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); - if (!PyTuple_Check(coFreevars)) { - PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not a tuple"); - return NULL; - } + if (!coFreevars) { + throw PythonExceptionSet(); + } - if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { - PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars had a different number of elements than the closure"); - return NULL; - } + if (!PyTuple_Check(coFreevars)) { + PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not a tuple"); + throw PythonExceptionSet(); + } - for (long ix = 0; ix < PyTuple_Size(coFreevars); ix++) { - PyObject* varname = PyTuple_GetItem(coFreevars, ix); - if (!PyUnicode_Check(varname)) { - PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not all strings"); - return NULL; + if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { + PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars had a different number of elements than the closure"); + throw PythonExceptionSet(); } - closureVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); - } - if (assumeClosureGlobal) { - for (long ix = 0; ix < PyTuple_Size(closure); ix++) { - PyObject* cell = PyTuple_GetItem(closure, ix); - if (!PyCell_Check(cell)) { - PyErr_Format(PyExc_TypeError, "Function closure needs to all be cells."); - return NULL; + for (long ix = 0; ix < PyTuple_Size(coFreevars); ix++) { + PyObject* varname = PyTuple_GetItem(coFreevars, ix); + if (!PyUnicode_Check(varname)) { + PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not all strings"); + throw PythonExceptionSet(); } + closureVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); + closureTupleVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); + } + + if (assumeClosureGlobal) { + for (long ix = 0; ix < PyTuple_Size(closure); ix++) { + PyObject* cell = PyTuple_GetItem(closure, ix); + if (!PyCell_Check(cell)) { + PyErr_Format(PyExc_TypeError, "Function closure needs to all be cells."); + throw PythonExceptionSet(); + } - globalsInCells[closureVarnames[ix]] = incref(cell); + globals[closureVarnames[ix]] = FunctionGlobal::GlobalInCell(cell); + } + } + else { + for (long ix = 0; ix < PyTuple_Size(closure); ix++) { + closureVarTypes.push_back(PyCellType::Make()); + closureBindings[closureVarnames[ix]] = ( + ClosureVariableBinding() + 0 + ix + ClosureVariableBindingStep::AccessCell() + ); + } } } - else { - for (long ix = 0; ix < PyTuple_Size(closure); ix++) { - closureVarTypes.push_back(PyCellType::Make()); - closureBindings[closureVarnames[ix]] = ( - ClosureVariableBinding() + 0 + ix + ClosureVariableBindingStep::AccessCell() - ); + + if (!assumeClosureGlobal + && !CompilerVisibleObjectVisitor::isPyObjectGloballyIdentifiableModuleDict( + PyFunction_GetGlobals(funcObj) + ).size() + ) { + // this function has non-identifiable globals, so we're actually going to place the + // globals dict into the closure so that its part of the 'data' rather than the type + // of the class + closureVarTypes.push_back(PythonObjectOfType::AnyPyDict()); + // using a space in the name prevents this from colliding with an actual + // variable name + closureTupleVarnames.push_back(" _globals"); + + // ensure that every global lookup is now routed through the function overload + std::set allNamesString; + FunctionOverload::extractGlobalAccessesFromCode( + (PyCodeObject*)PyFunction_GetCode(funcObj), + allNamesString + ); + + for (auto name: allNamesString) { + globalsInClosureVarnames.push_back(name); + closureBindings[name] = + ClosureVariableBinding() + + 0 + + ClosureVariableBindingStep(" _globals") + + ClosureVariableBindingStep(name); } + } else { + FunctionOverload::buildInitialGlobalsDict( + globals, + PyFunction_GetGlobals(funcObj), + (PyCodeObject*)PyFunction_GetCode(funcObj) + ); } + + overloads.push_back( + FunctionOverload( + PyFunction_GetCode(funcObj), + PyFunction_GetDefaults(funcObj), + ((PyFunctionObject*)(PyObject*)funcObj)->func_annotations, + globals, + closureVarnames, + globalsInClosureVarnames, + closureBindings, + rType, + rSignature, + argList, + nullptr + ) + ); + + resType = Function::Make( + PyUnicode_AsUTF8(nameObj), + PyUnicode_AsUTF8(qualnameObj), + moduleName, + overloads, + Tuple::Make({ + assumeClosureGlobal ? + NamedTuple::Make({}, {}) : + NamedTuple::Make(closureVarTypes, closureTupleVarnames) + }), + false, + false + ); } - overloads.push_back( - Function::Overload( - PyFunction_GetCode(funcObj), - PyFunction_GetGlobals(funcObj), - PyFunction_GetDefaults(funcObj), - ((PyFunctionObject*)(PyObject*)funcObj)->func_annotations, - globalsInCells, - closureVarnames, - closureBindings, - rType, - rSignature, - argList, - nullptr + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(resType) ) ); - - resType = Function::Make( - PyUnicode_AsUTF8(nameObj), - PyUnicode_AsUTF8(qualnameObj), - moduleName, - overloads, - Tuple::Make({ - assumeClosureGlobal ? - NamedTuple::Make({}, {}) : - NamedTuple::Make(closureVarTypes, closureVarnames) - }), - false, - false - ); - } - - return incref((PyObject*)PyInstance::typeObj(resType)); + }); } PyObject *MakeClassType(PyObject* nullValue, PyObject* args) { @@ -1249,20 +1402,18 @@ PyObject *MakeClassType(PyObject* nullValue, PyObject* args) { return translateExceptionToPyObject([&]() { return incref( (PyObject*)PyInstance::typeObj( - Class::Make( - name, - baseClasses, - final == Py_True, - members, - memberFuncs, - staticFuncs, - propertyFuncs, - clsMembers, - classMethodFuncs, - // this is the first time this class exists, - // so we need the constructor to rebuild the - // function objects with apprioriated methodOf - true + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + Class::Make( + name, + baseClasses, + final == Py_True, + members, + memberFuncs, + staticFuncs, + propertyFuncs, + clsMembers, + classMethodFuncs + ) ) ) ); @@ -2057,6 +2208,54 @@ PyObject *createPyCell(PyObject* nullValue, PyObject* args) { return PyCell_New(PyTuple_GetItem(args, 0)); } +PyDoc_STRVAR( + compilerHash_doc, + "compilerHash(T) -> str\n\n" + "Compute the 'compiler hash' of a type or function T.\n\n" + "The 'compilerHash' of a function or Type is a sha-hash of all objects visible to the\n" + "compiler starting from 'T'. If two python objects have the same compiler hash,\n" + "then the compiler will produce identical code for them.\n\n" + "The compilerHash is computed by walking into a type or function and looking at all\n" + "the objects that the compiler would be willing to look at. For instance, if 'f' calls\n" + "a function 'g' and 'g' is visibile to 'f' through its module dictionary, the compiler\n" + "will know to compile and call 'f' and so 'f' will be included in this walk. In contrast,\n" + "if 'f' accesses a list object through its closure, the contents of that list will not be\n" + "contained in the cache since the contents of the list could change.\n" +); +PyObject *compilerHash(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "compilerHash takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + ShaHash hash; + + return translateExceptionToPyObject([&]() { + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)a1, + VisibilityType::Compiler + ); + hash = MutuallyRecursiveTypeGroup::shaHash( + (PyObject*)a1, + VisibilityType::Compiler + ); + + return PyBytes_FromStringAndSize((const char*)&hash, sizeof(ShaHash)); + }); +} + +PyDoc_STRVAR( + identityHash_doc, + "identityHash(T) -> str\n\n" + "Compute the 'identity hash' of a type or function T.\n\n" + "The 'identityHash' of a function or Type is a sha-hash of all non-mutable content\n" + "reachable from the object. If two Type objects have the same identityHash then they\n" + "will in fact be identical instances.\n\n" + "The identityHash is computed similarly to the compilerHash, except that we don't look\n" + "inside of modules. This allows our identity hash to be stable even as a module\n" + "is being defined." +); PyObject *identityHash(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "identityHash takes 1 positional argument"); @@ -2067,8 +2266,14 @@ PyObject *identityHash(PyObject* nullValue, PyObject* args) { ShaHash hash; return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - hash = MutuallyRecursiveTypeGroup::shaHash((PyObject*)a1); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)a1, + VisibilityType::Identity + ); + hash = MutuallyRecursiveTypeGroup::shaHash( + (PyObject*)a1, + VisibilityType::Identity + ); return PyBytes_FromStringAndSize((const char*)&hash, sizeof(ShaHash)); }); @@ -2370,58 +2575,174 @@ PyObject *deserializeStream(PyObject* nullValue, PyObject* args) { PyObjectHolder a2(PyTuple_GetItem(args, 1)); PyObjectHolder a3(PyTuple_Size(args) == 3 ? PyTuple_GetItem(args, 2) : nullptr); - Type* serializeType = PyInstance::unwrapTypeArgToTypePtr(a1); + Type* serializeType = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!serializeType) { + PyErr_SetString(PyExc_TypeError, "first argument to deserialize must be a type object"); + return NULL; + } + if (!PyBytes_Check(a2)) { + PyErr_SetString(PyExc_TypeError, "second argument to deserialize must be a bytes object"); + return NULL; + } + + std::shared_ptr context(new NullSerializationContext()); + if (a3 && a3 != Py_None) { + context.reset(new PythonSerializationContext(a3)); + } + + DeserializationBuffer buf((uint8_t*)PyBytes_AsString(a2), PyBytes_GET_SIZE((PyObject*)a2), *context); + + try { + serializeType->assertForwardsResolved(); + + TupleOfType* tupType = TupleOfType::Make(serializeType); + + tupType->assertForwardsResolved(); + + Instance i = Instance::createAndInitialize(tupType, [&](instance_ptr p) { + PyEnsureGilReleased releaseTheGil; + + tupType->constructorUnbounded(p, [&](instance_ptr tupElt, int index) { + if (buf.isDone()) { + return false; + } + + auto fieldAndWireType = buf.readFieldNumberAndWireType(); + serializeType->deserialize(tupElt, buf, fieldAndWireType.second); + + return true; + }); + }); + + return PyInstance::extractPythonObject(i.data(), i.type()); + } catch(std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + return NULL; + } catch(PythonExceptionSet& e) { + return NULL; + } +} + +PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "isForwardDefined takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + return translateExceptionToPyObject([&]() { + // Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->isForwardDefined() ? Py_True : Py_False); + // } + + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + throw std::runtime_error("first argument to 'isForwardDefined' must be a type object"); + } + + return incref(t->isForwardDefined() ? Py_True : Py_False); + }); +} + +PyObject *isResolved(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "isResolved takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + return translateExceptionToPyObject([&]() { + // Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->isResolved() ? Py_True : Py_False); + // } + + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + throw std::runtime_error("first argument to 'isResolved' must be a type object"); + } + + return incref(t->isResolved() ? Py_True : Py_False); + }); +} + +PyObject *enableTypeAutoresolution(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + int enabled = 1; + + static const char *kwlist[] = {"enabled", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|p", (char**)kwlist, &enabled)) { + throw PythonExceptionSet(); + } + + PyTemporaryReferenceTracer::globalTracer.enableTypeAutoresolution(enabled); + + return incref(Py_None); + }); +} + +PyObject *typeLooksResolvable(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* type; + int unambiguously = 0; - if (!serializeType) { - PyErr_SetString(PyExc_TypeError, "first argument to deserialize must be a type object"); - return NULL; - } - if (!PyBytes_Check(a2)) { - PyErr_SetString(PyExc_TypeError, "second argument to deserialize must be a bytes object"); - return NULL; - } + static const char *kwlist[] = {"type", "unambiguously", NULL}; - std::shared_ptr context(new NullSerializationContext()); - if (a3 && a3 != Py_None) { - context.reset(new PythonSerializationContext(a3)); - } + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|p", (char**)kwlist, &type, &unambiguously)) { + throw PythonExceptionSet(); + } - DeserializationBuffer buf((uint8_t*)PyBytes_AsString(a2), PyBytes_GET_SIZE((PyObject*)a2), *context); + // see first if its an unclosured function object + // Type* typeOfArg = PyInstance::extractTypeFrom(type->ob_type, false); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->looksResolvable(unambiguously) ? Py_True : Py_False); + // } - try { - serializeType->assertForwardsResolved(); + Type* t = PyInstance::unwrapTypeArgToTypePtr(type); - TupleOfType* tupType = TupleOfType::Make(serializeType); + if (!t) { + throw std::runtime_error("type must be a TP type instance"); + } - tupType->assertForwardsResolved(); + return incref(t->looksResolvable(unambiguously) ? Py_True : Py_False); + }); +} - Instance i = Instance::createAndInitialize(tupType, [&](instance_ptr p) { - PyEnsureGilReleased releaseTheGil; +PyObject *assertTypeResolvable(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* type; + int unambiguously = 0; - tupType->constructorUnbounded(p, [&](instance_ptr tupElt, int index) { - if (buf.isDone()) { - return false; - } + static const char *kwlist[] = {"type", "unambiguously", NULL}; - auto fieldAndWireType = buf.readFieldNumberAndWireType(); - serializeType->deserialize(tupElt, buf, fieldAndWireType.second); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|p", (char**)kwlist, &type, &unambiguously)) { + throw PythonExceptionSet(); + } - return true; - }); - }); + // see first if its an unclosured function object + // Type* typeOfArg = PyInstance::extractTypeFrom(type->ob_type, false); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->looksResolvable(unambiguously) ? Py_True : Py_False); + // } - return PyInstance::extractPythonObject(i.data(), i.type()); - } catch(std::exception& e) { - PyErr_SetString(PyExc_TypeError, e.what()); - return NULL; - } catch(PythonExceptionSet& e) { - return NULL; - } + Type* t = PyInstance::unwrapTypeArgToTypePtr(type); + + if (!t) { + throw std::runtime_error("type must be a TP type instance"); + } + + return incref(t->assertResolvable(unambiguously) ? Py_True : Py_False); + }); } -PyObject *allForwardTypesResolved(PyObject* nullValue, PyObject* args) { +PyObject *forwardDefinitionsFor(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "allForwardTypesResolved takes 1 positional argument"); + PyErr_SetString(PyExc_TypeError, "forwardDefinitionsFor takes 1 positional argument"); return NULL; } PyObjectHolder a1(PyTuple_GetItem(args, 0)); @@ -2429,31 +2750,47 @@ PyObject *allForwardTypesResolved(PyObject* nullValue, PyObject* args) { Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); if (!t) { - PyErr_SetString( - PyExc_TypeError, - "first argument to 'allForwardTypesResolved' must be a type object" - ); + PyErr_SetString(PyExc_TypeError, "first argument to 'forwardDefinitionsFor' must be a type object"); return NULL; } - return incref(t->resolved() ? Py_True : Py_False); + return ::translateExceptionToPyObject([&]() { + PyObject* res = PyList_New(0); + + for (auto fwd: t->getForwardDefinitions()) { + PyList_Append( + res, + (PyObject*)PyInstance::typeObj(fwd) + ); + } + + return res; + }); } -PyObject *isSimple(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "isSimple takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { + return ::translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + throw std::runtime_error("resolveForwardDefinedType takes 1 positional argument"); + } - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + PyObjectHolder a1(PyTuple_GetItem(args, 0)); - if (!t) { - PyErr_SetString(PyExc_TypeError, "first argument to 'isSimple' must be a type object"); - return NULL; - } + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + throw std::runtime_error( + "first argument to 'resolveForwardDefinedType' must be a type object" + ); + } - return incref(t->isSimple() ? Py_True : Py_False); + return incref( + (PyObject*) + PyInstance::typeObj( + t->forwardResolvesTo() + ) + ); + }); } PyObject *isPOD(PyObject* nullValue, PyObject* args) { @@ -2535,21 +2872,27 @@ PyObject *wantsToDefaultConstruct(PyObject* nullValue, PyObject* args) { return incref(HeldClass::wantsToDefaultConstruct(t) ? Py_True : Py_False); } -PyObject *bytecount(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "bytecount takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *bytecount(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* type; + static const char *kwlist[] = {"type", NULL}; - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", (char**)kwlist, &type)) { + throw PythonExceptionSet(); + } - if (!t) { - PyErr_SetString(PyExc_TypeError, "first argument to 'bytecount' must be a type object"); - return NULL; - } + Type* t = PyInstance::unwrapTypeArgToTypePtr(type); + + if (!t) { + throw std::runtime_error("type must be a TP type instance"); + } + + if (t->isForwardDefined()) { + throw std::runtime_error("type must not be a ForwardDefined type."); + } - return PyLong_FromLong(t->bytecount()); + return PyLong_FromLong(t->bytecount()); + }); } PyObject *typesAreEquivalent(PyObject* nullValue, PyObject* args) { @@ -2682,18 +3025,28 @@ PyObject *touchCompiledSpecializations(PyObject* nullValue, PyObject* args) { return incref(Py_None); } -PyObject *isRecursive(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "isRecursive takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *isRecursive(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + MutuallyRecursiveTypeGroup* group = MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; if (!group) { return incref(Py_False); @@ -2731,10 +3084,27 @@ PyDoc_STRVAR( "errors while computing hashes because the assumptions of the hasher are violated." ); -PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args) { +PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { + const char* visibility = "compiler"; + static const char *kwlist[] = {"visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|s", (char**)kwlist, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + try { - CompilerVisibleObjectVisitor::singleton().checkForInstability(); + CompilerVisibleObjectVisitor::singleton().checkForInstability(vis); return incref(Py_None); } catch(std::runtime_error& err) { @@ -2743,33 +3113,59 @@ PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args) { }); } -PyObject *typeWalkRecord(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "typeWalkRecord takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - +PyObject *typeWalkRecord(PyObject* nullValue, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { - TypeOrPyobj obj((PyObject*)a1); + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + TypeOrPyobj obj(instance); return PyUnicode_FromString( - ("Walk of " + obj.name() + ":\n\n" + CompilerVisibleObjectVisitor::recordWalkAsString(obj)).c_str() + ("Walk of " + obj.name() + ":\n\n" + + CompilerVisibleObjectVisitor::recordWalkAsString(obj, vis) + ).c_str() ); }); } -PyObject *typesAndObjectsVisibleToCompilerFrom(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "typesAndObjectsVisibleToCompilerFrom takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - +PyObject *typesAndObjectsVisibleToCompilerFrom(PyObject* nullValue, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + std::vector visible; - MutuallyRecursiveTypeGroup::visibleFrom((PyObject*)a1, visible); + MutuallyRecursiveTypeGroup::visibleFrom(instance, visible, vis); PyObjectStealer res(PyList_New(0)); @@ -2786,26 +3182,39 @@ PyObject *typesAndObjectsVisibleToCompilerFrom(PyObject* nullValue, PyObject* ar }); } -PyObject *recursiveTypeGroupHash(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "recursiveTypeGroupHash takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *recursiveTypeGroupHash(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } - MutuallyRecursiveTypeGroup* group = nullptr; + MutuallyRecursiveTypeGroup* group = nullptr; + + if (PyType_Check(instance) && vis == VisibilityType::Identity) { + Type* typeArg = PyInstance::extractTypeFrom((PyTypeObject*)instance); - return translateExceptionToPyObject([&]() { - if (PyType_Check(a1)) { - Type* typeArg = PyInstance::extractTypeFrom(a1); if (typeArg) { group = typeArg->getRecursiveTypeGroup(); } } if (!group) { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + group = MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; } if (!group) { @@ -2816,18 +3225,30 @@ PyObject *recursiveTypeGroupHash(PyObject* nullValue, PyObject* args) { }); } -PyObject *recursiveTypeGroupReprDeepFlag(PyObject* nullValue, PyObject* args, bool deep) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "recursiveTypeGroupRepr takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *recursiveTypeGroupReprDeepFlag(PyObject* nullValue, PyObject* args, PyObject* kwargs, bool deep) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + MutuallyRecursiveTypeGroup* group = nullptr; + + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + group = MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; if (!group) { return incref(Py_None); @@ -2837,28 +3258,43 @@ PyObject *recursiveTypeGroupReprDeepFlag(PyObject* nullValue, PyObject* args, bo }); } -PyObject *recursiveTypeGroupRepr(PyObject* nullValue, PyObject* args) { - return recursiveTypeGroupReprDeepFlag(nullValue, args, false); +PyObject *recursiveTypeGroupRepr(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return recursiveTypeGroupReprDeepFlag(nullValue, args, kwargs, false); } -PyObject *recursiveTypeGroupDeepRepr(PyObject* nullValue, PyObject* args) { - return recursiveTypeGroupReprDeepFlag(nullValue, args, true); +PyObject *recursiveTypeGroupDeepRepr(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return recursiveTypeGroupReprDeepFlag(nullValue, args, kwargs, true); } -PyObject *recursiveTypeGroup(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "recursiveTypeGroup takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *recursiveTypeGroup(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + int createIfNotExist = 1; + static const char *kwlist[] = {"instance", "visibility", "createIfNotExist", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|sp", (char**)kwlist, &instance, &visibility, &createIfNotExist)) { + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + if (createIfNotExist) { + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + } + + MutuallyRecursiveTypeGroup* group = + MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; if (!group) { return incref(Py_None); @@ -2907,63 +3343,38 @@ PyObject *referencedTypes(PyObject* nullValue, PyObject* args) { }); } -PyObject *isBinaryCompatible(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 2) { - PyErr_SetString(PyExc_TypeError, "isBinaryCompatible takes 2 positional arguments"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - PyObjectHolder a2(PyTuple_GetItem(args, 1)); - - Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); - Type* t2 = PyInstance::unwrapTypeArgToTypePtr(a2); - - if (!t1) { - PyErr_SetString(PyExc_TypeError, "first argument to 'isBinaryCompatible' must be a type object"); - return NULL; - } - if (!t2) { - PyErr_SetString(PyExc_TypeError, "second argument to 'isBinaryCompatible' must be a type object"); - return NULL; - } - - return incref(t1->isBinaryCompatibleWith(t2) ? Py_True : Py_False); -} - -PyObject *MakeForward(PyObject* nullValue, PyObject* args) { - int num_args = PyTuple_Size(args); - - PyThreadState * ts = PyThreadState_Get(); - std::string moduleName; - PyObject* pyModuleName; +PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + int num_args = PyTuple_Size(args); - if (ts->frame && ts->frame->f_globals && - (pyModuleName = PyDict_GetItemString(ts->frame->f_globals, "__name__"))) { - if (PyUnicode_Check(pyModuleName)) { - moduleName = PyUnicode_AsUTF8(pyModuleName); + if (num_args == 1 && PyUnicode_Check(PyTuple_GetItem(args, 0))) { + return incref((PyObject*)PyInstance::typeObj( + Forward::Make( + PyUnicode_AsUTF8(PyTuple_GetItem(args,0)) + ) + )); } - } - if (num_args > 1 || !PyUnicode_Check(PyTuple_GetItem(args,0))) { - PyErr_SetString(PyExc_TypeError, "Forward takes a zero or one string positional arguments."); - return NULL; - } + if (num_args == 1 && PyFunction_Check(PyTuple_GetItem(args, 0))) { + return incref((PyObject*)PyInstance::typeObj( + Forward::MakeFromFunction(PyTuple_GetItem(args, 0)) + )); + } - if (num_args == 1) { - std::string name = PyUnicode_AsUTF8(PyTuple_GetItem(args,0)); + if (num_args == 0) { + return incref((PyObject*)PyInstance::typeObj( + Forward::Make() + )); + } - if (moduleName.size()) { - name = moduleName + "." + name; + if (num_args > 1) { + throw std::runtime_error("Can't make a Forward with more than 1 argument."); } - return incref((PyObject*)PyInstance::typeObj( - ::Forward::Make(name) - )); - } else { - return incref((PyObject*)PyInstance::typeObj( - ::Forward::Make() - )); - } + throw std::runtime_error( + "Forward requires either a string or a function with a simple name lookup." + ); + }); } /********* @@ -3183,7 +3594,13 @@ PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwa std::string fieldName(PyUnicode_AsUTF8(key)); if (PyFunction_Check(value)) { - functions[fieldName] = PyFunctionInstance::convertPythonObjectToFunctionType(key, value, true, false); + functions[fieldName] = PyFunctionInstance::convertPythonObjectToFunctionType( + key, + PyTuple_GetItem(args, 0), + value, + true, + false + ); if (functions[fieldName] == nullptr) { //error code is already set @@ -3229,9 +3646,13 @@ PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwa std::sort(definitions.begin(), definitions.end()); } - return incref((PyObject*)PyInstance::typeObj( - ::Alternative::Make(name, moduleName, definitions, functions) - )); + Type* t = Alternative::Make(name, moduleName, definitions, functions); + + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(t) + ) + ); } PyObject *getTypePointer(PyObject* nullValue, PyObject* args) { @@ -3289,27 +3710,31 @@ static PyMethodDef module_methods[] = { {"decodeSerializedObject", (PyCFunction)decodeSerializedObject, METH_VARARGS, NULL}, {"validateSerializedObject", (PyCFunction)validateSerializedObject, METH_VARARGS, NULL}, {"validateSerializedObjectStream", (PyCFunction)validateSerializedObjectStream, METH_VARARGS, NULL}, - {"identityHash", (PyCFunction)identityHash, METH_VARARGS, NULL}, + {"compilerHash", (PyCFunction)compilerHash, METH_VARARGS, compilerHash_doc}, + {"identityHash", (PyCFunction)identityHash, METH_VARARGS, identityHash_doc}, {"serializeStream", (PyCFunction)serializeStream, METH_VARARGS, NULL}, {"deserializeStream", (PyCFunction)deserializeStream, METH_VARARGS, NULL}, {"is_default_constructible", (PyCFunction)is_default_constructible, METH_VARARGS, NULL}, {"buildCodeObject", (PyCFunction)buildCodeObject, METH_VARARGS | METH_KEYWORDS, NULL}, {"buildPyFunctionObject", (PyCFunction)buildPyFunctionObject, METH_VARARGS | METH_KEYWORDS, NULL}, {"isPOD", (PyCFunction)isPOD, METH_VARARGS, NULL}, - {"isSimple", (PyCFunction)isSimple, METH_VARARGS, NULL}, - {"bytecount", (PyCFunction)bytecount, METH_VARARGS, NULL}, - {"isBinaryCompatible", (PyCFunction)isBinaryCompatible, METH_VARARGS, NULL}, - {"Forward", (PyCFunction)MakeForward, METH_VARARGS, NULL}, - {"allForwardTypesResolved", (PyCFunction)allForwardTypesResolved, METH_VARARGS, NULL}, - {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS, NULL}, - {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS, NULL}, - {"recursiveTypeGroupDeepRepr", (PyCFunction)recursiveTypeGroupDeepRepr, METH_VARARGS, NULL}, - {"recursiveTypeGroupHash", (PyCFunction)recursiveTypeGroupHash, METH_VARARGS, NULL}, - {"checkForHashInstability", (PyCFunction)checkForHashInstability, METH_VARARGS, checkForHashInstability_doc}, + {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, + {"isResolved", (PyCFunction)isResolved, METH_VARARGS, NULL}, + {"_enableTypeAutoresolution", (PyCFunction)enableTypeAutoresolution, METH_VARARGS, NULL}, + {"forwardDefinitionsFor", (PyCFunction)forwardDefinitionsFor, METH_VARARGS, NULL}, + {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, + {"typeLooksResolvable", (PyCFunction)typeLooksResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, + {"assertTypeResolvable", (PyCFunction)assertTypeResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, + {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroupDeepRepr", (PyCFunction)recursiveTypeGroupDeepRepr, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroupHash", (PyCFunction)recursiveTypeGroupHash, METH_VARARGS | METH_KEYWORDS, NULL}, + {"checkForHashInstability", (PyCFunction)checkForHashInstability, METH_VARARGS | METH_KEYWORDS, checkForHashInstability_doc}, {"resetCompilerVisibleObjectHashCache", (PyCFunction)resetCompilerVisibleObjectHashCache, METH_VARARGS, resetCompilerVisibleObjectHashCache_doc}, - {"typeWalkRecord", (PyCFunction)typeWalkRecord, METH_VARARGS, NULL}, - {"typesAndObjectsVisibleToCompilerFrom", (PyCFunction)typesAndObjectsVisibleToCompilerFrom, METH_VARARGS, NULL}, - {"isRecursive", (PyCFunction)isRecursive, METH_VARARGS, NULL}, + {"typeWalkRecord", (PyCFunction)typeWalkRecord, METH_VARARGS | METH_KEYWORDS, NULL}, + {"typesAndObjectsVisibleToCompilerFrom", (PyCFunction)typesAndObjectsVisibleToCompilerFrom, METH_VARARGS | METH_KEYWORDS, NULL}, + {"isRecursive", (PyCFunction)isRecursive, METH_VARARGS | METH_KEYWORDS, NULL}, {"referencedTypes", (PyCFunction)referencedTypes, METH_VARARGS, NULL}, {"wantsToDefaultConstruct", (PyCFunction)wantsToDefaultConstruct, METH_VARARGS, NULL}, {"all_alternatives_empty", (PyCFunction)all_alternatives_empty, METH_VARARGS, NULL}, @@ -3367,13 +3792,6 @@ static struct PyModuleDef moduledef = { .m_free = NULL }; -void updateTypeRepForType(Type* type, PyTypeObject* pyType) { - //deliberately leak the name. - pyType->tp_name = (new std::string(type->nameWithModule()))->c_str(); - - PyInstance::mirrorTypeInformationIntoPyType(type, pyType); -} - PyMODINIT_FUNC PyInit__types(void) { @@ -3416,24 +3834,49 @@ PyInit__types(void) PyModule_AddObject(module, "TypedCell", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catTypedCell))); PyModule_AddObject(module, "PyCell", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPyCell))); PyModule_AddObject(module, "PythonObjectOfType", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPythonObjectOfType))); - PyModule_AddObject(module, "PythonSubclass", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPythonSubclass))); PyModule_AddObject(module, "SubclassOf", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catSubclassOf))); + PyModule_AddObject(module, "Forward", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catForward))); if (module == NULL) return NULL; - // initialize a couple of global references to things in typed_python.internals - PythonObjectOfType::AnyPyObject(); - PythonObjectOfType::AnyPyType(); + try { + // initialize a couple of global references to things in typed_python.internals + PythonObjectOfType::AnyPyObject(); + PythonObjectOfType::AnyPyType(); + } catch (std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + return NULL; + } if (PyType_Ready(&PyType_Slab) < 0) { return NULL; } + if (PyType_Ready(&PyType_FunctionOverload) < 0) { + return NULL; + } + if (PyType_Ready(&PyType_ModuleRepresentation) < 0) { return NULL; } + if (PyType_Ready(&PyType_FunctionGlobal) < 0) { + return NULL; + } + + if (PyType_Ready(&PyType_PyObjSnapshot) < 0) { + return NULL; + } + + if (PyType_Ready(&PyType_PyObjGraphSnapshot) < 0) { + return NULL; + } + + PyModule_AddObject(module, "FunctionOverload", (PyObject*)incref(&PyType_FunctionOverload)); + PyModule_AddObject(module, "PyObjSnapshot", (PyObject*)incref(&PyType_PyObjSnapshot)); + PyModule_AddObject(module, "PyObjGraphSnapshot", (PyObject*)incref(&PyType_PyObjGraphSnapshot)); + PyModule_AddObject(module, "FunctionGlobal", (PyObject*)incref(&PyType_FunctionGlobal)); PyModule_AddObject(module, "Slab", (PyObject*)incref(&PyType_Slab)); PyModule_AddObject(module, "ModuleRepresentation", (PyObject*)incref(&PyType_ModuleRepresentation)); diff --git a/typed_python/_types.hpp b/typed_python/_types.hpp index ed088c35b..6244ac00f 100644 --- a/typed_python/_types.hpp +++ b/typed_python/_types.hpp @@ -17,6 +17,7 @@ #pragma once #include +#include "PythonTypeInternals.hpp" PyObject *MakeTupleOrListOfType(PyObject* nullValue, PyObject* args, bool isTuple); PyObject *MakePointerToType(PyObject* nullValue, PyObject* args); @@ -52,24 +53,4 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args); PyObject *MakeClassType(PyObject* nullValue, PyObject* args); PyObject *MakeSubclassOfType(PyObject* nullValue, PyObject* args); PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwargs); - - -typedef struct { - PyObject_HEAD - PyObject *prop_get; - PyObject *prop_set; - PyObject *prop_del; - PyObject *prop_doc; - - #if PY_MINOR_VERSION >= 10 - PyObject *prop_name; - #endif - - int getter_doc; -} JustLikeAPropertyObject; - -typedef struct { - PyObject_HEAD - PyObject *cm_callable; - PyObject *cm_dict; -} JustLikeAClassOrStaticmethod; +PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs); diff --git a/typed_python/all.cpp b/typed_python/all.cpp index 70d282c56..b639da242 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -57,11 +57,13 @@ compile the entire group all at once. #include "PythonSerializationContext.cpp" #include "PythonSerializationContext_serialization.cpp" #include "PythonSerializationContext_deserialization.cpp" -#include "PythonSubclassType.cpp" #include "StringType.cpp" #include "TupleOrListOfType.cpp" #include "Type.cpp" -#include "FunctionType.cpp" +#include "ClosureVariableBinding.cpp" +#include "FunctionOverload.cpp" +#include "FunctionGlobal.cpp" +#include "ForwardType.cpp" #include "ValueType.cpp" #include "SerializationBuffer.cpp" @@ -74,6 +76,14 @@ compile the entire group all at once. #include "PyModuleRepresentation.cpp" #include "Slab.cpp" #include "PyTemporaryReferenceTracer.cpp" +#include "PyFunctionOverload.cpp" +#include "PyFunctionGlobal.cpp" +#include "PyObjSnapshot.cpp" +#include "PyObjSnapshotMaker.cpp" +#include "PyObjGraphSnapshot.cpp" +#include "PyObjRehydrator.cpp" +#include "PyPyObjSnapshot.cpp" +#include "PyPyObjGraphSnapshot.cpp" #include "lz4.c" #include "lz4frame.c" diff --git a/typed_python/compiler/expression_conversion_context.py b/typed_python/compiler/expression_conversion_context.py index 24cb5948d..8e3404909 100644 --- a/typed_python/compiler/expression_conversion_context.py +++ b/typed_python/compiler/expression_conversion_context.py @@ -23,7 +23,8 @@ from typed_python.compiler.type_wrappers.slice_type_object_wrapper import SliceWrapper from typed_python.compiler.conversion_level import ConversionLevel from typed_python.SerializationContext import SerializationContext -from typed_python.internals import makeFunctionType, FunctionOverload +from typed_python.internals import makeFunctionType +from typed_python._types import FunctionOverload from typed_python.compiler.function_stack_state import FunctionStackState from typed_python.compiler.python_object_representation import pythonObjectRepresentation from typed_python.compiler.python_object_representation import pythonObjectRepresentationType @@ -806,6 +807,22 @@ def call_function_pointer(self, funcPtr, args, returnType): native_ast.CallTarget.Pointer(expr=funcPtr.cast(nativeFunType.pointer())).call(*native_args) ) + @staticmethod + def minPositionalCount(args): + for i in range(len(args)): + a = args[i] + if a.defaultValue or a.isStarArg or a.isKwarg: + return i + return len(args) + + @staticmethod + def maxPositionalCount(args): + for i in range(len(args)): + a = args[i] + if a.isStarArg: + return None + return len(args) + @staticmethod def mapFunctionArguments(functionOverload: FunctionOverload, args, kwargs) -> OneOf(str, ListOf(FunctionArgMapping)): """Figure out how to call 'functionOverload' with 'args' and 'kwargs'. @@ -831,8 +848,8 @@ def mapFunctionArguments(functionOverload: FunctionOverload, args, kwargs) -> On outArgs = ListOf(FunctionArgMapping)() curTargetIx = 0 - minPositional = functionOverload.minPositionalCount() - maxPositional = functionOverload.maxPositionalCount() + minPositional = ExpressionConversionContext.minPositionalCount(functionOverload.args) + maxPositional = ExpressionConversionContext.maxPositionalCount(functionOverload.args) consumedPositionalNames = set() @@ -1144,12 +1161,7 @@ def call_py_function(self, f, args, kwargs, returnTypeOverload=None): globalsInCells.append(f.__code__.co_freevars[i]) call_target = self.functionContext.converter.convert( - f.__name__, - f.__code__, - funcGlobals, - f.__globals__, - globalsInCells, - [], + typedFunc.overloads[0], [a.expr_type for a in concreteArgs], returnTypeOverload ) @@ -1188,12 +1200,7 @@ def call_overload(self, overload, funcObj, args, kwargs, returnTypeOverload=None returnType = typeWrapper(overload.returnType) if overload.returnType is not None else None call_target = self.functionContext.converter.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - list(overload.funcGlobalsInCells), - list(overload.closureVarLookups), + overload, [a.expr_type for a in closureArgs] + [a.expr_type for a in concreteArgs], returnType diff --git a/typed_python/compiler/native_compiler/native_ast.py b/typed_python/compiler/native_compiler/native_ast.py index 171ade17f..6f818c7ee 100644 --- a/typed_python/compiler/native_compiler/native_ast.py +++ b/typed_python/compiler/native_compiler/native_ast.py @@ -15,7 +15,9 @@ from typed_python.compiler.native_compiler.global_variable_definition import ( GlobalVariableMetadata ) -from typed_python import Tuple, TupleOf, Alternative, NamedTuple, OneOf, Forward +from typed_python import ( + Tuple, TupleOf, Alternative, NamedTuple, OneOf, Forward, resolveForwardDefinedType +) import textwrap @@ -86,8 +88,8 @@ def typeZeroConstant(self): raise Exception(f"Can't make a zero value from {self}") -Type = Forward("Type") -Type = Type.define(Alternative( +Type = Forward(lambda: Type) +Type = Alternative( "Type", # the 'Void' type. should only be used in function signatures. Use an empty # struct if you want the 'empty' data structure instead. @@ -107,7 +109,7 @@ def typeZeroConstant(self): all(valType.isEmpty() for keyType, valType in self.element_types) if self.matches.Struct else False, zeroConstant=typeZeroConstant -)) +) def const_truth_value(c): @@ -143,8 +145,8 @@ def const_str(c): assert False, type(c) -Constant = Forward("Constant") -Constant = Constant.define(Alternative( +Constant = Forward(lambda: Constant) +Constant = Alternative( "Constant", Void={}, Float={'val': float, 'bits': int}, @@ -155,7 +157,8 @@ def const_str(c): NullPointer={'value_type': Type}, truth_value=const_truth_value, __str__=const_str -)) +) + UnaryOp = Alternative( "UnaryOp", @@ -197,10 +200,6 @@ def const_str(c): ) -# loads and stores - no assignments -Expression = Forward("Expression") -Teardown = Forward("Teardown") - NamedCallTarget = NamedTuple( name=str, arg_types=TupleOf(Type), @@ -245,6 +244,9 @@ def intm_could_throw(intm): assert False, f"Unrecognized expression intermediate {intm}" +Expression = Forward() +Teardown = Forward() + ExpressionIntermediate = Alternative( "ExpressionIntermediate", Effect={"expr": Expression}, @@ -294,7 +296,7 @@ def teardown_str(self): assert False, type(self) -Teardown = Teardown.define(Alternative( +Teardown.define(Alternative( "Teardown", ByTag={'tag': str, 'expr': Expression}, Always={'expr': Expression}, @@ -558,7 +560,7 @@ def expr_could_throw(self): return False -Expression = Expression.define(Alternative( +Expression.define(Alternative( "Expression", Constant={'val': Constant}, Comment={'comment': str, 'expr': Expression}, @@ -659,6 +661,14 @@ def expr_could_throw(self): couldThrow=expr_could_throw )) +Expression = resolveForwardDefinedType(Expression) +NamedCallTarget = resolveForwardDefinedType(NamedCallTarget) +Type = resolveForwardDefinedType(Type) +Constant = resolveForwardDefinedType(Constant) +ExpressionIntermediate = resolveForwardDefinedType(ExpressionIntermediate) +Teardown = resolveForwardDefinedType(Teardown) +CallTarget = resolveForwardDefinedType(CallTarget) + def ensureExpr(x): if isinstance(x, int): diff --git a/typed_python/compiler/python_ast_analysis.py b/typed_python/compiler/python_ast_analysis.py index 589b1a519..eff99e596 100644 --- a/typed_python/compiler/python_ast_analysis.py +++ b/typed_python/compiler/python_ast_analysis.py @@ -205,11 +205,11 @@ def extractLineNumbersWithStatements(astNode): def visit(x): if isinstance(x, Statement): + res.add(x.line_number) + if x.matches.FunctionDef or x.matches.ClassDef or x.matches.AsyncFunctionDef: return False - res.add(x.line_number) - if isinstance(x, Expr): if ( x.matches.Lambda @@ -233,7 +233,11 @@ def extractFunctionDefsInOrder(astNode): def visit(x): if isinstance(x, Statement): if x.matches.FunctionDef or x.matches.ClassDef or x.matches.AsyncFunctionDef: + if x.matches.FunctionDef: + visitPyAstChildren(x.returns, visit) + res.append(x) + return False if isinstance(x, Expr): diff --git a/typed_python/compiler/python_ast_util.py b/typed_python/compiler/python_ast_util.py index 50e085b5a..6049096dc 100644 --- a/typed_python/compiler/python_ast_util.py +++ b/typed_python/compiler/python_ast_util.py @@ -80,6 +80,9 @@ def pyAstForCode(codeObject): wholeFileAst = pyAstFromText(sourceText) + if codeObject.co_name == '': + return wholeFileAst + defs = functionDefsOrLambdaAtLineNumber(sourceText, wholeFileAst, codeObject.co_firstlineno) if len(defs) == 0: @@ -177,6 +180,7 @@ class AtLineNumberVisitor(ast.NodeVisitor): def __init__(self, lineNumber): self.funcDefSubnodesAtLineNumber = [] self.lambdaSubnodesAtLineNumber = [] + self.classDefSubnodesAtLineNumber = [] self.lineNumber = lineNumber def visit_FunctionDef(self, node): @@ -190,6 +194,17 @@ def visit_FunctionDef(self, node): ast.NodeVisitor.generic_visit(self, node) + def visit_ClassDef(self, node): + if node.lineno == self.lineNumber: + self.classDefSubnodesAtLineNumber.append(node) + else: + for d in node.decorator_list: + if d.lineno == self.lineNumber: + self.classDefSubnodesAtLineNumber.append(node) + break + + ast.NodeVisitor.generic_visit(self, node) + def visit_AsyncFunctionDef(self, node): if node.lineno == self.lineNumber: self.funcDefSubnodesAtLineNumber.append(node) @@ -208,5 +223,6 @@ def functionDefsOrLambdaAtLineNumber(sourceText, sourceAst, lineNumber): return ( visitor.funcDefSubnodesAtLineNumber + - visitor.lambdaSubnodesAtLineNumber + visitor.lambdaSubnodesAtLineNumber + + visitor.classDefSubnodesAtLineNumber ) diff --git a/typed_python/compiler/python_object_representation.py b/typed_python/compiler/python_object_representation.py index 3143b7bae..07780505d 100644 --- a/typed_python/compiler/python_object_representation.py +++ b/typed_python/compiler/python_object_representation.py @@ -26,7 +26,6 @@ from typed_python.compiler.type_wrappers.python_type_object_wrapper import PythonTypeObjectWrapper from typed_python.compiler.type_wrappers.module_wrapper import ModuleWrapper from typed_python.compiler.type_wrappers.typed_cell_wrapper import TypedCellWrapper -from typed_python.compiler.type_wrappers.unresolved_forward_type_wrapper import UnresolvedForwardTypeWrapper from typed_python.compiler.type_wrappers.python_free_function_wrapper import PythonFreeFunctionWrapper from typed_python.compiler.type_wrappers.python_free_object_wrapper import PythonFreeObjectWrapper from typed_python.compiler.type_wrappers.python_typed_function_wrapper import PythonTypedFunctionWrapper @@ -77,8 +76,8 @@ from typed_python.compiler.type_wrappers.repr_wrapper import ReprWrapper from types import ModuleType from typed_python._types import ( - TypeFor, bytecount, prepareArgumentToBePassedToCompiler, allForwardTypesResolved, - serialize, deserialize + TypeFor, bytecount, prepareArgumentToBePassedToCompiler, + serialize, deserialize, isForwardDefined ) from typed_python import ( Type, Int32, Int16, Int8, UInt64, UInt32, UInt16, @@ -152,8 +151,8 @@ def _typedPythonTypeToTypeWrapper(t): assert isinstance(t, type), t - if not allForwardTypesResolved(t): - return UnresolvedForwardTypeWrapper(t) + if isForwardDefined(t): + raise Exception("Can't compile against Forward types") if not hasattr(t, '__typed_python_category__'): # this is will be a PythonObjectOfType, but we never actually return such an object diff --git a/typed_python/compiler/python_to_native_converter.py b/typed_python/compiler/python_to_native_converter.py index d73392792..e9597a841 100644 --- a/typed_python/compiler/python_to_native_converter.py +++ b/typed_python/compiler/python_to_native_converter.py @@ -89,6 +89,34 @@ def addVisitor(self, visitor): def removeVisitor(self, visitor): self._visitors.remove(visitor) + def hashObjectToIdentity(self, hashable, isModuleVal=False): + if isinstance(hashable, Hash): + return hashable + + if isinstance(hashable, int): + return Hash.from_integer(hashable) + + if isinstance(hashable, str): + return Hash.from_string(hashable) + + if hashable is None: + return Hash.from_integer(1) + Hash.from_integer(0) + + if isinstance(hashable, (dict, list)) and isModuleVal: + # don't look into dicts and lists at module level + return Hash.from_integer(2) + + if isinstance(hashable, (tuple, list)): + res = Hash.from_integer(len(hashable)) + for t in hashable: + res += self.hashObjectToIdentity(t, isModuleVal) + return res + + if isinstance(hashable, Wrapper): + return hashable.compilerHash() + + return Hash(_types.compilerHash(hashable)) + def identityToName(self, identity): """Convert a function identity to the link-time name for the function. @@ -124,38 +152,34 @@ def extract_new_function_definitions(self): return res - def identityHashToLinkerName(self, name, identityHash, prefix="tp."): + def identityHashToLinkerName(self, name, compilerHash, prefix="tp."): assert isinstance(name, str) - assert isinstance(identityHash, str) + assert isinstance(compilerHash, str) assert isinstance(prefix, str) - return prefix + name + "." + identityHash + return prefix + name + "." + compilerHash def createConversionContext( self, identity, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - closureVars, + overload, input_types, output_type, conversionType ): ConverterType = conversionType or FunctionConversionContext - pyast = self._code_to_ast(funcCode) + pyast = self._code_to_ast(overload.functionCode) return ConverterType( self, - funcName, + overload.name, identity, input_types, output_type, - closureVars, - funcGlobals, - funcGlobalsRaw, + overload.closureVarLookups, + overload.realizedGlobals, + overload.globals, pyast.args, pyast, ) @@ -591,103 +615,27 @@ def convertTypedFunctionCall(self, functionType, overloadIx, inputWrappers, asse returnType = None return self.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - overload.funcGlobalsInCells, - list(overload.closureVarLookups), + overload, realizedInputWrappers, returnType, assertIsRoot=assertIsRoot ) - def hashObjectToIdentity(self, hashable, isModuleVal=False): - if isinstance(hashable, Hash): - return hashable - - if isinstance(hashable, int): - return Hash.from_integer(hashable) - - if isinstance(hashable, str): - return Hash.from_string(hashable) - - if hashable is None: - return Hash.from_integer(1) + Hash.from_integer(0) - - if isinstance(hashable, (dict, list)) and isModuleVal: - # don't look into dicts and lists at module level - return Hash.from_integer(2) - - if isinstance(hashable, (tuple, list)): - res = Hash.from_integer(len(hashable)) - for t in hashable: - res += self.hashObjectToIdentity(t, isModuleVal) - return res - - if isinstance(hashable, Wrapper): - return hashable.identityHash() - - return Hash(_types.identityHash(hashable)) - - def hashGlobals(self, funcGlobals, code, funcGlobalsFromCells): - """Hash a given piece of code's accesses to funcGlobals. - - We're trying to make sure that if we have a reference to module 'x' - in our globals, but we only ever use 'x' by writing 'x.f' or 'x.g', then - we shouldn't depend on the entirety of the definition of 'x'. - """ - - res = Hash.from_integer(0) - - for dotSeq in _types.getCodeGlobalDotAccesses(code): - res += self.hashDotSeq(dotSeq, funcGlobals) - - for globalName in funcGlobalsFromCells: - res += self.hashDotSeq([globalName], funcGlobals) - - return res - - def hashDotSeq(self, dotSeq, funcGlobals): - if not dotSeq or dotSeq[0] not in funcGlobals: - return Hash.from_integer(0) - - item = funcGlobals[dotSeq[0]] - - if not isinstance(item, ModuleType) or len(dotSeq) == 1: - return Hash.from_string(dotSeq[0]) + self.hashObjectToIdentity(item, True) - - if not hasattr(item, dotSeq[1]): - return Hash.from_integer(0) - - return Hash.from_string(dotSeq[0] + "." + dotSeq[1]) + self.hashObjectToIdentity(getattr(item, dotSeq[1]), True) - def convert( self, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - funcGlobalsFromCells, - closureVars, + overload, input_types, output_type, assertIsRoot=False, conversionType=None ): - """Convert a single pure python function using args of 'input_types'. + """Convert a single PyFunctionOverload using args of 'input_types'. It will return no more than 'output_type'. if output_type is None we produce the tightest output type possible. Args: - funcName - the name of the function - funcCode - a Code object representing the code to compile - funcGlobals - the globals object from the relevant function - funcGlobalsRaw - the original globals object (with no merging or filtering done) - which we use to figure out the location of global variables that are not in cells. - funcGlobalsFromCells - a list of the names that are globals that are actually accessed - as cells. + overload - a PyFunctionOverload instance we want to compile input_types - a type for each free variable in the function closure, and then again for each input argument output_type - the output type of the function, if known. if this is None, @@ -698,28 +646,19 @@ def convert( conversionType - if None, this is a normal function conversion. Otherwise, this must be a subclass of FunctionConversionContext """ - assert isinstance(funcName, str) - assert isinstance(funcCode, types.CodeType) - assert isinstance(funcGlobals, dict) + funcName = overload.name input_types = tuple([typedPythonTypeToTypeWrapper(i) for i in input_types]) - identityHash = ( + compilerHash = ( Hash.from_integer(1) - + self.hashObjectToIdentity(( - funcCode, - funcName, - input_types, - output_type, - closureVars, - conversionType - )) + - self.hashGlobals(funcGlobals, funcCode, funcGlobalsFromCells) + + Hash.from_integer(overload.index) + + Hash(_types.compilerHash(overload.functionTypeObject)) ) - assert not identityHash.isPoison() + assert not compilerHash.isPoison() - identity = identityHash.hexdigest + identity = compilerHash.hexdigest # determine the linkName we'll use in the NativeCompiler to identify # this particular function @@ -729,7 +668,11 @@ def convert( if identity not in self._identifier_to_pyfunc: self._identifier_to_pyfunc[identity] = ( - funcName, funcCode, funcGlobals, closureVars, input_types, output_type, conversionType + overload.functionTypeObject, + overload.index, + input_types, + output_type, + conversionType ) isRoot = len(self._inflight_function_conversions) == 0 @@ -759,11 +702,7 @@ def convert( if identity not in self._inflight_function_conversions: functionConverter = self.createConversionContext( identity, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - closureVars, + overload, input_types, output_type, conversionType @@ -820,19 +759,17 @@ def _installInflightFunctions(self): if identifier in self._identifier_to_pyfunc: for v in self._visitors: - funcName, funcCode, funcGlobals, closureVars, input_types, output_type, conversionType = ( + functionTypeObject, overloadIx, input_types, output_type, conversionType = ( self._identifier_to_pyfunc[identifier] ) + overload = functionTypeObject.overloads[overloadIx] try: v.onNewFunction( identifier, functionConverter, nativeFunction, - funcName, - funcCode, - funcGlobals, - closureVars, + overload, input_types, functionConverter._varname_to_type.get(FunctionOutput), functionConverter._varname_to_type.get(FunctionYield), diff --git a/typed_python/compiler/runtime.py b/typed_python/compiler/runtime.py index 2a45a85bb..4c4d4bcdd 100644 --- a/typed_python/compiler/runtime.py +++ b/typed_python/compiler/runtime.py @@ -346,10 +346,12 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments fp = self.converter.functionPointerByName(wrappingCallTargetName) - overload._installNativePointer( + typed_python._types.installNativeFunctionPointer( + functionType, + overloadIx, fp.fp, callTarget.output_type.typeRepresentation if callTarget.output_type is not None else type(None), - [i.typeRepresentation for i in callTarget.input_types] + tuple([i.typeRepresentation for i in callTarget.input_types]) ) self.converter.flushDelayedVMIs() diff --git a/typed_python/compiler/tests/closures_test.py b/typed_python/compiler/tests/closures_test.py index 2cbb55791..65e9e4fb6 100644 --- a/typed_python/compiler/tests/closures_test.py +++ b/typed_python/compiler/tests/closures_test.py @@ -24,7 +24,7 @@ ) from typed_python.compiler.runtime import RuntimeEventVisitor, Entrypoint -from typed_python._types import refcount, identityHash +from typed_python._types import refcount, compilerHash from typed_python.test_util import currentMemUsageMb @@ -221,7 +221,7 @@ def alternatingCall(c1, c2, ct): c1 = makeAppender(aList1) c2 = makeAppender(aList2) - assert identityHash(type(c1)) == identityHash(type(c2)) + assert compilerHash(type(c1)) == compilerHash(type(c2)) self.assertEqual(type(c1), type(c2)) @@ -841,7 +841,7 @@ def f(self): C1 = makeClass(1) C2 = makeClass(2) - assert identityHash(C1.f) != identityHash(C2.f) + assert compilerHash(C1.f) != compilerHash(C2.f) @Entrypoint def callF(c): diff --git a/typed_python/compiler/tests/conversion_test.py b/typed_python/compiler/tests/conversion_test.py index f16181e17..7710b20ff 100644 --- a/typed_python/compiler/tests/conversion_test.py +++ b/typed_python/compiler/tests/conversion_test.py @@ -25,7 +25,7 @@ from typed_python import ( Function, OneOf, TupleOf, ListOf, Tuple, NamedTuple, Class, NotCompiled, Dict, _types, Compiled, Member, Final, isCompiled, ConstDict, - makeNamedTuple, UInt32, Int32, Type, identityHash, typeKnownToCompiler, checkOneOfType, + makeNamedTuple, UInt32, Int32, Type, compilerHash, typeKnownToCompiler, checkOneOfType, refcount, checkType, map ) @@ -2867,7 +2867,7 @@ def call(x): assert newCall.__code__.co_filename == call.__code__.co_filename - assert identityHash(call.__code__) == identityHash(newCall.__code__) + assert compilerHash(call.__code__) == compilerHash(newCall.__code__) def test_code_with_nested_listcomp(self): def call(x): diff --git a/typed_python/compiler/type_wrappers/alternative_wrapper.py b/typed_python/compiler/type_wrappers/alternative_wrapper.py index e0836bd6f..29261769e 100644 --- a/typed_python/compiler/type_wrappers/alternative_wrapper.py +++ b/typed_python/compiler/type_wrappers/alternative_wrapper.py @@ -61,7 +61,7 @@ def convert_comparison(self, context, lhs, op, rhs): if py_code < 0: return super().convert_comparison(context, lhs, op, rhs) - if _types.isBinaryCompatible(lhs.expr_type.typeRepresentation, rhs.expr_type.typeRepresentation): + if lhs.expr_type.typeRepresentation is rhs.expr_type.typeRepresentation: lhs = lhs.ensureIsReference() rhs = rhs.ensureIsReference() diff --git a/typed_python/compiler/type_wrappers/class_wrapper.py b/typed_python/compiler/type_wrappers/class_wrapper.py index 083ab0aeb..2c9f3c4b5 100644 --- a/typed_python/compiler/type_wrappers/class_wrapper.py +++ b/typed_python/compiler/type_wrappers/class_wrapper.py @@ -141,7 +141,7 @@ def _classCouldBeInstanceOf(cls, other): return False for name in set(cls.ClassMembers) & set(other.ClassMembers): - if name not in ['__qualname__', '__name__', '__module__', '__typed_python_template__', '__classcell__']: + if name not in ['__qualname__', '__name__', '__module__', '__classcell__']: if cls.ClassMembers[name] is not other.ClassMembers[name]: return False diff --git a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py index b71cac261..f2027eed8 100644 --- a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py +++ b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py @@ -13,9 +13,12 @@ # limitations under the License. -from typed_python import PointerTo, bytecount, NamedTuple, Class, OneOf, ListOf, TupleOf, Tuple, Set, ConstDict, Dict, Value +from typed_python import ( + PointerTo, bytecount, NamedTuple, Class, OneOf, ListOf, TupleOf, Tuple, Set, + ConstDict, Dict, Value, isForwardDefined +) from typed_python.compiler.merge_type_wrappers import mergeTypeWrappers -from typed_python._types import is_default_constructible, allForwardTypesResolved +from typed_python._types import is_default_constructible from typed_python.internals import CellAccess from typed_python.compiler.conversion_level import ConversionLevel from typed_python.compiler.type_wrappers.wrapper import Wrapper @@ -118,8 +121,9 @@ def computeFunctionOverloadReturnType(overload, argTypes, kwargTypes): Returns: NoReturnTypeSpecified, a type, CannotBeDetermined, or SomeInvalidClassReturnType """ - if not allForwardTypesResolved(overload.functionTypeObject): - return NoReturnTypeSpecified + # TODO: what should we be doing here? + if isForwardDefined(overload.functionTypeObject): + raise Exception("Can't compile against forward defined types.") return typeWrapper(overload.functionTypeObject).signatureCalculator.returnTypeForOverload( overload.index, argTypes, kwargTypes @@ -343,12 +347,7 @@ def convert_comprehension(self, context, instance, ConvertionContextType): # there's definitely a better way to organize this code. singleConvertedOverload = context.functionContext.converter.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - list(overload.funcGlobalsInCells), - list(overload.closureVarLookups), + overload, [typeWrapper(self.closurePathToCellType(path, closureType)) for path in overload.closureVarLookups.values()], None, conversionType=ConvertionContextType @@ -508,12 +507,7 @@ def convert_call(self, context, left, args, kwargs): # just one overload will do. We can just instantiate this particular function # with a signature that comes from the method overload signature itself. singleConvertedOverload = context.functionContext.converter.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - list(overload.funcGlobalsInCells), - list(overload.closureVarLookups), + overload, [typeWrapper(self.closurePathToCellType(path, closureType)) for path in overload.closureVarLookups.values()] + [a.expr_type for a in argsToPass], returnType @@ -730,12 +724,7 @@ def determinePossibleReturnTypes(converter, func, argTypes, kwargTypes): ) callTarget = converter.convert( - o.name, - o.functionCode, - o.realizedGlobals, - o.functionGlobals, - list(o.funcGlobalsInCells), - list(o.closureVarLookups), + o, [typeWrapper(PythonTypedFunctionWrapper.closurePathToCellType(path, func.ClosureType)) for path in o.closureVarLookups.values()] + actualArgTypes, None diff --git a/typed_python/compiler/type_wrappers/range_wrapper.py b/typed_python/compiler/type_wrappers/range_wrapper.py index efbe758b4..502ddaf54 100644 --- a/typed_python/compiler/type_wrappers/range_wrapper.py +++ b/typed_python/compiler/type_wrappers/range_wrapper.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typed_python import NamedTuple, Class, Final, PointerTo -from typed_python import pointerTo +from typed_python import Class, Final, PointerTo, Held, Member, pointerTo, resolveForwardDefinedType class Range(Class, Final, __name__='range'): @@ -33,10 +32,12 @@ def __call__(self, start, stop, step): # noqa return RangeCls(start=start, stop=stop, step=step) -range = Range() - +@Held +class RangeCls(Class, Final): + start = Member(int, nonempty=True) + stop = Member(int, nonempty=True) + step = Member(int, nonempty=True) -class RangeCls(NamedTuple(start=int, stop=int, step=int)): def __str__(self): return repr(self) @@ -62,7 +63,12 @@ def __typed_python_int_iter_value__(self, x): return self.start + self.step * x -class RangeIterator(NamedTuple(start=int, stop=int, step=int)): +@Held +class RangeIterator(Class, Final): + start = Member(int, nonempty=True) + stop = Member(int, nonempty=True) + step = Member(int, nonempty=True) + def __iter__(self) -> int: resPtr = self.__fastnext__() if resPtr: @@ -84,3 +90,11 @@ def __fastnext__(self) -> PointerTo(int): return startPtr return PointerTo(int)() + + +RangeCls = resolveForwardDefinedType(RangeCls) +RangeIterator = resolveForwardDefinedType(RangeIterator) +Range = resolveForwardDefinedType(Range) + + +range = Range() diff --git a/typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py b/typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py deleted file mode 100644 index fdabb7730..000000000 --- a/typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py +++ /dev/null @@ -1,131 +0,0 @@ -from typed_python.compiler.type_wrappers.wrapper import Wrapper -import typed_python.compiler.native_compiler.native_ast as native_ast -from typed_python.compiler.conversion_level import ConversionLevel - - -class UnresolvedForwardTypeWrapper(Wrapper): - def __init__(self, t): - super().__init__(t) - - def getNativeLayoutType(self): - return native_ast.Void - - def throwUnresolvedForwardException(self, context): - return context.pushException( - TypeError, - f"Type {self.typeRepresentation.__name__} has unresolved forwards" - ) - - def convert_incref(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_fastnext(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_attribute_pointerTo(self, context, pointerInstance, attribute): - return self.throwUnresolvedForwardException(context) - - def convert_attribute(self, context, instance, attribute): - return self.throwUnresolvedForwardException(context) - - def convert_set_attribute(self, context, instance, attribute, value): - return self.throwUnresolvedForwardException(context) - - def convert_delitem(self, context, instance, item): - return self.throwUnresolvedForwardException(context) - - def convert_getitem(self, context, instance, item): - return self.throwUnresolvedForwardException(context) - - def convert_getslice(self, context, instance, start, stop, step): - return self.throwUnresolvedForwardException(context) - - def convert_setitem(self, context, instance, index, value): - return self.throwUnresolvedForwardException(context) - - def convert_assign(self, context, target, toStore): - return self.throwUnresolvedForwardException(context) - - def convert_copy_initialize(self, context, target, toStore): - return self.throwUnresolvedForwardException(context) - - def convert_destroy(self, context, instance): - return self.throwUnresolvedForwardException(context) - - def convert_default_initialize(self, context, target): - return self.throwUnresolvedForwardException(context) - - def convert_typeof(self, context, instance): - return self.throwUnresolvedForwardException(context) - - def convert_issubclass(self, context, instance, ofType, isSubclassCall): - return self.throwUnresolvedForwardException(context) - - def convert_masquerade_to_untyped(self, context, instance): - return self.throwUnresolvedForwardException(context) - - def convert_call(self, context, left, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def convert_len(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_hash(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_abs(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_index_cast(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_math_float_cast(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_builtin(self, f, context, expr, a1=None): - return self.throwUnresolvedForwardException(context) - - def convert_repr(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_unary_op(self, context, expr, op): - return self.throwUnresolvedForwardException(context) - - def convert_to_type(self, context, expr, target_type, level: ConversionLevel, assumeSuccessful=False): - return self.throwUnresolvedForwardException(context) - - def convert_to_type_with_target(self, context, expr, targetVal, level: ConversionLevel, mayThrowOnFailure=False): - return self.throwUnresolvedForwardException(context) - - def convert_to_self_with_target(self, context, targetVal, sourceVal, level: ConversionLevel, mayThrowOnFailure=False): - return self.throwUnresolvedForwardException(context) - - def convert_bin_op(self, context, l, op, r, inplace): - return self.throwUnresolvedForwardException(context) - - def convert_bin_op_reverse(self, context, r, op, l, inplace): - return self.throwUnresolvedForwardException(context) - - def convert_format(self, context, instance, formatSpecOrNone=None): - return self.throwUnresolvedForwardException(context) - - def convert_type_attribute(self, context, typeInst, attribute): - return self.throwUnresolvedForwardException(context) - - def convert_type_call(self, context, typeInst, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def convert_call_on_container_expression(self, context, inst, argExpr): - return self.throwUnresolvedForwardException(context) - - def convert_type_call_on_container_expression(self, context, typeInst, argExpr): - return self.throwUnresolvedForwardException(context) - - def convert_type_method_call(self, context, typeInst, methodname, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def convert_method_call(self, context, instance, methodname, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def get_iteration_expressions(self, context, expr): - return self.throwUnresolvedForwardException(context) diff --git a/typed_python/compiler/type_wrappers/wrapper.py b/typed_python/compiler/type_wrappers/wrapper.py index 863ec42ab..ede3c5e45 100644 --- a/typed_python/compiler/type_wrappers/wrapper.py +++ b/typed_python/compiler/type_wrappers/wrapper.py @@ -79,6 +79,9 @@ class Wrapper: # is this wrapping a OneOf object is_oneof_wrapper = False + def __reduce__(self): + return (typeWrapper, (self.typeRepresentation,)) + def __repr__(self): return "Wrapper(%s)" % str(self) @@ -110,10 +113,10 @@ def __init__(self, typeRepresentation): def isIterable(self): return "Maybe" - def identityHash(self): + def compilerHash(self): return ( - Hash(_types.identityHash(self.typeRepresentation)) - + Hash(_types.identityHash(self.interpreterTypeRepresentation)) + Hash(_types.compilerHash(self.typeRepresentation)) + + Hash(_types.compilerHash(self.interpreterTypeRepresentation)) ) def __eq__(self, other): diff --git a/typed_python/internals.py b/typed_python/internals.py index b3894b41d..e2e378f0d 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -44,6 +44,7 @@ class UndefinedBehaviorException(BaseException): # needed by the C api object = object type = type +lockType = type(threading.Lock()) class Final: @@ -170,11 +171,11 @@ def makeFunctionType( if f.isEntrypoint: # reapply the entrypoint flag - res = type(res().withEntrypoint(True)) + res = typed_python._types.Function(res, 'Entrypoint') if f.isNocompile: - # reapply the entrypoint flag - res = type(res().withNocompile(True)) + # reapply the notcompiled flag + res = typed_python._types.Function(res, 'Nocompile') return res @@ -325,7 +326,9 @@ def __new__(cls, name, bases, namespace, **kwds): and issubclass(elt, typed_python._types.Function) ): if eltName == '__typed_python_template__': - classMembers[eltName] = elt + staticFunctions[eltName] = makeFunctionType( + eltName, elt, assumeClosuresGlobal=True + ) elif eltName not in memberFunctions: memberFunctions[eltName] = makeFunctionType( eltName, elt, classname=name, assumeClosuresGlobal=True @@ -380,10 +383,15 @@ def __instancecheck__(cls, instance): return cls in type(instance).MRO -def Function(f, assumeClosuresGlobal=False, returnTypeOverride=None): +def Function(f, returnTypeOverride=None, assumeClosuresGlobal=False): """Turn a normal python function into a 'typed_python.Function' which obeys type restrictions.""" - return makeFunctionType( - f.__name__, f, assumeClosuresGlobal=assumeClosuresGlobal, returnTypeOverride=returnTypeOverride + return typed_python._types.resolveForwardDefinedType( + makeFunctionType( + f.__name__, + f, + assumeClosuresGlobal=assumeClosuresGlobal, + returnTypeOverride=returnTypeOverride + ) )(f) @@ -427,112 +435,6 @@ def __repr__(self): return res -class FunctionOverload: - def __init__(self, functionTypeObject, index, code, funcGlobalsInCells, closureVarLookups, returnType, signatureFunction, methodOf): - """Initialize a FunctionOverload. - - Args: - functionTypeObject - a _types.Function type object representing the function - index - the index within the _types.Function sequence of overloads we represent - code - the code object for the function we're wrapping - funcGlobals - the globals for the function we're wrapping - funcGlobalsInCells - a dict of cells that also act like globals - closureVarLookups - a dict from str to a list of ClosureVariableBindingSteps indicating - how each function's closure variables are found in the closure of the - actual function. - returnType - the return type annotation, or None if None provided. (if None was - specified, that would be the NoneType) - signatureFunction - None, or the user-provided callable that determines the - return type as a function of the types of the arguments to the function. - methodOf - if we are a method of a class, the class object - """ - self.functionTypeObject = functionTypeObject - self.index = index - self.closureVarLookups = closureVarLookups - self.functionCode = code - self.funcGlobalsInCells = funcGlobalsInCells - self.returnType = returnType - self.signatureFunction = signatureFunction - self._realizedGlobals = None - self.methodOf = methodOf - self.args = () - - @property - def functionGlobals(self): - return self.functionTypeObject.extractOverloadGlobals(self.index) - - @staticmethod - def extractGlobalNamesFromCode(codeObj): - res = set() - res.update(codeObj.co_names) - - for const in codeObj.co_consts: - if isinstance(const, type(codeObj)): - res.update(FunctionOverload.extractGlobalNamesFromCode(const)) - - return res - - @property - def realizedGlobals(self): - """Merge the 'functionGlobals' and the set of globals in 'cells' into a single dict.""" - if self._realizedGlobals is None: - res = {} - - globalNames = set(self.extractGlobalNamesFromCode(self.functionCode)) - - for name in self.functionGlobals: - if name.split(".")[0] in globalNames: - res[name] = self.functionGlobals[name] - - if '__module_hash__' in self.functionGlobals: - res['__module_hash__'] = self.functionGlobals['__module_hash__'] - - for varname, cell in self.funcGlobalsInCells.items(): - res[varname] = cell.cell_contents - - if self.methodOf is not None: - res['__class__'] = self.methodOf.Class - - self._realizedGlobals = res - - return self._realizedGlobals - - @property - def name(self): - return self.functionTypeObject.__name__ - - def minPositionalCount(self): - for i in range(len(self.args)): - a = self.args[i] - if a.defaultValue or a.isStarArg or a.isKwarg: - return i - return len(self.args) - - def maxPositionalCount(self): - for i in range(len(self.args)): - a = self.args[i] - if a.isStarArg: - return None - return len(self.args) - - def addArg(self, name, defaultVal, typeFilter, isStarArg, isKwarg): - self.args = self.args + (FunctionOverloadArg(name, defaultVal, typeFilter, isStarArg, isKwarg),) - - def __str__(self): - if self.methodOf: - return "FunctionOverload(%s, returns %s, %s)" % (self.methodOf.Class.__name__, self.returnType, self.args) - return "FunctionOverload(returns %s, %s)" % (self.returnType, self.args) - - def _installNativePointer(self, fp, returnType, argumentTypes): - typed_python._types.installNativeFunctionPointer( - self.functionTypeObject, - self.index, - fp, - returnType, - tuple(argumentTypes)[len(self.closureVarLookups):], - ) - - class DisableCompiledCode: def __init__(self): pass @@ -637,13 +539,14 @@ def extractCodeObjectNewStatementLineNumbers(codeObject): res = set() - if ast.matches.FunctionDef: + if ast.matches.FunctionDef or ast.matches.Module: res = extractLineNumbersWithStatements(ast.body) return res except Exception: # nasty to swallow the exception like this. At least we report it... - import traceback - traceback.print_exc() + print("FAILED in extractCodeObjectNewStatementLineNumbers") + # import traceback + # traceback.print_exc() return [] diff --git a/typed_python/module_representation_test.py b/typed_python/module_representation_test.py index 9d9380eb7..5a226be35 100644 --- a/typed_python/module_representation_test.py +++ b/typed_python/module_representation_test.py @@ -1,5 +1,5 @@ from typed_python import ModuleRepresentation, SerializationContext -from typed_python import sha_hash, identityHash, ListOf +from typed_python import sha_hash, compilerHash, ListOf import tempfile import unittest @@ -223,7 +223,7 @@ def test_tp_functions(self): assert mr2.getDict()['f'](10) == 11 - assert identityHash(mr.getDict()['f']) != identityHash(mr2.getDict()['f']) + assert compilerHash(mr.getDict()['f']) != compilerHash(mr2.getDict()['f']) def test_tp_classes(self): with tempfile.TemporaryDirectory() as td: diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py new file mode 100644 index 000000000..7ec3e25aa --- /dev/null +++ b/typed_python/py_obj_snapshot_test.py @@ -0,0 +1,307 @@ +import numpy + +from typed_python import ListOf, Alternative, Forward, Function +from typed_python._types import PyObjSnapshot, PyObjGraphSnapshot, _enableTypeAutoresolution + + +class DisableForwardAutoresolve: + def __enter__(self): + _enableTypeAutoresolution(False) + + def __exit__(self, *args): + _enableTypeAutoresolution(True) + + +def test_make_snapshot_basic(): + assert PyObjSnapshot.create(int).kind == 'PrimitiveType' + + assert PyObjSnapshot.create((1, 2, 3)).kind == 'PyTuple' + assert PyObjSnapshot.create((1, 2, 3)).elements[0].kind == 'Instance' + + assert PyObjSnapshot.create({1, 2, 3}).kind == 'PySet' + assert PyObjSnapshot.create({1, 2, 3}).elements[0].kind == 'Instance' + + assert PyObjSnapshot.create([1, 2, 3]).kind == 'PyList' + assert PyObjSnapshot.create([1, 2, 3]).elements[0].kind == 'Instance' + + assert PyObjSnapshot.create({1: 2}).kind == 'PyDict' + assert PyObjSnapshot.create({'1': 2}).elements[0].kind == 'Instance' + assert PyObjSnapshot.create({'1': 2}).keys[0].kind == 'String' + + assert PyObjSnapshot.create(1).kind == 'Instance' + assert PyObjSnapshot.create(1).instance == 1 + + assert PyObjSnapshot.create('1').kind == 'String' + assert PyObjSnapshot.create('1').stringValue == '1' + + assert PyObjSnapshot.create(ListOf(int)((1, 2, 3))).kind == 'Instance' + assert PyObjSnapshot.create(ListOf(int)((1, 2, 3))).instance[1] == 2 + + +def test_snapshot_function(): + y = 10 + + def f(x: int, z=20): + return 10 + y + + snapshot = PyObjSnapshot.create(f) + + assert snapshot.kind == 'PyFunction' + assert snapshot.name == 'f' + assert snapshot.moduleName == 'typed_python.py_obj_snapshot_test' + assert snapshot.func_name.kind == 'String' + assert snapshot.func_module.kind == 'String' + assert snapshot.func_closure.kind == 'PyTuple' + assert snapshot.func_closure.elements[0].kind == 'PyCell' + assert snapshot.func_closure.elements[0].cell_contents.pyobj == 10 + assert snapshot.func_globals.kind == "PyModuleDict" + assert snapshot.func_globals.name == 'typed_python.py_obj_snapshot_test' + assert snapshot.func_annotations.pyobj['x'] is int + assert snapshot.func_defaults.pyobj == (20,) + assert not hasattr(snapshot, 'func_kwdefaults') + + assert snapshot.func_code.kind == 'PyCodeObject' + assert snapshot.func_code.co_argcount == 2 + assert snapshot.func_code.co_kwonlyargcount == 0 + assert snapshot.func_code.co_flags == f.__code__.co_flags + assert snapshot.func_code.co_posonlyargcount == f.__code__.co_posonlyargcount + assert snapshot.func_code.co_nlocals == f.__code__.co_nlocals + assert snapshot.func_code.co_stacksize == f.__code__.co_stacksize + assert snapshot.func_code.co_firstlineno == f.__code__.co_firstlineno + assert snapshot.func_code.co_code.pyobj == f.__code__.co_code + assert snapshot.func_code.co_consts.pyobj == f.__code__.co_consts + assert snapshot.func_code.co_names.pyobj == f.__code__.co_names + assert snapshot.func_code.co_varnames.pyobj == f.__code__.co_varnames + assert snapshot.func_code.co_freevars.pyobj == f.__code__.co_freevars + assert snapshot.func_code.co_cellvars.pyobj == f.__code__.co_cellvars + assert snapshot.func_code.co_name.pyobj == f.__code__.co_name + assert snapshot.func_code.co_filename.pyobj == f.__code__.co_filename + + +def test_snapshot_numpy_internals(): + assert PyObjSnapshot.create(numpy.array).kind == 'NamedPyObject' + + # in theory, we could do better than this by using the reduce pathway. + # however, that would involve calling into arbitrary python code during walk time + # which is dangerous because it can (a) modify objects during the walk and + # (b) it can release the GIL allowing other threads to run (who might then + # modify objects) all of which make it impossible to get a good trace of the + # graph. + assert PyObjSnapshot.create(numpy.sin).kind == 'ArbitraryPyObject' + assert PyObjSnapshot.create(numpy.array([1])).kind == 'ArbitraryPyObject' + + +def test_snapshot_builtins(): + assert PyObjSnapshot.create(__builtins__).kind == 'PyModuleDict' + assert PyObjSnapshot.create(__builtins__).module_dict_of.kind == 'PyModule' + assert PyObjSnapshot.create(set).kind == 'NamedPyObject' + assert PyObjSnapshot.create(print).kind == 'NamedPyObject' + assert PyObjSnapshot.create(len).kind == 'NamedPyObject' + assert PyObjSnapshot.create(range).kind == 'NamedPyObject' + + +def test_snapshot_class(): + class C: + def f(self): + return gInst + + def fMeth(self): + return gMeth + + @property + def aProp(self): + return 20 + + class G(C): + def __init__(self): + self.y = 23 + + @staticmethod + def staticMeth(): + return 10 + + @classmethod + def clsMeth(cls): + return 10 + + gInst = G() + gMeth = gInst.f + + snapshot = PyObjSnapshot.create(C) + + assert snapshot.kind == 'PyClass' + assert snapshot.name == 'C' + assert snapshot.moduleName == 'typed_python.py_obj_snapshot_test' + assert snapshot.cls_dict.kind == 'PyClassDict' + + assert snapshot.cls_dict.byKey['aProp'].kind == 'PyProperty' + assert snapshot.cls_dict.byKey['aProp'].prop_get.kind == 'PyFunction' + + assert snapshot.cls_dict.byKey['f'].kind == 'PyFunction' + + gMethPO = snapshot.cls_dict.byKey['fMeth'].func_closure.elements[0].cell_contents + assert gMethPO.kind == 'PyBoundMethod' + assert gMethPO.meth_self.pyobj is gInst + assert gMethPO.meth_func.kind == 'PyFunction' + + gInstPO = snapshot.cls_dict.byKey['f'].func_closure.elements[0].cell_contents + assert gInstPO.pyobj is gInst + assert gInstPO.inst_dict.kind == 'PyDict' + gInstPO.inst_dict.byKey['y'].kind == 'Instance' + + GPO = gInstPO.inst_type + assert GPO.kind == 'PyClass' + assert GPO.cls_bases.pyobj == (C,) + + # we can't actually tell that this is a class dict because + # its not the base class + assert GPO.cls_dict.kind == 'PyDict' + + assert GPO.cls_dict.byKey['staticMeth'].kind == 'PyStaticMethod' + assert GPO.cls_dict.byKey['staticMeth'].meth_func.kind == 'PyFunction' + + assert GPO.cls_dict.byKey['clsMeth'].kind == 'PyClassMethod' + assert GPO.cls_dict.byKey['clsMeth'].meth_func.kind == 'PyFunction' + + +def test_snapshot_rehydration(): + assert PyObjSnapshot.create(1, False).pyobj == 1 + assert PyObjSnapshot.create('1', False).pyobj == '1' + assert PyObjSnapshot.create(b'1', False).pyobj == b'1' + assert PyObjSnapshot.create([1, 2, 'hi'], False).pyobj == [1, 2, 'hi'] + assert PyObjSnapshot.create((1, 2, 'hi'), False).pyobj == (1, 2, 'hi') + assert PyObjSnapshot.create({1, 2}, False).pyobj == {1, 2} + assert PyObjSnapshot.create({1: 2, '3': 4}, False).pyobj == {1: 2, '3': 4} + assert PyObjSnapshot.create(print, False).pyobj is print + + def f(): + return 1 + + assert PyObjSnapshot.create(f, False).pyobj() == 1 + + class C: + @property + def aProp(self): + return 10 + + @staticmethod + def aSM(x): + return x + 1 + + @classmethod + def aCM(x): + return x + 1 + + def getCInst(self): + return cInst + + def getCInstMeth(self): + return cInstMeth + + cInst = C() + cInstMeth = cInst.getCInstMeth + + C2snapshot = PyObjSnapshot.create(C, False) + C2 = C2snapshot.pyobj + + assert C2 is not C + assert C2.__name__ == C.__name__ + assert C2.__module__ == C.__module__ + assert C2.getCInst is not C.getCInst + + c2Inst = C2snapshot.cls_dict.byKey['getCInst'].func_closure.elements[0].cell_contents.pyobj + c2InstMethSnap = C2snapshot.cls_dict.byKey['getCInstMeth'].func_closure.elements[0].cell_contents + c2InstMeth = c2InstMethSnap.pyobj + + assert c2InstMethSnap.meth_func.func_closure.elements[0].cell_contents.pyobj is c2InstMeth + + assert C2().getCInst() is c2Inst + assert C2().getCInstMeth() is c2InstMeth + assert c2InstMeth() is c2InstMeth + + +def test_snapshot_graph(): + def f(): + return f + + s1 = PyObjSnapshot.create(f) + s2 = PyObjSnapshot.create(f) + + assert s1.graph is not s2.graph + assert s1.kind == 'PyFunction' + assert s1.func_closure.graph is s1.graph + assert s1.func_closure.graph is not s2.func_closure.graph + + +def test_sha_hashing_of_primitive_types(): + assert PyObjSnapshot.create(int).shaHash != PyObjSnapshot.create(float).shaHash + assert PyObjSnapshot.create((1, 2, 3)).shaHash == PyObjSnapshot.create((1, 2, 3)).shaHash + + +def graphIdempotenceCheck(x): + snap = PyObjSnapshot.create(x, False) + snap2 = PyObjSnapshot.create(snap.pyobj, False) + + assert snap.shaHash == snap2.shaHash + + +def test_rehydration_preserves_sha_hashing(): + # disable forward autoresolution so we can look at the type graphs without + # the autoresolver going in and resolving forwards + with DisableForwardAutoresolve(): + graphIdempotenceCheck(int) + graphIdempotenceCheck(ListOf(int)) + + +def test_snapshot_of_object(): + assert PyObjSnapshot.create(object).kind == 'NamedPyObject' + assert PyObjSnapshot.create(type).kind == 'NamedPyObject' + + +def test_snapshot_of_types_basic(): + snap = PyObjSnapshot.create(ListOf(float), False) + assert snap.kind == 'ListOfType' + assert snap.element_type.kind == 'PrimitiveType' + + newGraph = PyObjGraphSnapshot() + newGraph.internalize(snap.graph) + newsnap = newGraph.hashToObject(snap.shaHash) + assert newsnap.kind == 'ListOfType' + assert newsnap.type.__typed_python_category__ == 'ListOf' + + +def test_snapshot_of_types_with_object(): + snap = PyObjSnapshot.create(ListOf(object), False) + assert snap.kind == 'ListOfType' + assert snap.element_type.kind == 'PythonObjectOfTypeType' + + newGraph = PyObjGraphSnapshot() + newGraph.internalize(snap.graph) + newsnap = newGraph.hashToObject(snap.shaHash) + assert newsnap.kind == 'ListOfType' + assert newsnap.type.__typed_python_category__ == 'ListOf' + + +def test_alternative_methods(): + C = Forward(lambda: C) + + X = Alternative("X", x={'c': C}, f=lambda self: 1) + + snap = PyObjSnapshot.create(X, False) + snap2 = PyObjSnapshot.create(snap.pyobj, False) + + assert snap.shaHash == snap2.shaHash + assert snap.pyobj.f.__name__ == 'f' + + +def test_snapshot_of_function(): + closureVar = 10 + + @Function + def f(): + return closureVar + + snap = PyObjSnapshot.create(type(f), False) + snap2 = PyObjSnapshot.create(snap.pyobj, False) + + assert snap.shaHash == snap2.shaHash diff --git a/typed_python/python_ast.py b/typed_python/python_ast.py index 49fbccd1c..453091345 100644 --- a/typed_python/python_ast.py +++ b/typed_python/python_ast.py @@ -24,29 +24,30 @@ import types import traceback -from typed_python._types import Forward, Alternative, TupleOf, OneOf - +from typed_python._types import ( + Forward, Alternative, TupleOf, OneOf, resolveForwardDefinedType +) # forward declarations. -Module = Forward("Module") -Statement = Forward("Statement") -Expr = Forward("Expr") -Arg = Forward("Arg") -NumericConstant = Forward("NumericConstant") -ExprContext = Forward("ExprContext") -BooleanOp = Forward("BooleanOp") -BinaryOp = Forward("BinaryOp") -UnaryOp = Forward("UnaryOp") -ComparisonOp = Forward("ComparisonOp") -Comprehension = Forward("Comprehension") -ExceptionHandler = Forward("ExceptionHandler") -Arguments = Forward("Arguments") -Keyword = Forward("Keyword") -Alias = Forward("Alias") -WithItem = Forward("WithItem") -TypeIgnore = Forward("TypeIgnore") - -Module = Module.define(Alternative( +Module = Forward(lambda: Module) +Statement = Forward(lambda: Statement) +Expr = Forward(lambda: Expr) +Arg = Forward(lambda: Arg) +NumericConstant = Forward(lambda: NumericConstant) +ExprContext = Forward(lambda: ExprContext) +BooleanOp = Forward(lambda: BooleanOp) +BinaryOp = Forward(lambda: BinaryOp) +UnaryOp = Forward(lambda: UnaryOp) +ComparisonOp = Forward(lambda: ComparisonOp) +Comprehension = Forward(lambda: Comprehension) +ExceptionHandler = Forward(lambda: ExceptionHandler) +Arguments = Forward(lambda: Arguments) +Keyword = Forward(lambda: Keyword) +Alias = Forward(lambda: Alias) +WithItem = Forward(lambda: WithItem) +TypeIgnore = Forward(lambda: TypeIgnore) + +Module = Alternative( "Module", Module={ "body": TupleOf(Statement), @@ -55,12 +56,12 @@ Expression={'body': Expr}, Interactive={'body': TupleOf(Statement)}, Suite={"body": TupleOf(Statement)} -)) +) -TypeIgnore = TypeIgnore.define(Alternative( +TypeIgnore = Alternative( "TypeIgnore", Item={'lineno': int, 'tag': str} -)) +) def statementStrLines(self): @@ -161,7 +162,7 @@ def StatementStr(self): return "\n".join(list(statementStrLines(self))) -Statement = Statement.define(Alternative( +Statement = Alternative( "Statement", FunctionDef={ "name": str, @@ -356,7 +357,7 @@ def StatementStr(self): 'filename': str }, __str__=StatementStr -)) +) def ExpressionStr(self): @@ -475,7 +476,7 @@ def ExpressionStr(self): return str(type(self)) -Expr = Expr.define(Alternative( +Expr = Alternative( "Expr", BoolOp={ "op": BooleanOp, @@ -682,9 +683,9 @@ def ExpressionStr(self): 'filename': str }, __str__=ExpressionStr -)) +) -NumericConstant = NumericConstant.define(Alternative( +NumericConstant = Alternative( "NumericConstant", Int={"value": int}, Long={"value": str}, @@ -700,9 +701,9 @@ def ExpressionStr(self): ) else "None" if self.matches.None_ else f"{self.real} + {self.imag}j" if self.matches.Complex else "Unknown" ) -)) +) -ExprContext = ExprContext.define(Alternative( +ExprContext = Alternative( "ExprContext", Load={}, Store={}, @@ -710,15 +711,15 @@ def ExpressionStr(self): AugLoad={}, AugStore={}, Param={} -)) +) -BooleanOp = BooleanOp.define(Alternative( +BooleanOp = Alternative( "BooleanOp", And={}, Or={} -)) +) -BinaryOp = BinaryOp.define(Alternative( +BinaryOp = Alternative( "BinaryOp", Add={}, Sub={}, @@ -733,17 +734,17 @@ def ExpressionStr(self): BitAnd={}, FloorDiv={}, MatMult={} -)) +) -UnaryOp = UnaryOp.define(Alternative( +UnaryOp = Alternative( "UnaryOp", Invert={}, Not={}, UAdd={}, USub={} -)) +) -ComparisonOp = ComparisonOp.define(Alternative( +ComparisonOp = Alternative( "ComparisonOp", Eq={}, NotEq={}, @@ -755,9 +756,9 @@ def ExpressionStr(self): IsNot={}, In={}, NotIn={} -)) +) -Comprehension = Comprehension.define(Alternative( +Comprehension = Alternative( "Comprehension", Item={ "target": Expr, @@ -765,9 +766,9 @@ def ExpressionStr(self): "ifs": TupleOf(Expr), "is_async": bool } -)) +) -ExceptionHandler = ExceptionHandler.define(Alternative( +ExceptionHandler = Alternative( "ExceptionHandler", Item={ "type": OneOf(Expr, None), @@ -777,9 +778,9 @@ def ExpressionStr(self): 'col_offset': int, 'filename': str } -)) +) -Arguments = Arguments.define(Alternative( +Arguments = Alternative( "Arguments", Item={ **({'posonlyargs': TupleOf(Arg)} if sys.version_info.minor >= 8 else {}), @@ -800,9 +801,9 @@ def ExpressionStr(self): + ([self.vararg.arg] if self.vararg else []) + [a.arg for a in self.kwonlyargs] + ([self.kwarg.arg] if self.kwarg else []) -)) +) -Arg = Arg.define(Alternative( +Arg = Alternative( "Arg", Item={ 'arg': str, @@ -811,18 +812,18 @@ def ExpressionStr(self): 'col_offset': int, 'filename': str } -)) +) -Keyword = Keyword.define(Alternative( +Keyword = Alternative( "Keyword", Item={ "arg": OneOf(None, str), "value": Expr, **({'line_number': int, 'col_offset': int, 'filename': str} if sys.version_info.minor >= 9 else {}) } -)) +) -Alias = Alias.define(Alternative( +Alias = Alternative( "Alias", Item={ "name": str, @@ -833,15 +834,37 @@ def ExpressionStr(self): 'filename': str } if sys.version_info.minor >= 10 else {}) } -)) +) -WithItem = WithItem.define(Alternative( +WithItem = Alternative( "WithItem", Item={ "context_expr": Expr, "optional_vars": OneOf(Expr, None), } -)) +) + + +# because this module is defined inside of typed_python, the autoresolver will not be +# enabled yet. +Module = resolveForwardDefinedType(Module) +Statement = resolveForwardDefinedType(Statement) +Expr = resolveForwardDefinedType(Expr) +Arg = resolveForwardDefinedType(Arg) +NumericConstant = resolveForwardDefinedType(NumericConstant) +ExprContext = resolveForwardDefinedType(ExprContext) +BooleanOp = resolveForwardDefinedType(BooleanOp) +BinaryOp = resolveForwardDefinedType(BinaryOp) +UnaryOp = resolveForwardDefinedType(UnaryOp) +ComparisonOp = resolveForwardDefinedType(ComparisonOp) +Comprehension = resolveForwardDefinedType(Comprehension) +ExceptionHandler = resolveForwardDefinedType(ExceptionHandler) +Arguments = resolveForwardDefinedType(Arguments) +Keyword = resolveForwardDefinedType(Keyword) +Alias = resolveForwardDefinedType(Alias) +WithItem = resolveForwardDefinedType(WithItem) +TypeIgnore = resolveForwardDefinedType(TypeIgnore) + numericConverters = { int: lambda x: NumericConstant.Int(value=x), @@ -886,123 +909,6 @@ def makeExtSlice(dims): return Expr.Tuple(elts=dims) -# map Python AST types to our syntax-tree types (defined `ove) -converters = { - ast.Module: Module.Module, - ast.Expression: Module.Expression, - ast.Interactive: Module.Interactive, - ast.Suite: Module.Suite, - ast.FunctionDef: Statement.FunctionDef, - ast.ClassDef: Statement.ClassDef, - ast.Return: Statement.Return, - ast.Delete: Statement.Delete, - ast.Assign: Statement.Assign, - ast.AugAssign: Statement.AugAssign, - ast.AnnAssign: Statement.AnnAssign, - ast.For: Statement.For, - ast.While: Statement.While, - ast.If: Statement.If, - ast.With: Statement.With, - ast.Raise: Statement.Raise, - ast.Try: Statement.Try, - ast.Assert: Statement.Assert, - ast.Import: Statement.Import, - ast.ImportFrom: Statement.ImportFrom, - ast.Global: Statement.Global, - ast.Nonlocal: Statement.NonLocal, - ast.Expr: Statement.Expr, - ast.Pass: Statement.Pass, - ast.Break: Statement.Break, - ast.Continue: Statement.Continue, - ast.BoolOp: Expr.BoolOp, - ast.BinOp: Expr.BinOp, - ast.UnaryOp: Expr.UnaryOp, - ast.Lambda: Expr.Lambda, - ast.IfExp: Expr.IfExp, - ast.Dict: Expr.Dict, - ast.Set: Expr.Set, - ast.JoinedStr: Expr.JoinedStr, - ast.Bytes: Expr.Bytes, - ast.Constant: Expr.Constant, - ast.FormattedValue: Expr.FormattedValue, - ast.ListComp: Expr.ListComp, - ast.AsyncFunctionDef: Statement.AsyncFunctionDef, - ast.AsyncWith: Statement.AsyncWith, - ast.AsyncFor: Statement.AsyncFor, - ast.Await: Expr.Await, - ast.SetComp: Expr.SetComp, - ast.DictComp: Expr.DictComp, - ast.GeneratorExp: Expr.GeneratorExp, - ast.Yield: Expr.Yield, - ast.YieldFrom: Expr.YieldFrom, - ast.Compare: Expr.Compare, - ast.Call: Expr.Call, - ast.Num: createPythonAstConstant, - ast.Str: createPythonAstString, - ast.Attribute: Expr.Attribute, - ast.Subscript: Expr.Subscript, - ast.Name: Expr.Name, - ast.NameConstant: makeNameConstant, - ast.List: Expr.List, - ast.Tuple: Expr.Tuple, - ast.Starred: Expr.Starred, - ast.Load: ExprContext.Load, - ast.Store: ExprContext.Store, - ast.Del: ExprContext.Del, - ast.AugLoad: ExprContext.AugLoad, - ast.AugStore: ExprContext.AugStore, - ast.Param: ExprContext.Param, - ast.Ellipsis: makeEllipsis, - ast.Slice: Expr.Slice, - ast.ExtSlice: makeExtSlice, - ast.Index: lambda value: value, - ast.And: BooleanOp.And, - ast.Or: BooleanOp.Or, - ast.Add: BinaryOp.Add, - ast.Sub: BinaryOp.Sub, - ast.Mult: BinaryOp.Mult, - ast.MatMult: BinaryOp.MatMult, - ast.Div: BinaryOp.Div, - ast.Mod: BinaryOp.Mod, - ast.Pow: BinaryOp.Pow, - ast.LShift: BinaryOp.LShift, - ast.RShift: BinaryOp.RShift, - ast.BitOr: BinaryOp.BitOr, - ast.BitXor: BinaryOp.BitXor, - ast.BitAnd: BinaryOp.BitAnd, - ast.FloorDiv: BinaryOp.FloorDiv, - ast.Invert: UnaryOp.Invert, - ast.Not: UnaryOp.Not, - ast.UAdd: UnaryOp.UAdd, - ast.USub: UnaryOp.USub, - ast.Eq: ComparisonOp.Eq, - ast.NotEq: ComparisonOp.NotEq, - ast.Lt: ComparisonOp.Lt, - ast.LtE: ComparisonOp.LtE, - ast.Gt: ComparisonOp.Gt, - ast.GtE: ComparisonOp.GtE, - ast.Is: ComparisonOp.Is, - ast.IsNot: ComparisonOp.IsNot, - ast.In: ComparisonOp.In, - ast.NotIn: ComparisonOp.NotIn, - ast.comprehension: Comprehension.Item, - ast.excepthandler: lambda x: x, - ast.ExceptHandler: ExceptionHandler.Item, - ast.arguments: Arguments.Item, - ast.arg: Arg.Item, - ast.keyword: Keyword.Item, - ast.alias: Alias.Item, - ast.withitem: WithItem.Item, - **({'ast.type_ignore': TypeIgnore.Item} if sys.version_info.minor >= 8 else {}), -} - -# most converters map to an alternative type -reverseConverters = { - t: v for v, t in converters.items() - if hasattr(t, '__typed_python_category__') and t.__typed_python_category__ == "ConcreteAlternative" -} - - def convertAlgebraicArgs(pyAst, *members): members = [x for x in members if x not in ['line_number', 'col_offset']] return {m: convertAlgebraicToPyAst(getattr(pyAst, m)) for m in members} @@ -1372,7 +1278,7 @@ def cacheAstForCode(code, pyAst): codeConstants = [c for c in code.co_consts if isinstance(c, types.CodeType)] - if isinstance(pyAst, (Statement.FunctionDef, Expr.Lambda, Statement.ClassDef, Statement.AsyncFunctionDef)): + if isinstance(pyAst, (Statement.FunctionDef, Expr.Lambda, Statement.ClassDef, Statement.AsyncFunctionDef, Module)): funcDefs = extractFunctionDefsInOrder(pyAst.body) else: funcDefs = extractFunctionDefsInOrder(pyAst.generators) @@ -1501,3 +1407,120 @@ def evaluateFunctionDefWithLocalsInCells(pyAst, globals, locals, stripAnnotation cacheAstForCode(inner.__code__, pyAst) return inner + + +# map Python AST types to our syntax-tree types +converters = { + ast.Module: Module.Module, + ast.Expression: Module.Expression, + ast.Interactive: Module.Interactive, + ast.Suite: Module.Suite, + ast.FunctionDef: Statement.FunctionDef, + ast.ClassDef: Statement.ClassDef, + ast.Return: Statement.Return, + ast.Delete: Statement.Delete, + ast.Assign: Statement.Assign, + ast.AugAssign: Statement.AugAssign, + ast.AnnAssign: Statement.AnnAssign, + ast.For: Statement.For, + ast.While: Statement.While, + ast.If: Statement.If, + ast.With: Statement.With, + ast.Raise: Statement.Raise, + ast.Try: Statement.Try, + ast.Assert: Statement.Assert, + ast.Import: Statement.Import, + ast.ImportFrom: Statement.ImportFrom, + ast.Global: Statement.Global, + ast.Nonlocal: Statement.NonLocal, + ast.Expr: Statement.Expr, + ast.Pass: Statement.Pass, + ast.Break: Statement.Break, + ast.Continue: Statement.Continue, + ast.BoolOp: Expr.BoolOp, + ast.BinOp: Expr.BinOp, + ast.UnaryOp: Expr.UnaryOp, + ast.Lambda: Expr.Lambda, + ast.IfExp: Expr.IfExp, + ast.Dict: Expr.Dict, + ast.Set: Expr.Set, + ast.JoinedStr: Expr.JoinedStr, + ast.Bytes: Expr.Bytes, + ast.Constant: Expr.Constant, + ast.FormattedValue: Expr.FormattedValue, + ast.ListComp: Expr.ListComp, + ast.AsyncFunctionDef: Statement.AsyncFunctionDef, + ast.AsyncWith: Statement.AsyncWith, + ast.AsyncFor: Statement.AsyncFor, + ast.Await: Expr.Await, + ast.SetComp: Expr.SetComp, + ast.DictComp: Expr.DictComp, + ast.GeneratorExp: Expr.GeneratorExp, + ast.Yield: Expr.Yield, + ast.YieldFrom: Expr.YieldFrom, + ast.Compare: Expr.Compare, + ast.Call: Expr.Call, + ast.Num: createPythonAstConstant, + ast.Str: createPythonAstString, + ast.Attribute: Expr.Attribute, + ast.Subscript: Expr.Subscript, + ast.Name: Expr.Name, + ast.NameConstant: makeNameConstant, + ast.List: Expr.List, + ast.Tuple: Expr.Tuple, + ast.Starred: Expr.Starred, + ast.Load: ExprContext.Load, + ast.Store: ExprContext.Store, + ast.Del: ExprContext.Del, + ast.AugLoad: ExprContext.AugLoad, + ast.AugStore: ExprContext.AugStore, + ast.Param: ExprContext.Param, + ast.Ellipsis: makeEllipsis, + ast.Slice: Expr.Slice, + ast.ExtSlice: makeExtSlice, + ast.Index: lambda value: value, + ast.And: BooleanOp.And, + ast.Or: BooleanOp.Or, + ast.Add: BinaryOp.Add, + ast.Sub: BinaryOp.Sub, + ast.Mult: BinaryOp.Mult, + ast.MatMult: BinaryOp.MatMult, + ast.Div: BinaryOp.Div, + ast.Mod: BinaryOp.Mod, + ast.Pow: BinaryOp.Pow, + ast.LShift: BinaryOp.LShift, + ast.RShift: BinaryOp.RShift, + ast.BitOr: BinaryOp.BitOr, + ast.BitXor: BinaryOp.BitXor, + ast.BitAnd: BinaryOp.BitAnd, + ast.FloorDiv: BinaryOp.FloorDiv, + ast.Invert: UnaryOp.Invert, + ast.Not: UnaryOp.Not, + ast.UAdd: UnaryOp.UAdd, + ast.USub: UnaryOp.USub, + ast.Eq: ComparisonOp.Eq, + ast.NotEq: ComparisonOp.NotEq, + ast.Lt: ComparisonOp.Lt, + ast.LtE: ComparisonOp.LtE, + ast.Gt: ComparisonOp.Gt, + ast.GtE: ComparisonOp.GtE, + ast.Is: ComparisonOp.Is, + ast.IsNot: ComparisonOp.IsNot, + ast.In: ComparisonOp.In, + ast.NotIn: ComparisonOp.NotIn, + ast.comprehension: Comprehension.Item, + ast.excepthandler: lambda x: x, + ast.ExceptHandler: ExceptionHandler.Item, + ast.arguments: Arguments.Item, + ast.arg: Arg.Item, + ast.keyword: Keyword.Item, + ast.alias: Alias.Item, + ast.withitem: WithItem.Item, + **({'ast.type_ignore': TypeIgnore.Item} if sys.version_info.minor >= 8 else {}), +} + +# most converters map to an alternative type +reverseConverters = { + t: v for v, t in converters.items() + if hasattr(t, '__typed_python_category__') and t.__typed_python_category__ == "ConcreteAlternative" +} diff --git a/typed_python/test_serialize_recursive_types.py b/typed_python/test_serialize_recursive_types.py new file mode 100644 index 000000000..3db7766c2 --- /dev/null +++ b/typed_python/test_serialize_recursive_types.py @@ -0,0 +1,115 @@ +# Copyright 2017-2023 typed_python Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typed_python.test_util import callFunctionInFreshProcess +from typed_python import ( + Alternative, SerializationContext, Forward, resolveForwardDefinedType, Function +) + + +s = SerializationContext() + + +def test_can_serialize_function(): + @Function + def f(x: int): + return x + 1 + + assert type(f) is type(s.deserialize(s.serialize(f))) + + +def test_can_serialize_recursive_alternative(): + def makeA(): + A = Forward("A") + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: self + ) + ) + + A = A.resolve() + + return A + + assert makeA() is makeA() + assert makeA() is s.deserialize(s.serialize(makeA())) + + +def test_can_serialize_recursive_alternative_out_of_proc(): + def makeA(): + A = Forward("A") + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: self + ) + ) + + A = A.resolve() + + return A + + A_external = callFunctionInFreshProcess(makeA) + + assert makeA() is A_external + + +def test_can_serialize_recursive_alternative_out_of_proc_with_ref_to_self(): + def makeA(): + A = Forward("A") + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: A + ) + ) + + A = A.resolve() + + return A + + A_external = callFunctionInFreshProcess(makeA) + + assert makeA() is A_external + + +def test_can_serialize_recursive_alternative_out_of_proc_with_ref_to_own_forward(): + def makeA(): + A = Forward("A") + A_fwd = A + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: A, + forward=lambda self: A_fwd + ) + ) + + A = A.resolve() + + return A + + A_external = callFunctionInFreshProcess(makeA) + + assert makeA() is A_external diff --git a/typed_python/test_util.py b/typed_python/test_util.py index 062a3a664..351c45642 100644 --- a/typed_python/test_util.py +++ b/typed_python/test_util.py @@ -15,6 +15,7 @@ import psutil import time import threading +import textwrap import tempfile import pickle import subprocess @@ -94,7 +95,7 @@ def instantiateFiles(filesToWrite, tf): ) -def callFunctionInFreshProcess(func, argTup, compilerCacheDir=None, showStdout=False, extraEnvs={}): +def callFunctionInFreshProcess(func, argTup=(), compilerCacheDir=None, showStdout=False, extraEnvs={}): """Return the value of a function evaluated on some arguments in a subprocess. We use this to test the semantics of anonymous functions and classes in a process @@ -198,7 +199,7 @@ def evaluateExprInFreshProcess(filesToWrite, expression, compilerCacheDir=None, "".join(f"import {modname};" for modname in namesToImport) + ( f"import pickle;" f"import typed_python;" - f"from typed_python._types import identityHash, recursiveTypeGroup;" + f"from typed_python._types import compilerHash, recursiveTypeGroup;" f"from typed_python import *;" f"print(repr(pickle.dumps({expression})))" ) @@ -244,6 +245,14 @@ def __init__(self): self.dir = tempfile.TemporaryDirectory() def evaluateInto(self, code, moduleDict): + # strip any blank lines at the end of 'code' and dedent it + codeLines = code.split("\n") + while codeLines and not codeLines[-1].strip(): + codeLines = codeLines[:-1] + code = "\n".join(codeLines) + + code = textwrap.dedent(code) + filename = os.path.abspath( os.path.join(self.dir.name, sha_hash(code).hexdigest) ) diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py new file mode 100644 index 000000000..48ab4ec87 --- /dev/null +++ b/typed_python/type_construction_test.py @@ -0,0 +1,902 @@ +import pytest + +from typed_python import ( + TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, + Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, + Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled, + forwardDefinitionsFor, Entrypoint +) + +from typed_python.test_util import CodeEvaluator + + +def test_function_in_anonymous_module_callable(): + c = CodeEvaluator() + m = {} + + c.evaluateInto(""" + from typed_python import Function + + y = 1 + + @Function + def f(x): + return x + y + """, m) + f = m['f'] + + assert f(10) == 11 + assert f.ClosureType.ElementTypes[0].ElementNames[0] == ' _globals' + + +def test_entrypoint_basic(): + @Entrypoint + def f(x: int): + return x + 1 + + @Entrypoint + def g(): + return f(2) + + print(g.ClosureType) + print(g.overloads[0]) + + from typed_python import _types + g2 = _types.prepareArgumentToBePassedToCompiler(g) + print("g2: ") + print(g2.overloads[0].globals) + print(g2.overloads[0].closureVarLookups) + print(g2.overloads[0].closureVarLookups['f'][0]) + print(g2.overloads[0].closureVars) + print() + + assert g() == 3 + + +def test_identity_hash_of_alternative_stable(): + e = CodeEvaluator() + m = {} + + e.evaluateInto(""" + from typed_python._types import identityHash, Alternative + + A = Alternative( + "A", + f=lambda x: (A, g) + ) + + h1 = identityHash(A) + + def g(): + pass + + h2 = identityHash(A) + """, m) + + assert m['h1'] == m['h2'] + + +def test_forward_class_exposes_functions(): + class C(Class): + x = Member(lambda: B) + + def f(self): + return C + + B = int + + CResolved = resolveForwardDefinedType(C) + + assert C.HeldClass.Class is C + assert CResolved.HeldClass.Class is CResolved + + assert not C.IsFinal + assert not CResolved.IsFinal + + assert C.MRO == (C,) + assert CResolved.MRO == (CResolved,) + + assert CResolved.MemberNames == ('x',) + + assert C.MemberNames == ('x',) + + assert C.f.overloads[0].globals['C'].kind == 'GlobalInCell' + + +def test_function_global_resolving_to_builtin(): + @NotCompiled + def f(): + return str(x) + + assert f.overloads[0].globals['str'].getValue() is str + + +def test_reference_to_builtin_is_resolvable(): + A = Alternative("A", X={}, f=lambda self: str(self) + B) + + # A is forward defined because it depends on B + assert isForwardDefined(A) + + assert A.f.overloads[0].globals['str'].kind == 'NamedModuleMember' + assert A.f.overloads[0].globals['str'].moduleName == 'builtins' + assert A.f.overloads[0].globals['str'].name == 'str' + + +def test_function_cell_bindings_are_resolved_to_constants(): + def CofX(x): + class C(Class): + def f(self): + return x + + return C + + assert CofX(1).f.overloads[0].globals['x'].kind == 'GlobalInCell' + assert CofX(1).f.overloads[0].globals['x'].getValue() is 1 + assert CofX(2).f.overloads[0].globals['x'].getValue() is 2 + + +def test_reference_to_builtin_doesnt_prevent_autoresolve(): + A = Alternative("A", X={}, f=lambda self: str(self)) + + assert not isForwardDefined(A) + + assert A().f() == str(A()) + + +def test_type_looks_resolvable_alternative(): + A = Alternative("A_", X={}, f=lambda self: B) + + # hide a reference to this forward in a tuple that the autoresolver can't see + AFwd = (A,) + + assert isForwardDefined(A) + + assert typeLooksResolvable(A, unambiguously=False) + assert not typeLooksResolvable(A, unambiguously=True) + + B = Alternative("B_", X={}, g=lambda self: A) + + assert typeLooksResolvable(A, unambiguously=False) + assert typeLooksResolvable(A, unambiguously=True) + assert typeLooksResolvable(B, unambiguously=False) + assert typeLooksResolvable(B, unambiguously=True) + + A = resolveForwardDefinedType(A) + B = resolveForwardDefinedType(B) + + bGlobal = A.f.overloads[0].globals['B'] + assert bGlobal.kind == "GlobalInCell" + assert bGlobal.getValue() is B + + bFwdGlobal = AFwd[0].f.overloads[0].globals['B'] + assert bFwdGlobal.kind == "GlobalInCell" + assert bFwdGlobal.getValue() is B + + assert not isForwardDefined(A) + assert isForwardDefined(AFwd[0]) + + assert A().f() is B + assert B().g() is A + + +def test_autoresolve_unnamed_type_with_named_forward(): + # this defines 'F' as a forward. When 'F' gets assigned to something other than a forward + # the type will become resolvable. + F = Forward(lambda: F) + + # this defines 'G' - there's no reference in the type system to it + G = OneOf(None, F) + + # this allows 'F' and 'G' to be resolved. + F = ListOf(G) + + # the system was able to autoresolve both F and G because they were assigned + assert not isForwardDefined(F) + assert not isForwardDefined(G) + + +def test_held_of_forward_class(): + class C(Class): + def f(self): + return H + + assert isForwardDefined(C) + + @Held + class H(Class): + def f(self): + return C + + assert not isForwardDefined(H) + assert not isForwardDefined(C) + + +def test_autoresolve_on_decorator(): + B = Forward(lambda: B) + + class B(Class): + def blah(self) -> B: + return self + + @Function + def callIt(x: B): + pass + + assert not isForwardDefined(B) + assert not isForwardDefined(type(callIt)) + + +def test_class_with_entrypointed_recursive_method(): + class C(Class): + @NotCompiled + def f(self): + return C().g() + + C() + + +def test_autoresolve_forwards_with_nonforwards(): + class C(Class): + def f(self): + return T + + assert isForwardDefined(C) + + with pytest.raises(TypeError): + C() + + T = int + + assert typeLooksResolvable(C, unambiguously=True) + + C = resolveForwardDefinedType(C) + + C() + + +def test_autoresolve_unnamed_type_with_anonymous_forward(): + G = OneOf(None, lambda: F) + F = ListOf(G) + + # the system was able to autoresolve both F and G because they were assigned + assert not isForwardDefined(F) + assert not isForwardDefined(G) + + +def test_untyped_functions_are_not_forward(): + # this type is not forward defined because it should be holding 'g' in the closure of + # 'f' itself + f = Function(lambda: g()) + assert not isForwardDefined(type(f)) + + g = Function(lambda: f()) + assert not isForwardDefined(type(g)) + + +def test_autoresolve_forwards(): + A = Alternative("A", X={}, Y=dict(inst=lambda: B)) + B = Alternative("B", X={}, Y=dict(inst=lambda: A)) + + assert not isForwardDefined(A) + assert not isForwardDefined(B) + + +def test_autoresolve_forwards_3cycle(): + A = Alternative("A", X={}, Y=dict(inst=lambda: C)) + B = Alternative("B", X={}, Y=dict(inst=lambda: A)) + + assert isForwardDefined(A) + assert isForwardDefined(B) + + C = Alternative("C", X={}, Y=dict(inst=lambda: B)) + + assert not isForwardDefined(A) + assert not isForwardDefined(B) + assert not isForwardDefined(C) + + +def test_invalid_forwards(): + with pytest.raises(TypeError): + Forward(lambda: 1) + + with pytest.raises(TypeError): + Forward(lambda: A + B) # noqa + + X = 10 + with pytest.raises(TypeError): + Forward(lambda: X + B) # noqa + + +def test_implicitly_recursive_types(): + O_fwd = ListOf(OneOf(int, Forward(lambda: O))) + + assert isForwardDefined(O_fwd) + assert not typeLooksResolvable(O_fwd) + + O = O_fwd + + assert typeLooksResolvable(O_fwd) + + O = resolveForwardDefinedType(O) + O([O()]) + + +def test_dual_recursive_types_direct_instantiation(): + O1 = Forward("O1") + O2 = Forward("O2") + + O1.define(ListOf(OneOf(int, O2))) + O2.define(ListOf(OneOf(int, O1))) + + assert isForwardDefined(O1) + assert isForwardDefined(O2) + + O1 = resolveForwardDefinedType(O1) + O2 = resolveForwardDefinedType(O2) + + +def test_dual_recursive_types_direct_instantiation_with_alternatives(): + O1 = Forward("O1") + O2 = Forward("O2") + + O1.define(Alternative("O1", X={}, Y={'a': O2}, f=lambda self: O2)) + O2.define(Alternative("O2", X={}, Y={'a': O1}, f=lambda self: O1)) + + assert not isForwardDefined(O1) + assert not isForwardDefined(O2) + + +def test_nonforward_definition(): + assert not isForwardDefined(OneOf(int, float)) + assert not isForwardDefined(ListOf(int)) + assert not isForwardDefined(Dict(int, int)) + assert not isForwardDefined(TupleOf(int)) + assert not isForwardDefined(Set(int)) + assert not isForwardDefined(Tuple(int, int)) + assert not isForwardDefined(PointerTo(int)) + assert not isForwardDefined(RefTo(int)) + assert not isForwardDefined(NamedTuple(x=int, y=float)) + assert not isForwardDefined(Alternative("A", x={}, y={})) + assert not isForwardDefined(Alternative("A", x={}, y={}).x) + assert not isForwardDefined(Function(lambda x: x)) + + +def test_forward_definition(): + F = Forward() + + assert issubclass(F, Forward) + assert isForwardDefined(F) + + assert isForwardDefined(Dict(int, F)) + assert isForwardDefined(OneOf(int, F)) + assert isForwardDefined(ListOf(F)) + assert isForwardDefined(TupleOf(F)) + assert isForwardDefined(Set(F)) + assert isForwardDefined(Tuple(int, F)) + assert isForwardDefined(PointerTo(F)) + assert isForwardDefined(RefTo(F)) + assert isForwardDefined(NamedTuple(x=int, y=F)) + assert isForwardDefined(Alternative("A", x={'f': F})) + assert isForwardDefined(Alternative("A", x={'f': F}).x) + + +def test_forward_one_of(): + F = Forward() + + O = OneOf(int, F) + + assert isForwardDefined(O) + + with pytest.raises(TypeError): + bytecount(O) + + # F is now defined. + F.define(float) + + assert resolveForwardDefinedType(F) is float + + O_resolved = resolveForwardDefinedType(O) + assert not isForwardDefined(O_resolved) + + assert bytecount(O_resolved) == 1 + bytecount(float) + + +def test_oneof(): + assert OneOf(int, float) is OneOf(int, float) + assert OneOf(int, float) is not OneOf(int, None) + + +def test_recursive_definition_of_self(): + F = Forward() + + F.define(ListOf(F)) + + F = resolveForwardDefinedType(F) + + assert F.__name__ == 'ListOf(^0)' + + +def test_make_recursive_list_of_autoresolve(): + def makeRecursiveListOf(T): + L = ListOf(OneOf(T, Forward(lambda: L))) + return L + + assert not isForwardDefined(makeRecursiveListOf(int)) + + assert makeRecursiveListOf(int) is makeRecursiveListOf(int) + assert makeRecursiveListOf(int) is not makeRecursiveListOf(str) + + +def test_autoresolve_nonstorage_forwards(): + A = Alternative("A", X={}, fA=lambda: B) + B = Alternative("B", X={}, fB=lambda: A) + + assert not isForwardDefined(B) + assert not isForwardDefined(A) + + +def test_recursive_list_of_forward(): + F = Forward() + O = OneOf(None, F) + F.define(ListOf(O)) + + F_resolved = resolveForwardDefinedType(F) + O_resolved = resolveForwardDefinedType(O) + + assert issubclass(F_resolved, ListOf) + assert not isForwardDefined(F_resolved) + assert not isForwardDefined(O_resolved) + assert F_resolved.ElementType is O_resolved + + assert F_resolved.__name__ == 'ListOf(OneOf(None, ^1))' + assert O_resolved.__name__ == 'OneOf(None, ListOf(^1))' + + +def test_define_class_holding_self(): + class C(Class): + anotherC = Member(OneOf(None, lambda: C)) + + def f(self) -> Forward(lambda: C): + return self + + assert not isForwardDefined(C) + + c = C() + assert type(c.f()) is C + + +def test_define_class_holding_self_defined_as_forward(): + C = Forward(lambda: C) + + assert isForwardDefined(C) + assert not typeLooksResolvable(C, True) + + class C(Class): + anotherC = Member(OneOf(None, C)) + + def f(self) -> C: + return self + + assert not isForwardDefined(C) + + c = C() + assert type(c.f()) is C + + +def test_two_mutually_recursive_classes(): + C1 = Forward(lambda: C1) + C2 = Forward(lambda: C2) + + class C1(Class): + holdsC2 = Member(OneOf(None, C2)) + + def f(self) -> C1: + return self + + assert isForwardDefined(C1) + + class C2(Class): + holdsC1 = Member(OneOf(None, C1)) + + def f(self) -> C2: + return self + + assert not isForwardDefined(C1) + assert not isForwardDefined(C2) + + +def test_recursive_list_of_tuple_and_forward(): + F = Forward() + NT = NamedTuple(f=F) + F.define(ListOf(NT)) + + F_resolved = resolveForwardDefinedType(F) + NT_resolved = resolveForwardDefinedType(NT) + + assert issubclass(F_resolved, ListOf) + assert not isForwardDefined(F_resolved) + assert not isForwardDefined(NT_resolved) + assert F_resolved.ElementType is NT_resolved + assert NT_resolved.ElementTypes == (F_resolved,) + assert NT_resolved.ElementNames == ('f',) + + assert F_resolved.__name__ == 'ListOf(NamedTuple(f=^1))' + assert NT_resolved.__name__ == 'NamedTuple(f=ListOf(^1))' + + +def test_recursive_list_of_forward_memoizes(): + def makeTup(): + F = Forward() + O = OneOf(None, F) + F.define(ListOf(O)) + + return resolveForwardDefinedType(F) + + assert makeTup() is makeTup() + + +def test_nonrecursive_list_of_forward_memoizes(): + def makeTup(): + F = Forward() + O = ListOf(F) + F.define(int) + + return resolveForwardDefinedType(O) + + assert makeTup() is makeTup() + assert makeTup() is ListOf(int) + + +def test_recursive_pointer_to(): + def makeP(): + F = Forward() + F.define(PointerTo(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'PointerTo(^0)' + + +def test_recursive_ref_to(): + def makeP(): + F = Forward() + F.define(RefTo(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'RefTo(^0)' + + +def test_recursive_typed_cell(): + def makeP(): + F = Forward() + F.define(TypedCell(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'TypedCell(^0)' + + +def test_recursive_dict(): + def makeP(): + F = Forward() + F.define(Dict(int, F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'Dict(int, ^0)' + + +def test_recursive_const_dict(): + def makeP(): + F = Forward() + F.define(ConstDict(int, F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'ConstDict(int, ^0)' + + +def test_recursive_set(): + def makeP(): + F = Forward() + F.define(Set(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'Set(^0)' + + +def test_recursive_alternative(): + def makeA(): + F = Forward() + F.define( + Alternative( + "F", + A={'f': F}, + B={}, + C={'x': int} + ) + ) + return resolveForwardDefinedType(F) + + assert makeA() is makeA() + assert makeA().__name__ == 'F' + assert makeA().A is makeA().A + makeA().B() + makeA().C(x=10) + + +@pytest.mark.skip(reason='what to do with forwards functions?') +def test_cannot_call_forward_function_instances(): + t = Forward("T") + + @Function + def f(x: t): + return x + + t.define(int) + + with pytest.raises(TypeError): + f(10) + + +@pytest.mark.skip(reason='what to do with forwards functions?') +def test_function_types_coalesc(): + def makeFNonforward(T): + @Function + def f(x: T): + return x + + return type(f) + + def makeFforward(T): + t = Forward("T") + + @Function + def f(x: t): + return x + + t.define(T) + + return resolveForwardDefinedType(type(f)) + + assert makeFNonforward(int) is makeFNonforward(int) + assert identityHash(makeFforward(int)) == identityHash(makeFforward(int)) + assert makeFforward(int) is makeFforward(int) + + +@pytest.mark.skip(reason='what to do with forwards functions?') +def test_instances_of_forward_function_are_forward(): + T = Forward("T") + + @Function + def f(x: T): + return x + + assert isForwardDefined(f) + T.define(int) + + f2 = resolveForwardDefinedType(f) + + assert not isForwardDefined(f2) + + assert f2(1) == 1 + + with pytest.raises(TypeError): + f2("hi") + + +def test_create_value_type_with_forward(): + F = Forward("X") + T = Value(F) + F.define(int) + + assert isForwardDefined(T) + + T_resolved = resolveForwardDefinedType(T) + + assert not isForwardDefined(T_resolved) + assert identityHash(T_resolved).hex() == identityHash(Value(int)).hex() + assert T_resolved is Value(int) + + +def test_create_class_with_methods(): + class C(Class): + def f(self): + return "hi" + + assert not isForwardDefined(C) + + +def test_create_class_with_forward(): + def makeClass(T): + F = Forward("X") + + class C(Class): + m = Member(F) + + assert isForwardDefined(C) + + F.define(T) + + return resolveForwardDefinedType(C) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_create_concrete_class(): + def makeClass(T): + class C(Class): + m = Member(T) + + assert not isForwardDefined(C) + + return C + + assert makeClass(int) is makeClass(int) + assert makeClass(int) is not makeClass(str) + + +def test_create_class_with_recursion(): + def makeClass(T): + C_fwd = Forward("C") + + class C(Class): + c = Member(OneOf(None, C_fwd)) + m = Member(T) + + assert isForwardDefined(C_fwd) + assert isForwardDefined(C) + + C_fwd.define(C) + + return resolveForwardDefinedType(C) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_create_class_with_nonforward_base(): + def makeClass(T): + C_child_fwd = Forward("C_child") + + class CBase(Class): + m = Member(T) + + class CChild(CBase): + pass + + assert not isForwardDefined(CBase) + assert isForwardDefined(C_child_fwd) + + C_child_fwd.define(CChild) + + return resolveForwardDefinedType(CChild) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_create_class_with_forward_base(): + def makeClass(T): + C_base_fwd = Forward("C_base") + C_child_fwd = Forward("C_child") + + class CBase(Class): + c = Member(OneOf(None, C_child_fwd)) + + class CChild(CBase): + m = Member(T) + + assert isForwardDefined(C_base_fwd) + assert isForwardDefined(C_child_fwd) + + C_child_fwd.define(CChild) + C_base_fwd.define(CBase) + + return resolveForwardDefinedType(CChild) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_identity_of_leaked_forward(): + def makeClass(): + A = Forward("A") + A_fwd = A + + A.define( + Alternative( + "A", + X={}, + getA=lambda self: A, + getAForward=lambda self: A_fwd + ) + ) + A = resolveForwardDefinedType(A) + + return A, A_fwd + + A1, A1_fwd = makeClass() + A2, A2_fwd = makeClass() + + # the identity of 'A' should be stable + assert A1 is A2 + + # but the forwards we used to create them could be separate + assert A1_fwd is not A2_fwd + + # they have to point to the same concrete instance + assert isForwardDefined(A1_fwd.get()) + assert isForwardDefined(A2_fwd.get()) + + assert resolveForwardDefinedType(A1_fwd.get()) is A1 + assert resolveForwardDefinedType(A2_fwd.get()) is A1 + + +@pytest.mark.skip(reason='TODO: make this work') +def test_create_class_with_fully_forward_base(): + def makeClass(T): + C_base_fwd = Forward("C_base") + C_child_fwd = Forward("C_child") + + class CBase(Class): + c = Member(OneOf(None, C_child_fwd)) + + class CChild(C_base_fwd): + m = Member(T) + + assert isForwardDefined(C_base_fwd) + assert isForwardDefined(C_child_fwd) + + C_child_fwd.define(CChild) + C_base_fwd.define(CBase) + + return resolveForwardDefinedType(CChild) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +# TODO: +# 1. can we get better names and info for forward-defined types? what does ther +# partially defined state look like +# 2. the CVOV needs to be able to look into TP instances. Value(TupleOf(int)((1, 2, 3))) for +# instance ought to be visible and compilable +# 3. can regular python classes and functions be 'forward declared'. What do we do if we +# forward resolve them? +# 4. serialization +# 5. type functions +# 6. are we accidentally creating and installing forward subclasses in the vtable? +# maybe we need a second pass to wire in 'accepted' forward types +# 7. make sure we thoroughly test the idea that we create a new forward graph with +# part of the graph being new, part being old +# 8. clean-up in the whole Instance/PyInstance layer - kind of nasty +# 9. we should memoize instances of statless Function objects and instances of regular TP lists/tuple of, etc. + +# CHECK: +# OneOfs containing forwards that resolve to other oneofs changing the number +# of items - this needs to work +# Forwards resolving to simple types like ints/floats +# MRTG should be completely ignoring Forwards - ideally they go away entirely +# - if they get leaked, that should be an error! +# how does this play with the ModuleRepresentation stuff? +# how does this play with TypeFunction? +# mutually recursive across modules? diff --git a/typed_python/type_filter.py b/typed_python/type_filter.py deleted file mode 100644 index b697cf831..000000000 --- a/typed_python/type_filter.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017-2019 typed_python Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class TypeFilterBase(type): - def __instancecheck__(self, other): - if not isinstance(other, self.base_type): - return False - - try: - if not self.filter_function(other): - return False - except Exception: - return False - - return True - - -def TypeFilter(base_type, filter_function): - """TypeFilter(base_type, filter_function) - - Produce a 'type object' that can be used in typed python to filter objects by - arbitrary criteria. - """ - class TypeFilter(metaclass=TypeFilterBase): - pass - - TypeFilter.base_type = base_type - TypeFilter.filter_function = filter_function - - return TypeFilter diff --git a/typed_python/type_identity_test.py b/typed_python/type_identity_test.py index ff754759e..d211e7535 100644 --- a/typed_python/type_identity_test.py +++ b/typed_python/type_identity_test.py @@ -21,7 +21,7 @@ from typed_python.test_util import evaluateExprInFreshProcess, callFunctionInFreshProcess from typed_python import ( UInt64, UInt32, - ListOf, TupleOf, Tuple, NamedTuple, Dict, OneOf, Forward, identityHash, + ListOf, TupleOf, Tuple, NamedTuple, Dict, OneOf, Forward, compilerHash, Entrypoint, Class, Member, Final, TypeFunction, SerializationContext, Function, NotCompiled ) @@ -76,7 +76,7 @@ def checkHash(filesToWrite, expression): Returns: a bytes object containing the sha-hash of module.thingToGrab. """ - return Hash(evaluateExprInFreshProcess(filesToWrite, f"identityHash({expression})")) + return Hash(evaluateExprInFreshProcess(filesToWrite, f"compilerHash({expression})")) def returnSerializedValue(filesToWrite, expression, printComments=False): @@ -100,8 +100,8 @@ def test_identity_ignores_function_file_accesses(): def test_identities_of_basic_types_different(): - assert identityHash(int) != identityHash(float) - assert identityHash(TupleOf(int)) != identityHash(TupleOf(float)) + assert compilerHash(int) != compilerHash(float) + assert compilerHash(TupleOf(int)) != compilerHash(TupleOf(float)) def test_object_graph_instability_is_noticed(): @@ -139,8 +139,8 @@ def f(x: int): def g(x: int): return f(x) - identityHash(f) - identityHash(NotCompiled(f)) + compilerHash(f) + compilerHash(NotCompiled(f)) hashInstability = checkForHashInstability() @@ -160,9 +160,9 @@ def functionAccessingModuleLevelThreadLocal(): def test_identity_of_function_accessing_thread_local(): print(typesAndObjectsVisibleToCompilerFrom(type(functionAccessingModuleLevelThreadLocal))) - identityHash(functionAccessingModuleLevelThreadLocal) + compilerHash(functionAccessingModuleLevelThreadLocal) moduleLevelThreadLocal.anything = True - identityHash(functionAccessingModuleLevelThreadLocal) + compilerHash(functionAccessingModuleLevelThreadLocal) hashInstability = checkForHashInstability() @@ -172,8 +172,8 @@ def test_identity_of_function_accessing_thread_local(): def test_identity_of_method_descriptors(): - assert identityHash(ListOf(int).append) != identityHash(ListOf(float).append) - assert identityHash(ListOf(int).append) != identityHash(ListOf(int).extend) + assert compilerHash(ListOf(int).append) != compilerHash(ListOf(float).append) + assert compilerHash(ListOf(int).append) != compilerHash(ListOf(int).extend) def test_class_and_held_class_in_group(): @@ -190,34 +190,34 @@ class C(Class): def test_identity_of_register_types(): - assert isinstance(identityHash(UInt64), bytes) - assert len(identityHash(UInt64)) == 20 + assert isinstance(compilerHash(UInt64), bytes) + assert len(compilerHash(UInt64)) == 20 - assert identityHash(UInt64) != identityHash(UInt32) + assert compilerHash(UInt64) != compilerHash(UInt32) def test_identity_of_list_of(): - assert identityHash(ListOf(int)) != identityHash(ListOf(float)) - assert identityHash(ListOf(int)) == identityHash(ListOf(int)) - assert identityHash(ListOf(int)) != identityHash(TupleOf(int)) + assert compilerHash(ListOf(int)) != compilerHash(ListOf(float)) + assert compilerHash(ListOf(int)) == compilerHash(ListOf(int)) + assert compilerHash(ListOf(int)) != compilerHash(TupleOf(int)) def test_identity_of_named_tuple_and_tuple(): - assert identityHash(NamedTuple(x=int)) != identityHash(NamedTuple(x=float)) - assert identityHash(NamedTuple(x=int)) == identityHash(NamedTuple(x=int)) - assert identityHash(NamedTuple(x=int)) != identityHash(Tuple(float)) + assert compilerHash(NamedTuple(x=int)) != compilerHash(NamedTuple(x=float)) + assert compilerHash(NamedTuple(x=int)) == compilerHash(NamedTuple(x=int)) + assert compilerHash(NamedTuple(x=int)) != compilerHash(Tuple(float)) - assert identityHash(NamedTuple(x=int)) != identityHash(NamedTuple(y=int)) - assert identityHash(NamedTuple(x=int, y=float)) != identityHash(NamedTuple(y=float, x=int)) + assert compilerHash(NamedTuple(x=int)) != compilerHash(NamedTuple(y=int)) + assert compilerHash(NamedTuple(x=int, y=float)) != compilerHash(NamedTuple(y=float, x=int)) def test_identity_of_dict(): - assert identityHash(Dict(int, float)) != identityHash(Dict(int, int)) - assert identityHash(Dict(int, float)) != identityHash(Dict(float, int)) + assert compilerHash(Dict(int, float)) != compilerHash(Dict(int, int)) + assert compilerHash(Dict(int, float)) != compilerHash(Dict(float, int)) def test_identity_of_oneof(): - assert identityHash(OneOf(None, int)) != identityHash(OneOf(None, float)) + assert compilerHash(OneOf(None, int)) != compilerHash(OneOf(None, float)) def test_identity_of_recursive_types(): @@ -230,15 +230,15 @@ def test_identity_of_recursive_types(): X3 = Forward("X") X3 = X3.define(TupleOf(OneOf(float, X3))) - assert identityHash(X2) == identityHash(X) - assert identityHash(X3) != identityHash(X) + assert compilerHash(X2) == compilerHash(X) + assert compilerHash(X3) != compilerHash(X) def test_identity_of_recursive_types_2(): X = Forward("X") X = X.define(TupleOf(OneOf(int, TupleOf(X)))) - identityHash(X) + compilerHash(X) def test_identity_of_recursive_types_produced_same_way(): @@ -246,9 +246,9 @@ def make(name, T): X = Forward(name) return X.define(TupleOf(OneOf(T, X))) - assert identityHash(make("X", int)) == identityHash(make("X", int)) - assert identityHash(make("X", int)) != identityHash(make("X", float)) - assert identityHash(make("X", int)) != identityHash(make("X2", int)) + assert compilerHash(make("X", int)) == compilerHash(make("X", int)) + assert compilerHash(make("X", int)) != compilerHash(make("X", float)) + assert compilerHash(make("X", int)) != compilerHash(make("X2", int)) def test_identity_of_lambda_functions(): @@ -258,15 +258,15 @@ def makeAdder(a): # these two have the same closure type assert makeAdder(10).ClosureType == makeAdder(11).ClosureType - assert identityHash(type(makeAdder(10))) == identityHash(type(makeAdder(10))) - assert identityHash(type(makeAdder(10))) == identityHash(type(makeAdder(11))) + assert compilerHash(type(makeAdder(10))) == compilerHash(type(makeAdder(10))) + assert compilerHash(type(makeAdder(10))) == compilerHash(type(makeAdder(11))) # these two are different - assert identityHash(type(makeAdder(10))) != identityHash(type(makeAdder(10.5))) + assert compilerHash(type(makeAdder(10))) != compilerHash(type(makeAdder(10.5))) def test_checkHash_works(): - assert checkHash({"x.py": "A = TupleOf(int)\n"}, 'x.A') == Hash(identityHash(TupleOf(int))) + assert checkHash({"x.py": "A = TupleOf(int)\n"}, 'x.A') == Hash(compilerHash(TupleOf(int))) def test_mutually_recursive_group_basic(): @@ -400,12 +400,12 @@ def test_hash_of_oneof(): OneOf(None, Tuple(int)((1,))), ] - hashes = set([identityHash(t) for t in oneOfs]) + hashes = set([compilerHash(t) for t in oneOfs]) assert len(hashes) == len(oneOfs) def test_identityHash_of_none(): - assert not Hash(identityHash(type(None))).isPoison() + assert not Hash(compilerHash(type(None))).isPoison() def test_identityHash_of_a_typefunction(): @@ -417,7 +417,7 @@ def L(t): L2(int) - assert identityHash(L1) == identityHash(L2) + assert compilerHash(L1) == compilerHash(L2) def test_hash_of_TP_produced_lambdas_with_different_closure_types(): @@ -431,8 +431,8 @@ def f(): returnIt(2) returnIt(2.0) - assert identityHash(returnIt(1)) == identityHash(returnIt(2)) - assert identityHash(returnIt(1)) != identityHash(returnIt(2.0)) + assert compilerHash(returnIt(1)) == compilerHash(returnIt(2)) + assert compilerHash(returnIt(1)) != compilerHash(returnIt(2.0)) def test_hash_of_native_lambdas_with_different_closure_types(): @@ -445,8 +445,8 @@ def f(): t2 = prepareArgumentToBePassedToCompiler(Entrypoint(returnIt(2))) t3 = prepareArgumentToBePassedToCompiler(Entrypoint(returnIt(2.0))) - assert identityHash(t1) == identityHash(t2) - assert identityHash(t1) != identityHash(t3) + assert compilerHash(t1) == compilerHash(t2) + assert compilerHash(t1) != compilerHash(t3) def test_checkHash_type_functions(): @@ -519,15 +519,15 @@ def test_checkHash_references_to_typed_free_objects(): def test_hash_of_builtins(): - assert not Hash(identityHash(isinstance)).isPoison() + assert not Hash(compilerHash(isinstance)).isPoison() def test_hash_of_classObj(): class C(Class, Final): x = Member(int) - assert not Hash(identityHash(Member(int))).isPoison() - assert not Hash(identityHash(C)).isPoison() + assert not Hash(compilerHash(Member(int))).isPoison() + assert not Hash(compilerHash(C)).isPoison() MODULE = { @@ -584,7 +584,7 @@ def deserializeTwiceAndCall(rep): def deserializeAndReturnHash(rep): - return identityHash(SerializationContext().deserialize(rep)) + return compilerHash(SerializationContext().deserialize(rep)) def deserializeTwiceAndConfirmEquivalent(rep): @@ -640,7 +640,7 @@ def test_serialization_of_anonymous_functions_preserves_references(): def test_hash_stability(): idHash = evaluateExprInFreshProcess({ 'x.py': 'from typed_python.compiler.native_compiler.native_ast import NamedCallTarget\n' - }, 'identityHash(x.NamedCallTarget)') + }, 'compilerHash(x.NamedCallTarget)') ser = returnSerializedValue({ 'x.py': 'from typed_python.compiler.native_compiler.native_ast import NamedCallTarget\n' }, 'x.NamedCallTarget', printComments=True) @@ -688,11 +688,11 @@ def test_identity_of_entrypointed_functions(): def f(): return 0 - assert identityHash(Function(f)) != identityHash(Entrypoint(f)) + assert compilerHash(Function(f)) != compilerHash(Entrypoint(f)) def test_identity_of_singleton_classes(): - assert identityHash(A) != identityHash(B) + assert compilerHash(A) != compilerHash(B) def test_type_walk_for_named_tuple_subclass(): @@ -703,6 +703,7 @@ def f(self): print(typeWalkRecord(N)) +<<<<<<< HEAD def test_module_hash_magic_value(): with tempfile.TemporaryDirectory() as tempDir: @@ -832,3 +833,20 @@ def test_type_function_identity_referencing_int_in_function_only(): identityHash(RefsAValueInStaticmethodOnClass(1)) != identityHash(RefsAValueInStaticmethodOnClass(2)) ) + + +def test_recursive_type_groups_separate(): + class C: + pass + + assert recursiveTypeGroup(C, "compiler", False) is None + assert recursiveTypeGroup(C, "identity", False) is None + + g1 = recursiveTypeGroup(C, "compiler", True) + assert g1 + assert recursiveTypeGroup(C, "identity", False) is None + + g2 = recursiveTypeGroup(C, "identity", True) + assert g2 + + print(g1, g2) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 79554a397..f3eaa64fa 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -45,13 +45,13 @@ Dict, Set, SerializationContext, EmbeddedMessage, serializeStream, deserializeStream, decodeSerializedObject, Forward, Final, Function, Entrypoint, TypeFunction, PointerTo, - SubclassOf, NotCompiled + SubclassOf, NotCompiled, resolveForwardDefinedType, identityHash ) from typed_python._types import ( - refcount, isRecursive, identityHash, buildPyFunctionObject, + refcount, isRecursive, compilerHash, buildPyFunctionObject, setFunctionClosure, typesAreEquivalent, recursiveTypeGroupDeepRepr, - recursiveTypeGroupRepr + recursiveTypeGroupRepr, isForwardDefined ) module_level_testfun = dummy_test_module.testfunction @@ -84,11 +84,6 @@ def method(self): pass -class ModuleLevelNamedTupleSubclass(NamedTuple(x=int)): - def f(self): - return self.x - - class ModuleLevelClass(Class, Final): def f(self): return "HI!" @@ -565,11 +560,13 @@ def f(): ts = SerializationContext({'f': f}) self.assertIs(ping_pong(f, ts), f) - def test_serialize_alternatives_as_types(self): + def test_serialize_recursive_alternative(self): A = Forward("A") - A = A.define(Alternative("A", X={'a': int}, Y={'a': A})) + A.define(Alternative("A", X={'a': int}, Y={'a': A})) + # A = resolveForwardDefinedType(A) - ts = SerializationContext({'A': A}) + + ts = SerializationContext() self.assertIs(ping_pong(A, ts), A) self.assertIs(ping_pong(A.X, ts), A.X) @@ -1440,6 +1437,52 @@ def f(x: B = C) -> B: self.assertEqual(f(), f2()) + def test_serialize_function_types(self): + @Function + def f(x: int): + return x + + sc = SerializationContext() + + fType = sc.deserialize(sc.serialize(type(f))) + + assert fType is type(f) + + def test_serialize_empty_forward(self): + F1 = Forward("F1") + F2 = Forward("F2") + F1.define(ListOf(F2)) + + assert F1.get().ElementType is F2 + assert F2.get() is None + + F1_copy, F2_copy = SerializationContext().deserialize( + SerializationContext().serialize((F1, F2)) + ) + + assert F1_copy is not F1 + assert F2_copy is not F2 + + assert F1_copy.get().ElementType is F2_copy + assert F2_copy.get() is None + assert F1.__name__ == F1_copy.__name__ + assert F2.__name__ == F2_copy.__name__ + + assert isForwardDefined(F1_copy) + assert isForwardDefined(F1_copy.get()) + assert isForwardDefined(F1_copy.get().ElementType) + + F2.define(OneOf(None, F1)) + F2_copy.define(OneOf(None, F1_copy)) + + F2r = F2_copy.resolve() + + assert F2_copy.get().Types[1].get().ElementType is F2_copy + assert F2r.Types[1].ElementType is F2r + + assert F1.resolve() is F1_copy.resolve() + assert F2.resolve() is F2_copy.resolve() + def test_serialize_typed_classes(self): sc = SerializationContext() @@ -1491,7 +1534,7 @@ def getSelf(self) -> B: B2 = sc.deserialize(sc.serialize(B)) - assert identityHash(B) == identityHash(B2) + assert compilerHash(B) == compilerHash(B2) instance = B2() @@ -1621,7 +1664,7 @@ def aFunction(self, x): aFunction2 = sc.deserialize(sc.serialize(aFunction)) - assert identityHash(aFunction) == identityHash(aFunction2) + assert compilerHash(aFunction) == compilerHash(aFunction2) def test_serialize_subclasses(self): sc = SerializationContext() @@ -1877,56 +1920,6 @@ def test_serialize_module_level_class(self): sc.serialize(ModuleLevelClass), ) - def test_serialize_unnamed_subclass_of_named_tuple(self): - class SomeNamedTuple(NamedTuple(x=int)): - def f(self): - return self.x - - sc = SerializationContext() - - self.assertEqual( - sc.deserialize(sc.serialize(SomeNamedTuple))(x=10).f(), - 10 - ) - - self.assertEqual( - sc.deserialize(sc.serialize(SomeNamedTuple(x=10))).f(), - 10 - ) - - def test_serialize_named_subclass_of_named_tuple(self): - sc = SerializationContext() - - self.assertEqual( - ModuleLevelNamedTupleSubclass.__module__, - "typed_python.types_serialization_test" - ) - - self.assertEqual( - ModuleLevelNamedTupleSubclass.__name__, - "ModuleLevelNamedTupleSubclass" - ) - - self.assertIs( - sc.deserialize(sc.serialize(ModuleLevelNamedTupleSubclass)), - ModuleLevelNamedTupleSubclass - ) - - self.assertIs( - type(sc.deserialize(sc.serialize(ModuleLevelNamedTupleSubclass()))), - ModuleLevelNamedTupleSubclass - ) - - self.assertIs( - type(sc.deserialize(sc.serialize([ModuleLevelNamedTupleSubclass()]))[0]), - ModuleLevelNamedTupleSubclass - ) - - self.assertIs( - sc.deserialize(sc.serialize(ModuleLevelNamedTupleSubclass(x=10))).f(), - 10 - ) - def test_serialize_builtin_tp_functions(self): sc = SerializationContext() @@ -2066,7 +2059,7 @@ def __add__(self, other: Base) -> Base: assert isinstance(Base() + Base() + Base(), Child) def test_serialize_self_referencing_class_basic(self): - def g(x): + def g2(x): return 10 @TypeFunction @@ -2078,10 +2071,20 @@ def s(self): @Entrypoint def g(self): - return g(10) + return g2(10) return C_ + print(recursiveTypeGroupRepr(C(int), "identity")) + print(recursiveTypeGroupRepr(C(int), "compiler")) + from typed_python._types import checkForHashInstability + checkForHashInstability() + #from typed_python._types import typeWalkRecord + #print(typeWalkRecord(type(C(int).s), 'identity')) + #print(typeWalkRecord(type(C(int).s), 'compiler')) + + return + c = callFunctionInFreshProcess(C(int), ()) assert c.g() == 10 @@ -2126,8 +2129,8 @@ def test_names_of_builtin_alternatives(self): assert sc.deserialize(sc.serialize(TupleOf(native_ast.Type))) is TupleOf(native_ast.Type) assert sc.deserialize(sc.serialize(NamedCallTarget)) is NamedCallTarget - assert len(sc.serialize(native_ast.Type)) < 100 - assert len(sc.serialize(TupleOf(native_ast.Type))) < 100 + assert len(sc.serialize(native_ast.Type)) < 200 + assert len(sc.serialize(TupleOf(native_ast.Type))) < 200 def test_badly_named_module_works(self): sc = SerializationContext() @@ -2300,10 +2303,14 @@ def test_can_deserialize_untyped_forward_class_methods_1(self): @Cls.define class Cls: - # recall that regular classes ignore their annotations + # recall that regular classes ignore their annotations. This one will be a + # 'Forward' holding Cls. def f(self) -> Cls: return "HI" + with open("a.dat", "wb") as f: + f.write(SerializationContext().serialize((Cls, ()))) + assert callFunctionInFreshProcess(Cls, ()).f() == "HI" def test_can_deserialize_untyped_forward_class_methods_2(self): @@ -2320,7 +2327,7 @@ def f(self) -> Cls: callFunctionInFreshProcess(getCls, ()) - def test_can_deserialize_forward_class_methods_tp_class(self): + def test_can_deserialize_forward_class_methods_tp_class_2(self): Cls = Forward("Cls") @Cls.define @@ -2353,7 +2360,7 @@ def f(self): Cls2 = type(callFunctionInFreshProcess(Cls, ())) - assert identityHash(Cls).hex() == identityHash(Cls2).hex() + assert compilerHash(Cls).hex() == compilerHash(Cls2).hex() @pytest.mark.skipif( "sys.version_info.minor >= 8", @@ -2365,11 +2372,11 @@ def makeFunction(): def f(x: float): return x + 1 - return (f, identityHash(f)) + return (f, compilerHash(f)) f, identityHashOfF = callFunctionInFreshProcess(makeFunction, ()) - assert identityHash(f) == identityHashOfF + assert compilerHash(f) == identityHashOfF @pytest.mark.skipif( "sys.version_info.minor >= 8", @@ -2381,11 +2388,11 @@ def makeFunction(): def f(x=None): return x + 1 - return (f, identityHash(f)) + return (f, compilerHash(f)) f, identityHashOfF = callFunctionInFreshProcess(makeFunction, ()) - assert identityHash(f) == identityHashOfF + assert compilerHash(f) == identityHashOfF @pytest.mark.skip(reason='broken') def test_deserialize_untyped_class_in_forward_retains_identity(self): @@ -2418,7 +2425,7 @@ def pad(x): for i in range(len(l)): print(pad(l[i]), " " if l[i] == r[i] else " != ", pad(r[i])) - assert identityHash(Cls) == identityHash(Cls2) + assert compilerHash(Cls) == compilerHash(Cls2) def test_deserialize_tp_class_retains_identity(self): Cls = Forward("Cls") @@ -2431,7 +2438,7 @@ def f(self) -> Cls: Cls2 = type(callFunctionInFreshProcess(Cls, ())) - assert identityHash(Cls) == identityHash(Cls2) + assert compilerHash(Cls) == compilerHash(Cls2) def test_call_method_dispatch_on_two_versions_of_same_class_with_recursion_defined_in_host(self): Base = Forward("Base") @@ -2461,13 +2468,14 @@ def callF(x): aChild2 = SerializationContext().deserialize(someBytes) - assert identityHash(aChild) == identityHash(aChild2) + assert compilerHash(aChild) == compilerHash(aChild2) if callF(aChild) == callF(aChild2): return "OK" else: return "FAILED" + # assert deserializeAndCall(aChildBytes) == "OK" assert callFunctionInFreshProcess(deserializeAndCall, (aChildBytes,)) == "OK" def test_call_method_dispatch_on_two_versions_of_self_referential_class_produced_differently(self): @@ -2495,8 +2503,8 @@ def f(self, x) -> int: child1, callF1 = callFunctionInFreshProcess(deserializeAndCall, ()) child2, callF2 = callFunctionInFreshProcess(deserializeAndCall, ()) - assert identityHash(child1) == identityHash(child2) - assert identityHash(callF1) == identityHash(callF2) + assert compilerHash(child1) == compilerHash(child2) + assert compilerHash(callF1) == compilerHash(callF2) assert type(child1) is type(child2) assert type(callF1) is type(callF2) @@ -2529,8 +2537,8 @@ def callF(x: Child): Base1, callF1 = callFunctionInFreshProcess(deserializeAndCall, ()) Base2, callF2 = callFunctionInFreshProcess(deserializeAndCall, ()) - assert identityHash(Base1) == identityHash(Base2) - assert identityHash(callF1) == identityHash(callF2) + assert compilerHash(Base1) == compilerHash(Base2) + assert compilerHash(callF1) == compilerHash(callF2) assert Base1 is Base2 assert type(callF1) is type(callF2) @@ -2546,16 +2554,43 @@ def test_identity_hash_of_lambda_doesnt_change_serialization(self): ser1 = s.serialize(f) - identityHash(f) + compilerHash(f) ser2 = s.serialize(f) assert ser1 == ser2 + def test_deserialize_external_class_with_int_ref(self): + def deserializeAndCall(x): + class B(Class): + def f(self): + return x + + return B + + B1 = deserializeAndCall(1) + print(B1.f.overloads[0].globals['x']) + return; + + assert callFunctionInFreshProcess(deserializeAndCall, (1,)) is deserializeAndCall(1) + assert callFunctionInFreshProcess(deserializeAndCall, (1,)) is not deserializeAndCall(2) + assert callFunctionInFreshProcess(deserializeAndCall, (2,)) is deserializeAndCall(2) + + def test_deserialize_external_subclass(self): + class Base(Class): + pass + + def deserializeAndCall(): + class Child(Base): + pass + + return Child + + assert callFunctionInFreshProcess(deserializeAndCall, ()) is deserializeAndCall() + def test_call_method_dispatch_on_two_versions_of_same_class_with_recursion(self): - Base = Forward("Base") + Base = Forward(lambda: Base) - @Base.define class Base(Class): def blah(self) -> Base: return self @@ -2567,8 +2602,6 @@ def f(self, x) -> int: def callF(x: Base): return x.f(10) - identityHash(Base) - def deserializeAndCall(): class Child(Base, Final): def f(self, x) -> int: @@ -2792,7 +2825,7 @@ def makeC(): path = os.path.join(tempdir, "asdf.py") CONTENTS = ( - "from typed_python import Entrypoint, ListOf, Class, Final\n" + "from typed_python import Entrypoint, ListOf, Class, Final, isForwardDefined, Member, forwardDefinitionsFor\n" "class C(Class, Final):\n" " @staticmethod\n" " @Entrypoint\n" @@ -2858,7 +2891,7 @@ def test_serialization_independent_of_whether_function_is_hashed(self): s1 = s.serialize(moduleLevelFunctionUsedByExactlyOneSerializationTest) - identityHash(moduleLevelFunctionUsedByExactlyOneSerializationTest) + compilerHash(moduleLevelFunctionUsedByExactlyOneSerializationTest) s2 = s.serialize(moduleLevelFunctionUsedByExactlyOneSerializationTest) diff --git a/typed_python/types_test.py b/typed_python/types_test.py index 721c62b86..a015382e7 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -31,7 +31,9 @@ Float32, SubclassOf, TupleOf, ListOf, OneOf, Tuple, NamedTuple, Dict, ConstDict, Alternative, serialize, deserialize, Class, - TypeFilter, Function, Forward, Set, PointerTo, Entrypoint, Final + Function, Forward, Set, PointerTo, + Entrypoint, + Final ) from typed_python.type_promotion import ( computeArithmeticBinaryResultType, floatness, bitness, isSignedInt @@ -41,8 +43,7 @@ AnAlternative = Alternative("AnAlternative", X={'x': int}) -AForwardAlternative = Forward("AForwardAlternative") -AForwardAlternative.define(Alternative("AForwardAlternative", Y={}, X={'x': AForwardAlternative})) +AForwardAlternative = Alternative("AForwardAlternative", Y={}, X={'x': lambda: AForwardAlternative}) def typeFor(t): @@ -205,58 +206,6 @@ def check_expected_performance(self, elapsed, expected=1.0): .format(expected=expected, elapsed=elapsed) ) - def test_object_binary_compatibility(self): - ibc = _types.isBinaryCompatible - - self.assertTrue(ibc(type(None), type(None))) - self.assertTrue(ibc(Int8, Int8)) - - NT = NamedTuple(a=int, b=int) - - class X(NamedTuple(a=int, b=int)): - pass - - class Y(NamedTuple(a=int, b=int)): - pass - - self.assertTrue(ibc(X, X)) - self.assertTrue(ibc(X, Y)) - self.assertTrue(ibc(X, NT)) - self.assertTrue(ibc(Y, NT)) - self.assertTrue(ibc(NT, Y)) - - self.assertFalse(ibc(OneOf(int, float), OneOf(float, int))) - self.assertTrue(ibc(OneOf(int, X), OneOf(int, Y))) - - def test_binary_compatibility_incompatible_alternatives(self): - ibc = _types.isBinaryCompatible - - A1 = Alternative("A1", X={'a': int}, Y={'b': float}) - A2 = Alternative("A2", X={'a': int}, Y={'b': str}) - - self.assertTrue(ibc(A1, A1.X)) - self.assertTrue(ibc(A1, A1.Y)) - self.assertTrue(ibc(A1.Y, A1.Y)) - self.assertTrue(ibc(A1.Y, A1)) - self.assertTrue(ibc(A1.X, A1)) - self.assertFalse(ibc(A1.X, A1.Y)) - - self.assertFalse(ibc(A1, A2)) - self.assertFalse(ibc(A1.X, A2.X)) - self.assertFalse(ibc(A1.Y, A2.Y)) - - def test_binary_compatibility_compatible_alternatives(self): - ibc = _types.isBinaryCompatible - - A1 = Alternative("A1", X={'a': int}, Y={'b': float}) - A2 = Alternative("A2", X={'a': int}, Y={'b': float}) - - self.assertTrue(ibc(A1.X, A2.X)) - self.assertTrue(ibc(A1.Y, A2.Y)) - - self.assertFalse(ibc(A1.X, A2.Y)) - self.assertFalse(ibc(A1.Y, A2.X)) - def test_name_of_type_as_value_instances(self): T1 = Alternative("T1") @@ -389,31 +338,6 @@ def test_one_of_alternative(self): self.assertEqual(Ox(X.V(a=10)), X.V(a=10)) - def test_one_of_py_subclass(self): - class X(NamedTuple(x=int)): - def f(self): - return self.x - - Ox = OneOf(None, X) - - self.assertEqual(NamedTuple(x=int)(x=10).x, 10) - self.assertEqual(X(x=10).f(), 10) - self.assertEqual(Ox(X(x=10)).f(), 10) - - def test_one_of_distinguishes_py_subclasses(self): - class X(NamedTuple(x=int)): - def f(self): - return self.x - - class X2(NamedTuple(x=int)): - def f(self): - return self.x + 2 - - XorX2 = OneOf(X2, X) - - self.assertTrue(isinstance(XorX2(X()), X)) - self.assertTrue(isinstance(XorX2(X2()), X2)) - def test_function_as_type_arg(self): @Function def f(x: int): @@ -608,24 +532,6 @@ def test_one_of_flattening(self): def test_one_of_order_matters(self): self.assertNotEqual(OneOf(1.0, 2.0), OneOf(2.0, 1.0)) - def test_type_filter(self): - EvenInt = TypeFilter(int, lambda i: i % 2 == 0) - - self.assertTrue(isinstance(2, EvenInt)) - self.assertFalse(isinstance(1, EvenInt)) - self.assertFalse(isinstance(2.0, EvenInt)) - - EvenIntegers = TupleOf(EvenInt) - - e = EvenIntegers(()) - e2 = e + (2, 4, 0) - - with self.assertRaises(TypeError): - EvenIntegers((1,)) - - with self.assertRaises(TypeError): - e2 + (1,) - def test_tuple_of_one_of_fixed_size(self): t = TupleOf(OneOf(0, 1, 2, 3, 4)) @@ -653,9 +559,24 @@ def test_tuple_of_one_of_multi(self): self.assertEqual(tuple(typedThings), someThings) + def test_alternative_as_value(self): + A = Alternative( + "A", + a0=dict(x_0=str, x_1=TupleOf(ConstDict(int, int))), + ) + + V1 = Value(A.a0(x_0='hi')) + V2 = Value(A.a0(x_0='hi2')) + V3 = Value(A.a0(x_0='hi', x_1=[{1: 2}])) + V4 = Value(A.a0(x_0='hi', x_1=[{1: 3}])) + + assert V1 is not V2 + assert V1 is not V3 + assert V3 is not V4 + def test_compound_oneof(self): producer = RandomValueProducer() - producer.addEvenly(1000, 2) + producer.addEvenly(20, 2) for _ in range(1000): vals = (producer.pickRandomly(), producer.pickRandomly(), producer.pickRandomly()) @@ -1130,6 +1051,8 @@ def test_const_dict_iter_str(self): def test_alternative_matcher_type(self): A = Alternative("A", X=dict(x=int)) + assert type(A.X()).__name__ == "X" + assert type(A.X().matches).__qualname__ == "AlternativeMatcher(X)" assert type(A.X().matches).__name__ == "AlternativeMatcher(X)" assert type(A.X().matches).__typed_python_category__ == "AlternativeMatcher" assert type(A.X().matches).Alternative is A @@ -1236,6 +1159,9 @@ def A_delattr(s, n): __delattr__=A_delattr, ) + from typed_python import resolveForwardDefinedType + A = resolveForwardDefinedType(A) + self.assertEqual(A.a().__bool__(), False) self.assertEqual(bool(A.a()), False) self.assertEqual(A.b().__bool__(), True) @@ -1318,26 +1244,54 @@ def A_delattr(s, n): d = dir(A.a()) self.assertGreater(len(d), 50) - A2_items = dict() + def test_alternative_magic_methods_2(self): + A2_attrs = {"q": "value-q", "z": "value-z"} + + def A2_getattr(s, n): + if n not in A2_attrs: + raise AttributeError(f"no attribute {n}") + return A2_attrs[n] + + def A2_setattr(s, n, v): + A2_attrs[n] = v + + def A2_delattr(s, n): + A2_attrs.pop(n, None) + + A2_items = {'blah': 'blah'} def A2_setitem(self, i, v): A2_items[i] = v - return 0 - A2 = Alternative("A2", a={'a': int}, b={'b': str}, - __getattribute__=A_getattr, - __setattr__=A_setattr, - __delattr__=A_delattr, - __dir__=lambda self: list(A_attrs.keys()), - __getitem__=lambda self, i: A2_items.get(i, i), - __setitem__=A2_setitem - ) + def A2_delitem(self, i): + A2_items.pop(i) + + A2 = Alternative( + "A2", + a={'a': int}, + b={'b': str}, + __getattribute__=A2_getattr, + __setattr__=A2_setattr, + __delattr__=A2_delattr, + __dir__=lambda self: list(A2_attrs.keys()), + __getitem__=lambda self, i: A2_items.get(i, i), + __setitem__=A2_setitem, + __delitem__=A2_delitem + ) + + assert 'blah' in A2_items + del A2.b()['blah'] + assert 'blah' not in A2_items + self.assertEqual(A2.b().q, "value-q") + A2.b().q = "changedvalue for q" self.assertEqual(A2.b().q, "changedvalue for q") A2.a().Name = "can change Name" self.assertEqual(A2.b().Name, "can change Name") - self.assertEqual(dir(A2.b()), ["Name", "q"]) + self.assertEqual(dir(A2.b()), ["Name", "q", "z"]) + self.assertEqual(A2.b()[123], 123) + A2.b()[123] = 7 self.assertEqual(A2.b()[123], 7) @@ -1798,37 +1752,6 @@ def test_const_dict_of_tuple(self): self.assertTrue(k in x) x[k] - def test_conversion_of_binary_compatible(self): - class T1(NamedTuple(a=int)): - pass - - class T2(NamedTuple(a=int)): - pass - - class T1Comp(NamedTuple(d=ConstDict(str, T1))): - pass - - class T2Comp(NamedTuple(d=ConstDict(str, T1))): - pass - - self.assertTrue(_types.isBinaryCompatible(T1Comp, T2Comp)) - self.assertTrue(_types.isBinaryCompatible(T1, T2)) - - def test_binary_compatible_nested(self): - def make(): - class Interior(NamedTuple(a=int)): - pass - - class Exterior(NamedTuple(a=Interior)): - pass - - return Exterior - - E1 = make() - E2 = make() - - self.assertTrue(_types.isBinaryCompatible(E1, E2)) - def test_python_objects_in_tuples(self): class NormalPyClass: pass @@ -2415,57 +2338,6 @@ def test_mutable_dict_iteration_order(self): del d[1] self.assertEqual(list(d), [10, 2]) - def test_simplicity(self): - isSimple = _types.isSimple - - self.assertTrue(isSimple(int)) - self.assertTrue(isSimple(Int32())) - self.assertTrue(isSimple(Int16())) - self.assertTrue(isSimple(Int8())) - self.assertTrue(isSimple(UInt64())) - self.assertTrue(isSimple(UInt32())) - self.assertTrue(isSimple(UInt16())) - self.assertTrue(isSimple(UInt8())) - self.assertTrue(isSimple(str)) - self.assertTrue(isSimple(bytes)) - self.assertTrue(isSimple(bool)) - self.assertTrue(isSimple(float)) - - class C(Class): - pass - - self.assertFalse(isSimple(C)) - - self.assertTrue(isSimple(ListOf(int))) - self.assertFalse(isSimple(ListOf(C))) - - self.assertTrue(isSimple(TupleOf(int))) - self.assertFalse(isSimple(TupleOf(C))) - - self.assertTrue(isSimple(ConstDict(int, int))) - self.assertFalse(isSimple(ConstDict(C, int))) - self.assertFalse(isSimple(ConstDict(int, C))) - - self.assertTrue(isSimple(Dict(int, int))) - self.assertFalse(isSimple(Dict(C, int))) - self.assertFalse(isSimple(Dict(int, C))) - - self.assertTrue(isSimple(Set(int))) - self.assertFalse(isSimple(Set(C))) - - self.assertFalse(isSimple(Alternative("Alternative"))) - - self.assertTrue(isSimple(NamedTuple(x=int))) - self.assertFalse(isSimple(NamedTuple(x=C))) - - X = Forward("X") - X = X.define(Alternative("X", X={'x': X}, Y={'i': int})) - self.assertFalse(isSimple(X)) - self.assertFalse(isSimple(NamedTuple(x=X))) - - self.assertFalse(isSimple(OneOf(int, X))) - self.assertTrue(isSimple(OneOf(int, float))) - def test_oneof_picks_best_choice(self): T = OneOf(float, int, bool) diff --git a/typed_python/util.hpp b/typed_python/util.hpp index 5f30cc8c0..f60e6a95c 100644 --- a/typed_python/util.hpp +++ b/typed_python/util.hpp @@ -430,6 +430,15 @@ inline bool startsWith(std::string name, std::string prefix) { } +inline bool endsWith(std::string name, std::string suffix) { + if (name.size() < suffix.size()) { + return false; + } + + return name.substr(name.size() - suffix.size(), suffix.size()) == suffix; +} + + class PyErrorStasher { public: PyErrorStasher() {